diff --git a/.clang-format b/.clang-format new file mode 100755 index 0000000000..1690505fd8 --- /dev/null +++ b/.clang-format @@ -0,0 +1,124 @@ +# Style file for MLSE Libraries based on the modified rocBLAS style + +# Common settings +BasedOnStyle: WebKit +TabWidth: 4 +IndentWidth: 4 +UseTab: Never +ColumnLimit: 100 + +# Other languages JavaScript, Proto + +--- +Language: Cpp + +# http://releases.llvm.org/6.0.1/tools/clang/docs/ClangFormatStyleOptions.html#disabling-formatting-on-a-piece-of-code +# int formatted_code; +# // clang-format off +# void unformatted_code ; +# // clang-format on +# void formatted_code_again; + +DisableFormat: false +Standard: Cpp11 + +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: false +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: true +BinPackArguments: false +BinPackParameters: false + +# Configure each individual brace in BraceWrapping +BreakBeforeBraces: Custom +# Control of individual brace wrapping cases +BraceWrapping: { + AfterCaseLabel: 'true' + AfterClass: 'true' + AfterControlStatement: 'true' + AfterEnum : 'true' + AfterFunction : 'true' + AfterNamespace : 'true' + AfterStruct : 'true' + AfterUnion : 'true' + BeforeCatch : 'true' + BeforeElse : 'true' + IndentBraces : 'false' +# AfterExternBlock : 'true' +} + +#BreakAfterJavaFieldAnnotations: true +#BreakBeforeInheritanceComma: false +#BreakBeforeBinaryOperators: None +#BreakBeforeTernaryOperators: true +#BreakConstructorInitializersBeforeComma: true +#BreakStringLiterals: true + +CommentPragmas: '^ IWYU pragma:' +#CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +SpaceBeforeCpp11BracedList: false +DerivePointerAlignment: false +ExperimentalAutoDetectBinPacking: false +ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +IndentCaseLabels: false +IndentPPDirectives: None +#FixNamespaceComments: true +IndentWrappedFunctionNames: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: '' +MacroBlockEnd: '' +#JavaScriptQuotes: Double +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: All +ObjCBlockIndentWidth: 4 +#ObjCSpaceAfterProperty: true +#ObjCSpaceBeforeProtocolList: true +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 + +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Left +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: Never +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +#SpaceAfterTemplateKeyword: true +#SpaceBeforeInheritanceColon: true + +#SortUsingDeclarations: true +SortIncludes: true + +# Comments are for developers, they should arrange them +ReflowComments: false + +#IncludeBlocks: Preserve +--- diff --git a/CMakeLists.txt b/CMakeLists.txt index d404539afb..4128f31f9e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -179,7 +179,7 @@ else() virtualenv_install(${Tensile_TEST_LOCAL_PATH}) message (STATUS "using local Tensile from ${Tensile_TEST_LOCAL_PATH}, copied to ${Tensile_ROOT}") else() - message( FATAL_ERROR "Local Tensile is required." ) + virtualenv_install(${CMAKE_SOURCE_DIR}/tensilelite) endif() message(STATUS "Adding ${VIRTUALENV_HOME_DIR} to CMAKE_PREFIX_PATH") list(APPEND CMAKE_PREFIX_PATH ${VIRTUALENV_HOME_DIR}) diff --git a/README.md b/README.md index 2c26c3facd..c7f5b3c842 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,7 @@ cd hipBLASLt # -d|--dependencies - install build dependencies # -c|--clients - build library clients too (combines with -i & -d) # -g|--debug - build with debug flag -# -t|--test_local_path - build with local tensile codebase -./install.sh -dc -t +./install.sh -dc ``` #### CMake diff --git a/tensilelite/LICENSE.md b/tensilelite/LICENSE.md new file mode 100644 index 0000000000..f76a157728 --- /dev/null +++ b/tensilelite/LICENSE.md @@ -0,0 +1,7 @@ +Copyright 2016-2020 Advanced Micro Devices, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tensilelite/MANIFEST.in b/tensilelite/MANIFEST.in new file mode 100644 index 0000000000..ac359c27db --- /dev/null +++ b/tensilelite/MANIFEST.in @@ -0,0 +1,4 @@ +recursive-include Tensile *.py *.h *.hpp *.cpp *.txt *.cmake *.yaml +recursive-include Tensile/CustomKernels *.s +include Tensile/bin/Tensile Tensile/bin/TensileCreateLibrary +include requirements.txt diff --git a/tensilelite/Tensile/Activation.py b/tensilelite/Tensile/Activation.py new file mode 100644 index 0000000000..c45343a040 --- /dev/null +++ b/tensilelite/Tensile/Activation.py @@ -0,0 +1,1040 @@ +################################################################################ +# Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +import ctypes +from copy import deepcopy +import math +import re +import struct +from collections import OrderedDict + +from .TensileInstructions import Module, TextBlock, HolderContainer, RegisterContainer, \ + VCC, vgpr, sgpr, Holder +from .TensileInstructions.Enums import * +from .TensileInstructions.Instructions import * +from .Common import printExit, printWarning + +################################################################################ +# How to add an activation +# 1. Add a new type in ActivationType +# 2. Create a new getXXXAssembly function in class Activation +# 3. Add if-else condition in generateAssembly in class Activation +# 4. Add if-else condition in generateInlineAssemblyBody in class +# ActivationInline +# +# Helper function(s) +# 1. getRegAndInitAssembly +# ``` +# getRegAndInitAssembly(, +# , +# , +# , +# , +# ) +# ``` +# Returns +# 1. sgprinf: The original checkOut return value +# 2. regInitStr: The init instruction string +# +# Example, +# ``` +# sgprinf, regInitStr = self.getRegAndInitAssembly('s', False, 1, \ +# "0x3f4c422a", "FloatGeluK0", "float gelu k0") +# ``` +# this will generate ``regInitStr`` as +# ``` +# s_mov_b32 sXX, "0x3f4c422a" // float16 max +# ``` +# if the key "FloatGeluK0" is not found in sgprDict +# 2. class ActivationRegisterPool +# A wrapper of RegisterPool. All the checkOut-ed registers will be checkIn-ed +# at the end of the numBatches for loop. +# When ActivationType is set to 'all', the registers will be checkIn-ed after +# activation's gwvw for loop. +################################################################################ + +################################################################################ +# This is the ActivationType class +# stringList: +# This list stores the names of extra arguments, e.g. +# y = (x > 0) ? x : x * alpha +# lookup: +# This dict stores the supported activation types as keys and number of +# arguments as values. Insert any new type before 'none' and 'all'. The +# sequence of the table should match the enum in Activation.hpp. +# +# To add an activation type, see the instruction in Activation.py. +################################################################################ + +class ActivationAvailable: + def __init__(self, canHalf=False, canSingle=False, canDouble=False, canBFloat16=False, canInt8=False, canInt16=False, canInt32=False): + self.half = canHalf + self.single = canSingle + self.double = canDouble + self.bfloat16 = canBFloat16 + self.int8 = canInt8 + self.int16 = canInt16 + self.int32 = canInt32 + +class ActivationTypeRegister: + def __init__(self, name, extraArgs, canHalf=False, canSingle=False, canDouble=False, canBFloat16=False, canInt8=False, canInt16=False, canInt32=False): + self.name = name + self.extraArgs = extraArgs + self.can = ActivationAvailable(canHalf, canSingle, canDouble, canBFloat16, canInt8, canInt16, canInt32) + def typeAvailable(self, dataType): + if dataType.isHalf() and self.can.half: + return True + elif dataType.isSingle() and self.can.single: + return True + elif dataType.isDouble() and self.can.double: + return True + elif dataType.isBFloat16() and self.can.bfloat16: + return True + elif dataType.isInt8() and self.can.int8: + return True + elif dataType.isInt32() and self.can.int32: + return True + return False + +class ActivationType: + stringList = ['alpha', 'beta', 'gamma', 'delta' ] + # Exp is only for verification. So we will not return exp in the supported list. + # Half,Single,Double,BFloat16, Int8, Int16, Int32 + lookupVeri = OrderedDict([('exp', ActivationTypeRegister('exp', 0, True, True, False, False, False, False, False)) ]) + + # Note: The BFloat16 gemm uses Single type activations. The int8 gemm uses int32 type activations. + # Half,Single,Double,BFloat16, Int8, Int16, Int32 + lookup = OrderedDict([('abs', ActivationTypeRegister('abs', 0, True, True, True, True, False, False, True)), \ + ('clippedrelu', ActivationTypeRegister('clippedrelu', 2, True, True, True, False, False, False, True)), \ + ('gelu', ActivationTypeRegister('gelu', 0, True, True, False, False, False, False, False)), \ + ('leakyrelu', ActivationTypeRegister('leakyrelu', 1, True, True, True, False, False, False, True)), \ + ('relu', ActivationTypeRegister('relu', 0, True, True, True, False, False, False, True)), \ + ('sigmoid', ActivationTypeRegister('sigmoid', 0, True, True, False, False, False, False, False)), \ + ('tanh', ActivationTypeRegister('tanh', 2, True, True, False, False, False, False, False)), \ + ('none', ActivationTypeRegister('none', 0, True, True, True, True, True, True, True)), \ + ('all', ActivationTypeRegister('all', 0)) ]) + def __init__(self, value): + if isinstance(value, str): + strValue = value.lower() + if strValue in self.lookup: + self.value = strValue + elif strValue in self.lookupVeri: + self.value = strValue + else: + raise RuntimeError("Unrecognized activation type %s"%value) + elif isinstance(value, ActivationType): + self.value = value.value + else: + raise RuntimeError("Unrecognized input type %s, should be string or ActivationType"%str(value)) + def getAdditionalArgNum(self): + if self.value == 'all': + maxArgNum = 0 + for key, activationInst in self.lookup.items(): + maxArgNum = max(maxArgNum, activationInst.extraArgs) + return maxArgNum + elif self.value in self.lookup: + return self.lookup[self.value].extraArgs + return 0 + def getAdditionalArgStringList(self, addPrefix=True): + list = [] + for i in range(0, self.getAdditionalArgNum()): + if addPrefix: + list.append("activation" + self.stringList[i].capitalize()) + else: + list.append(self.stringList[i]) + return list + @classmethod + def getEnumIndex(cls, enumStr): + return list(cls.lookup.keys()).index(enumStr) + @classmethod + def getEnumStrList(cls, dataType, includeNone = True): + enumList = [] + for key, activationInst in cls.lookup.items(): + if (((key != 'none') or includeNone) and (key != 'all')): + if activationInst.typeAvailable(dataType): + enumList.append(key) + if not enumList: + printWarning("No available activation for this data type %s.\n"%str(dataType)) + return enumList + def state(self): return self.value.capitalize() + def __repr__(self): + return self.__str__() + def __str__(self): + return self.value.capitalize() + def __eq__(self, other): + if isinstance(other, str): + return self.value == other.lower() + elif isinstance(other, ActivationType): + return self.value == other.value + else: + raise RuntimeError("Unrecognized type in rhs, should be string or ActivationType") + def toEnum(self): + return self.value.capitalize() + +ActivationMagicNumbers = {"FloatGeluK0": 0x3f4c422a, \ + "FloatGeluK1": 0x3d372713, \ + "Float16GeluK1": 0x29b9 } + +# float32 union +class floatUnion(ctypes.Union): + _fields_ = [('u', ctypes.c_uint), ('f', ctypes.c_float)] + +class ActivationModule: + ################################################################################ + ################################################################################ + ### + ### Public Functions + ### + ################################################################################ + ################################################################################ + + def __init__(self) -> None: + self.usePK = True + self.vgprCounter = 0 + self.sgprCounter = 0 + self.saturateI8 = False + self.vgprPrefixFormat = "" + + # Public function + def getModule(self, cDataType, activationType, vgprIdx): + module = "" + self.resetGprCounter() + if (activationType == 'abs'): + module = self.getAbsModule(cDataType, vgprIdx) + elif (activationType == 'clippedrelu'): + module = self.getClippedReluModule(cDataType, vgprIdx, "activationAlpha", "activationBeta") + elif (activationType == 'exp'): + module = self.getExpModule(cDataType, vgprIdx) + elif (activationType == 'gelu'): + module = self.getGeluModule(cDataType, vgprIdx) + elif (activationType == 'leakyrelu'): + module = self.getLeakyReluModule(cDataType, vgprIdx, "activationAlpha") + elif (activationType == 'relu'): + module = self.getReluModule(cDataType, vgprIdx) + elif (activationType == 'sigmoid'): + module = self.getSigmoidModule(cDataType, vgprIdx) + elif (activationType == 'tanh'): + module = self.getTanhModule(cDataType, vgprIdx, "activationAlpha", "activationBeta") + elif (activationType == 'none'): + module = Module("No activation") + else: + module = Module("%s not implemented"%activationType) + + return self.postProcess(cDataType, module) + + def getAllGprUsage(self, cDataType) -> dict: + usage = {} + enumList = ActivationType.getEnumStrList(cDataType) + for enumStr in enumList: + _ = self.getModule(cDataType, enumStr, 0) # dummy vgpr + usage[enumStr] = {"vgpr": self.vgprCounter, "sgpr": self.sgprCounter} + return usage + + def postProcess(self, cDataType, module): + CombineInstructions(module) + module = ConvertCoeffToHex(module, cDataType, self.usePK) + return module + + def assignGpr(self, module, vgprIdx, sgprIdx): + patternPrefix = ["v", "s"] + gprIdx = [vgprIdx, sgprIdx] + for idx, pf in enumerate(patternPrefix): + module = HolderToGpr(module, gprIdx[idx], pf) + return module + + def setUsePK(self, usePK): + self.usePK = usePK + + def setSaturationForInt8(self, sat): + self.saturateI8 = sat + + def setVgprPrefixFormat(self, formatting): + self.vgprPrefixFormat = formatting + + ################################################################################ + ################################################################################ + ### + ### Internal Helper Functions + ### + ################################################################################ + ################################################################################ + + def resetGprCounter(self): + self.vgprCounter = 0 + self.sgprCounter = 0 + + def getVgpr(self, num): + value = self.vgprCounter + self.vgprCounter += num + return value + + def getSgpr(self, num): + value = self.sgprCounter + self.sgprCounter += num + return value + + def vgprPrefix(self, *args): + if isinstance(args[0], int) and self.vgprPrefixFormat: + vgprStr = self.vgprPrefixFormat%args[0] + else: + vgprStr = args[0] + + if len(args) == 1: + return vgpr(vgprStr) + else: + args = args[1] + return vgpr(vgprStr, args) + + ################################################################################ + ################################################################################ + ### + ### Activation Functions + ### + ################################################################################ + ################################################################################ + + def getAbsModule(self, cDataType, vgprIdx): + module = Module("Abs") + if cDataType.isHalf() or cDataType.isBFloat16(): + absMagic = "0x7fff7fff" if self.usePK else "0x7fff" + module.add(VAndB32(dst=self.vgprPrefix(vgprIdx), src0=absMagic, src1=self.vgprPrefix(vgprIdx), comment="Remove sign bit")) + elif cDataType.isSingle(): + module.add(VAndB32(dst=self.vgprPrefix(vgprIdx), src0="0x7fffffff", src1=self.vgprPrefix(vgprIdx), comment="Remove sign bit")) + elif cDataType.isDouble(): + module.add(VAndB32(dst=self.vgprPrefix(vgprIdx+1), src0="0x7fffffff", src1=self.vgprPrefix(vgprIdx+1), comment="Remove sign bit")) + elif cDataType.isInt32(): + vgprTemp = self.getVgpr(1) + module.add(VSubI32(dst=vgpr(Holder(idx=vgprTemp)), src0=0, src1=self.vgprPrefix(vgprIdx), comment="x2 = -x")) + if self.saturateI8: + vgprTemp2 = self.getVgpr(1) + module.add(VMovB32(dst=vgpr(Holder(idx=vgprTemp2)), src=hex(127), comment="value = 127")) + module.add(VMed3I32(dst=self.vgprPrefix(vgprIdx), src0=self.vgprPrefix(vgprIdx), src1=vgpr(Holder(idx=vgprTemp)), src2=vgpr(Holder(idx=vgprTemp2)), comment="y = min(127, max(x, x2))")) + else: + module.add(VSubI32(dst=self.vgprPrefix(vgprIdx), src0=vgpr(Holder(idx=vgprTemp)), src1=self.vgprPrefix(vgprIdx), comment="y = max(x, x2)")) + else: + raise RuntimeError("Unsupported data type %s."%cDataType.toDevice("HIP")) + return module + + def getClippedReluModule(self, cDataType, vgprIdx, activationAlpha, activationBeta): + module = Module("ClippedRelu") + if cDataType.isHalf(): + for i in range(0, 2): + select_bit = SelectBit.WORD_0 if i == 0 else SelectBit.WORD_1 + module.add(VCmpGEF16(dst=VCC(), src0=self.vgprPrefix(vgprIdx), src1=sgpr(activationAlpha), \ + sdwa=SDWAModifiers(src0_sel=select_bit, src1_sel=SelectBit.WORD_0), comment="x > alpha?")) + module.add(VMinF16(dst=self.vgprPrefix(vgprIdx), src0=sgpr(activationBeta), src1=self.vgprPrefix(vgprIdx), \ + sdwa=SDWAModifiers(dst_sel=select_bit, dst_unused=UnusedBit.UNUSED_PRESERVE, \ + src0_sel=select_bit, src1_sel=select_bit), \ + comment="min(x, beta)")) + module.add(VCndMaskB32(dst=self.vgprPrefix(vgprIdx), src0=0.0, src1=self.vgprPrefix(vgprIdx), \ + sdwa=SDWAModifiers(dst_sel=select_bit, dst_unused=UnusedBit.UNUSED_PRESERVE, \ + src0_sel=select_bit, src1_sel=select_bit), \ + comment="set x to 0 if < alpha")) + module.add(SNop(waitState=0, comment="1 wait states")) # workaround for emulator + elif cDataType.isSingle(): + module.add(VCmpGEF32(dst=VCC(), src0=self.vgprPrefix(vgprIdx), src1=sgpr(activationAlpha), comment="x >= alpha ?")) + module.add(VMinF32(dst=self.vgprPrefix(vgprIdx), src0=sgpr(activationBeta), src1=self.vgprPrefix(vgprIdx), comment="min(x, beta)")) + module.add(VCndMaskB32(dst=self.vgprPrefix(vgprIdx), src0=0.0, src1=self.vgprPrefix(vgprIdx), comment="set x to 0 if < alpha")) + elif cDataType.isDouble(): + module.add(VCmpGEF64(dst=VCC(), src0=self.vgprPrefix(vgprIdx, 2), src1=sgpr(activationAlpha, 2), comment="x >= alpha ?")) + module.add(VMinF64(dst=self.vgprPrefix(vgprIdx, 2), src0=sgpr(activationBeta, 2), src1=self.vgprPrefix(vgprIdx, 2), comment="min(x, beta)")) + module.add(VCndMaskB32(dst=self.vgprPrefix(vgprIdx), src0=0, src1=self.vgprPrefix(vgprIdx), comment="set x to 0 if < 0")) + module.add(VCndMaskB32(dst=self.vgprPrefix(vgprIdx+1), src0=0, src1=self.vgprPrefix(vgprIdx+1), comment="set x to 0 if < 0")) + elif cDataType.isInt32(): + module.add(VCmpGEI32(dst=VCC(), src0=self.vgprPrefix(vgprIdx), src1=sgpr(activationAlpha), comment="x >= alpha ?")) + module.add(VMinI32(dst=self.vgprPrefix(vgprIdx), src0=sgpr(activationBeta), src1=self.vgprPrefix(vgprIdx), comment="min(x, beta)")) + module.add(VCndMaskB32(dst=self.vgprPrefix(vgprIdx), src0=0.0, src1=self.vgprPrefix(vgprIdx), comment="set x to 0 if < alpha")) + return module + + def getExpModule(self, cDataType, vgprIdx): + module = Module("Exp") + if cDataType.isHalf(): + sgprMagic = self.getSgpr(1) + module.add(SMovB32(dst=sgpr(Holder(idx=sgprMagic)), src=math.log(math.e,2), comment="exp magic")) + if self.usePK: + module.add(VMulPKF16(dst=self.vgprPrefix(vgprIdx), src0=sgpr(Holder(idx=sgprMagic)), src1=self.vgprPrefix(vgprIdx), comment="exp step 1")) + for i in range(0, 2): + select_bit = SelectBit.WORD_0 if i == 0 else SelectBit.WORD_1 + module.add(VExpF16(dst=self.vgprPrefix(vgprIdx), src=self.vgprPrefix(vgprIdx), \ + sdwa=SDWAModifiers(dst_sel=select_bit, dst_unused=UnusedBit.UNUSED_PRESERVE, \ + src0_sel=select_bit), \ + comment="exp step 2")) + module.add(SNop(waitState=0, comment="1 wait states")) #workaround for emulator + else: + module.add(VMulF16(dst=self.vgprPrefix(vgprIdx), src0=sgpr(Holder(idx=sgprMagic)), src1=self.vgprPrefix(vgprIdx), comment="exp step 1")) + module.add(VExpF16(dst=self.vgprPrefix(vgprIdx), src=self.vgprPrefix(vgprIdx), comment="exp step 2")) + module.add(SNop(waitState=0, comment="1 wait states")) #workaround for emulator + elif cDataType.isSingle(): + module.add(VMulF32(dst=self.vgprPrefix(vgprIdx), src0=math.log(math.e,2), src1=self.vgprPrefix(vgprIdx), comment="exp step 1")) + module.add(VExpF32(dst=self.vgprPrefix(vgprIdx), src=self.vgprPrefix(vgprIdx), comment="exp step 2" )) + module.add(SNop(waitState=0, comment="1 wait states")) #workaround for emulator + else: + raise RuntimeError("Unsupported data type %s."%cDataType.toDevice("HIP")) + return module + + def getGeluModule(self, cDataType, vgprIdx): + module = Module("Gelu") + # Gelu(x) = 0.5 * x * (1 + tanh(k0 * x * (1 + k1 * x * x))) + if cDataType.isHalf(): + flt16GeluK1Str = HexToStr(cDataType, self.usePK, ActivationMagicNumbers["Float16GeluK1"]) + sgprMagicK1 = self.getSgpr(1) + module.add(SMovB32(dst=sgpr(Holder(idx=sgprMagicK1)), src=flt16GeluK1Str, comment="Float16GeluK1" )) + vgprTemp = self.getVgpr(1) + if self.usePK: + module.add(VMulPKF16(dst=vgpr(Holder(idx=vgprTemp)), src0=self.vgprPrefix(vgprIdx), src1=self.vgprPrefix(vgprIdx), comment="x * x" )) + module.add(VFmaPKF16(dst=vgpr(Holder(idx=vgprTemp)), src0=vgpr(Holder(idx=vgprTemp)), src1=sgpr(Holder(idx=sgprMagicK1)), src2=1.0, \ + vop3=VOP3PModifiers(op_sel_hi=[1,1,0,1]), comment="x^2 * k1 + 1")) + module.add(VMulPKF16(dst=vgpr(Holder(idx=vgprTemp)), src0=self.vgprPrefix(vgprIdx), src1=vgpr(Holder(idx=vgprTemp)), comment="x * (x^2 * k1 + 1)")) + coef = floatUnion(u=ActivationMagicNumbers["FloatGeluK0"]) + module.add(VMulPKF16(dst=vgpr(Holder(idx=vgprTemp)), src0=coef.f, src1=vgpr(Holder(idx=vgprTemp)), comment="k0 * x * (x^2 * k1 + 1)")) + module.add(self.getTanhModule(cDataType, Holder(idx=vgprTemp), "", "")) + module.add(VAddPKF16(dst=vgpr(Holder(idx=vgprTemp)), src0=1.0, src1=vgpr(Holder(idx=vgprTemp)), \ + vop3=VOP3PModifiers(op_sel_hi=[0,1,1]), comment="1 + tanh(...)" )) + module.add(VMulPKF16(dst=vgpr(Holder(idx=vgprTemp)), src0=self.vgprPrefix(vgprIdx), src1=vgpr(Holder(idx=vgprTemp)), comment="x * (1 + tanh(...))")) + module.add(VMulPKF16(dst=self.vgprPrefix(vgprIdx), src0=0.5, src1=vgpr(Holder(idx=vgprTemp)), \ + vop3=VOP3PModifiers(op_sel_hi=[0,1,1]), comment="0.5 * x * (1 + tanh(...))")) + else: + module.add(VMulF16(dst=vgpr(Holder(idx=vgprTemp)), src0=self.vgprPrefix(vgprIdx), src1=self.vgprPrefix(vgprIdx), comment="x * x" )) + module.add(VFmaF16(dst=vgpr(Holder(idx=vgprTemp)), src0=vgpr(Holder(idx=vgprTemp)), src1=sgpr(Holder(idx=sgprMagicK1)), src2=1.0, comment="x^2 * k1 + 1")) + module.add(VMulF16(dst=vgpr(Holder(idx=vgprTemp)), src0=self.vgprPrefix(vgprIdx), src1=vgpr(Holder(idx=vgprTemp)), comment="x * (x^2 * k1 + 1)")) + coef = floatUnion(u=ActivationMagicNumbers["FloatGeluK0"]) + module.add(VMulF16(dst=vgpr(Holder(idx=vgprTemp)), src0=coef.f, src1=vgpr(Holder(idx=vgprTemp)), comment="k0 * x * (x^2 * k1 + 1)")) + module.add(self.getTanhModule(cDataType, Holder(idx=vgprTemp), "", "")) + module.add(VAddF16(dst=vgpr(Holder(idx=vgprTemp)), src0=1.0, src1=vgpr(Holder(idx=vgprTemp)), comment="1 + tanh(...)" )) + module.add(VMulF16(dst=vgpr(Holder(idx=vgprTemp)), src0=self.vgprPrefix(vgprIdx), src1=vgpr(Holder(idx=vgprTemp)), comment="x * (1 + tanh(...))")) + module.add(VMulF16(dst=self.vgprPrefix(vgprIdx), src0=0.5, src1=vgpr(Holder(idx=vgprTemp)), comment="0.5 * x * (1 + tanh(...))")) + elif cDataType.isSingle(): + vgprTemp = self.getVgpr(1) + flt16GeluK1Str = HexToStr(cDataType, self.usePK, ActivationMagicNumbers["FloatGeluK1"]) + module.add(VMulF32(dst=vgpr(Holder(idx=vgprTemp)), src0=flt16GeluK1Str, src1=self.vgprPrefix(vgprIdx), comment="k1 * x")) + module.add(VFmaF32(dst=vgpr(Holder(idx=vgprTemp)), src0=self.vgprPrefix(vgprIdx), src1=vgpr(Holder(idx=vgprTemp)), src2=1.0, comment="1 + (k1 * x * x)")) + module.add(VMulF32(dst=vgpr(Holder(idx=vgprTemp)), src0=self.vgprPrefix(vgprIdx), src1=vgpr(Holder(idx=vgprTemp)), comment="x * (1 + k1 * x * x)")) + coef = floatUnion(u=ActivationMagicNumbers["FloatGeluK0"]) + module.add(VMulF32(dst=vgpr(Holder(idx=vgprTemp)), src0=coef.f, src1=vgpr(Holder(idx=vgprTemp)), comment="k0 * x * (x^2 * k1 + 1)")) + module.add(self.getTanhModule(cDataType, Holder(idx=vgprTemp), "", "")) + module.add(VAddF32(dst=vgpr(Holder(idx=vgprTemp)), src0=1.0, src1=vgpr(Holder(idx=vgprTemp)), comment="1 + tanh(...)" )) + module.add(VMulF32(dst=vgpr(Holder(idx=vgprTemp)), src0=self.vgprPrefix(vgprIdx), src1=vgpr(Holder(idx=vgprTemp)), comment="x * (1 + tanh(...))")) + module.add(VMulF32(dst=self.vgprPrefix(vgprIdx), src0=0.5, src1=vgpr(Holder(idx=vgprTemp)), comment="0.5 * x * (1 + tanh(...))")) + else: + raise RuntimeError("Unsupported data type %s."%cDataType.toDevice("HIP")) + return module + + def getLeakyReluModule(self, cDataType, vgprIdx, activationAlpha): + module = Module("LeakyRelu") + if cDataType.isHalf(): + vgprTemp = self.getVgpr(1) + module.add(VMulPKF16(dst=vgpr(Holder(idx=vgprTemp)), src0=sgpr(activationAlpha), src1=self.vgprPrefix(vgprIdx), comment="tmp = x * alpha")) + for i in range(0, 2): + select_bit = SelectBit.WORD_0 if i == 0 else SelectBit.WORD_1 + module.add(VCmpGEF16(dst=VCC(), src0=self.vgprPrefix(vgprIdx), src1=0.0, \ + sdwa=SDWAModifiers(src0_sel=select_bit, src1_sel=SelectBit.WORD_0), \ + comment="x > 0 ?")) + module.add(VCndMaskB32(dst=self.vgprPrefix(vgprIdx), src0=vgpr(Holder(idx=vgprTemp)), src1=self.vgprPrefix(vgprIdx), \ + sdwa=SDWAModifiers(dst_sel=select_bit, dst_unused=UnusedBit.UNUSED_PRESERVE, \ + src0_sel=select_bit, src1_sel=select_bit), \ + comment="set x to tmp if < 0")) + elif cDataType.isSingle(): + vgprTemp = self.getVgpr(1) + module.add(VMulF32(dst=vgpr(Holder(idx=vgprTemp)), src0=sgpr(activationAlpha), src1=self.vgprPrefix(vgprIdx), comment="tmp = x * alpha")) + module.add(VCmpGEF32(dst=VCC(), src0=self.vgprPrefix(vgprIdx), src1=0.0, comment="x >= 0 ?")) + module.add(VCndMaskB32(dst=self.vgprPrefix(vgprIdx), src0=vgpr(Holder(idx=vgprTemp)), src1=self.vgprPrefix(vgprIdx), comment="set x to tmp if < 0")) + elif cDataType.isDouble(): + vgprTemp = self.getVgpr(2) + module.add(VMulF64(dst=vgpr(Holder(idx=vgprTemp), 2), src0=sgpr(activationAlpha, 2), src1=self.vgprPrefix(vgprIdx, 2), comment="tmp = x * alpha")) + module.add(VCmpGEF64(dst=VCC(), src0=self.vgprPrefix(vgprIdx, 2), src1=0.0, comment="x >= 0 ?")) + module.add(VCndMaskB32(dst=self.vgprPrefix(vgprIdx), src0=vgpr(Holder(idx=vgprTemp)), src1=self.vgprPrefix(vgprIdx), comment="set x to tmp if < 0")) + module.add(VCndMaskB32(dst=self.vgprPrefix(vgprIdx+1), src0=vgpr(Holder(idx=vgprTemp+1)), src1=self.vgprPrefix(vgprIdx+1), comment="set x to tmp if < 0")) + elif cDataType.isInt32(): + vgprTemp = self.getVgpr(1) + module.add(VMulLOU32(dst=vgpr(Holder(idx=vgprTemp)), src0=sgpr(activationAlpha), src1=self.vgprPrefix(vgprIdx), comment="tmp = x * alpha")) + module.add(VCmpGEI32(dst=VCC(), src0=self.vgprPrefix(vgprIdx), src1=0, comment="x >= 0 ?")) + module.add(VCndMaskB32(dst=self.vgprPrefix(vgprIdx), src0=vgpr(Holder(idx=vgprTemp)), src1=self.vgprPrefix(vgprIdx), comment="set x to tmp if < 0")) + else: + raise RuntimeError("Unsupported data type %s."%cDataType.toDevice("HIP")) + return module + + def getReluModule(self, cDataType, vgprIdx): + module = Module("LeakyRelu") + if cDataType.isHalf(): + module.add(VMaxPKF16(dst=self.vgprPrefix(vgprIdx), src0=self.vgprPrefix(vgprIdx), src1=0, comment="x = max(0, x)" )) + elif cDataType.isSingle(): + module.add(VMaxF32(dst=self.vgprPrefix(vgprIdx), src0=self.vgprPrefix(vgprIdx), src1=0, comment="x = max(0, x)" )) + elif cDataType.isDouble(): + module.add(VMaxF64(dst=self.vgprPrefix(vgprIdx, 2), src0=self.vgprPrefix(vgprIdx, 2), src1=0, comment="x = max(0, x)" )) + elif cDataType.isInt32(): + if self.saturateI8: + vgprTemp = self.getVgpr(1) + module.add(VMovB32(dst=vgpr(Holder(idx=vgprTemp)), src=hex(127), comment="value = 127")) + module.add(VMed3I32(dst=self.vgprPrefix(vgprIdx), src0=self.vgprPrefix(vgprIdx), src1=0, src2=vgpr(Holder(idx=vgprTemp)), comment="x = min(127, max(0, x))" )) + else: + module.add(VMaxI32(dst=self.vgprPrefix(vgprIdx), src0=self.vgprPrefix(vgprIdx), src1=0, comment="x = max(0, x)" )) + else: + raise RuntimeError("Unsupported data type %s."%cDataType.toDevice("HIP")) + return module + + def getSigmoidModule(self, cDataType, vgprIdx): + module = Module("Sigmoid") + if cDataType.isHalf(): + if self.usePK: + module.add(VMulPKF16(dst=self.vgprPrefix(vgprIdx), src0=-1.0, src1=self.vgprPrefix(vgprIdx), comment=" x = -x")) + module.add(self.getExpModule(cDataType, vgprIdx)) + module.add(VAddPKF16(dst=self.vgprPrefix(vgprIdx), src0=1.0, src1=self.vgprPrefix(vgprIdx), \ + vop3=VOP3PModifiers(op_sel_hi=[0,1,1]), comment="1 + exp(-x)")) + for i in range(0, 2): + select_bit = SelectBit.WORD_0 if i == 0 else SelectBit.WORD_1 + module.add(VRcpF16(dst=self.vgprPrefix(vgprIdx), src=self.vgprPrefix(vgprIdx), \ + sdwa=SDWAModifiers(dst_sel=select_bit, dst_unused=UnusedBit.UNUSED_PRESERVE, src0_sel=select_bit), \ + comment="1 / (1 + exp(-x))")) + module.add(SNop(waitState=0, comment="1 wait states")) #workaround for emulator + else: + module.add(VMulF16(dst=self.vgprPrefix(vgprIdx), src0=-1.0, src1=self.vgprPrefix(vgprIdx), comment=" x = -x")) + module.add(self.getExpModule(cDataType, vgprIdx)) + module.add(VAddF16(dst=self.vgprPrefix(vgprIdx), src0=1.0, src1=self.vgprPrefix(vgprIdx), comment="1 + exp(-x)")) + module.add(VRcpF16(dst=self.vgprPrefix(vgprIdx), src=self.vgprPrefix(vgprIdx), comment="1 / (1 + exp(-x))")) + module.add(SNop(waitState=0, comment="1 wait states")) #workaround for emulator + elif cDataType.isSingle(): + module.add(VMulF32(dst=self.vgprPrefix(vgprIdx), src0=-1.0, src1=self.vgprPrefix(vgprIdx), comment=" x = -x")) + module.add(self.getExpModule(cDataType, vgprIdx)) + module.add(VAddF32(dst=self.vgprPrefix(vgprIdx), src0=1.0, src1=self.vgprPrefix(vgprIdx), comment="1 + exp(-x)" )) + module.add(VRcpF32(dst=self.vgprPrefix(vgprIdx), src=self.vgprPrefix(vgprIdx), comment="1 / (1 + exp(-x))" )) + module.add(SNop(waitState=0, comment="1 wait states")) #workaround for emulator + else: + raise RuntimeError("Unsupported data type %s."%cDataType.toDevice("HIP")) + return module + + def getTanhModule(self, cDataType, vgprIdx, activationAlpha, activationBeta): + module = Module("Tanh") + if cDataType.isHalf(): + # We don't need s_pack_ll_b32_b16 cause the input is already duplicated + if self.usePK: + if activationAlpha: + module.add(VMulPKF16(dst=self.vgprPrefix(vgprIdx), src0=sgpr(activationAlpha), src1=self.vgprPrefix(vgprIdx), comment="x * alpha")) + module.add(VMulPKF16(dst=self.vgprPrefix(vgprIdx), src0=2, src1=self.vgprPrefix(vgprIdx), comment=" x = 2 * x")) + module.add(self.getExpModule(cDataType, vgprIdx)) + module.add(VAddPKF16(dst=self.vgprPrefix(vgprIdx), src0=1.0, src1=self.vgprPrefix(vgprIdx), \ + vop3=VOP3PModifiers(op_sel_hi=[0,1,1]), comment="e^2x + 1")) + for i in range(0, 2): + select_bit = SelectBit.WORD_0 if i == 0 else SelectBit.WORD_1 + vgprCtrl = "dst_sel:WORD_%d dst_unused:UNUSED_PRESERVE src0_sel:WORD_%d"%(i, i) + module.add(VRcpF16(dst=self.vgprPrefix(vgprIdx), src=self.vgprPrefix(vgprIdx), \ + sdwa=SDWAModifiers(dst_sel=select_bit, dst_unused=UnusedBit.UNUSED_PRESERVE, \ + src0_sel=select_bit), \ + comment="1 / (1 + exp(-x))")) + module.add(SNop(waitState=0, comment="1 wait states")) #workaround for emulator + module.add(VFmaPKF16(dst=self.vgprPrefix(vgprIdx), src0=-2.0, src1=self.vgprPrefix(vgprIdx), src2=1.0, \ + vop3=VOP3PModifiers(op_sel_hi=[0,1,0,1]), comment="tanh(x) = (1 / (e^2x + 1)) * (-2) + 1")) + if activationBeta: + module.add(VMulPKF16(dst=self.vgprPrefix(vgprIdx), src0=sgpr(activationBeta), src1=self.vgprPrefix(vgprIdx), comment="beta * tanh(x)")) + else: + if activationAlpha: + module.add(VMulF16(dst=self.vgprPrefix(vgprIdx), src0=sgpr(activationAlpha), src1=self.vgprPrefix(vgprIdx), comment="x * alpha")) + module.add(VMulF16(dst=self.vgprPrefix(vgprIdx), src0=2, src1=self.vgprPrefix(vgprIdx), comment=" x = 2 * x")) + module.add(self.getExpModule(cDataType, vgprIdx)) + module.add(VAddF16(dst=self.vgprPrefix(vgprIdx), src0=1.0, src1=self.vgprPrefix(vgprIdx), comment="e^2x + 1")) + module.add(VRcpF16(dst=self.vgprPrefix(vgprIdx), src=self.vgprPrefix(vgprIdx), comment="1 / (1 + exp(-x))")) + module.add(SNop(waitState=0, comment="1 wait states")) #workaround for emulator + module.add(VFmaF16(dst=self.vgprPrefix(vgprIdx), src0=-2.0, src1=self.vgprPrefix(vgprIdx), src2=1.0, comment="tanh(x) = (1 / (e^2x + 1)) * (-2) + 1")) + if activationBeta: + module.add(VMulF16(dst=self.vgprPrefix(vgprIdx), src0=sgpr(activationBeta), src1=self.vgprPrefix(vgprIdx), comment="beta * tanh(x)")) + elif cDataType.isSingle(): + if activationAlpha: + module.add(VMulF32(dst=self.vgprPrefix(vgprIdx), src0=sgpr(activationAlpha), src1=self.vgprPrefix(vgprIdx), comment="x * alpha")) + module.add(VMulF32(dst=self.vgprPrefix(vgprIdx), src0=2, src1=self.vgprPrefix(vgprIdx), comment=" x = 2 * x")) + module.add(self.getExpModule(cDataType, vgprIdx)) + module.add(VAddF32(dst=self.vgprPrefix(vgprIdx), src0=1.0, src1=self.vgprPrefix(vgprIdx), comment="e^2x + 1")) + module.add(VRcpF32(dst=self.vgprPrefix(vgprIdx), src=self.vgprPrefix(vgprIdx), comment="1 / (e^2x + 1)")) + module.add(SNop(waitState=0, comment="1 wait states")) #workaround for emulator + module.add(VFmaF32(dst=self.vgprPrefix(vgprIdx), src0=-2.0, src1=self.vgprPrefix(vgprIdx), src2=1.0, comment="(-2) * (1 / (e^2x + 1)) + 1")) + if activationBeta: + module.add(VMulF32(dst=self.vgprPrefix(vgprIdx), src0=sgpr(activationBeta), src1=self.vgprPrefix(vgprIdx), comment="beta * tanh(x)")) + else: + raise RuntimeError("Unsupported data type %s."%cDataType.toDevice("HIP")) + return module + +################################################################################ +################################################################################ +### +### Post Functions +### +################################################################################ +################################################################################ + +__FUSE_MAGIC_NAME__ = "dbiw@I$HONIhnjf4_fused" + +# Public +def CombineInstructions(module, fuseDebug = False): + moduleAndIndex = dict() + CombineInstructionsBetweenModules(module, moduleAndIndex, fuseDebug) + # Remove Empty Blocks + module = RemoveEmptyBlocks(module) + return module + +# Does not support modules with branches +def CombineInstructionsBetweenModules(module, moduleAndIndex, fuseDebug): + index = 0 + while index < len(module.items()): + item = module.items()[index] + if isinstance(item, Module): + CombineInstructionsBetweenModules(item, moduleAndIndex, fuseDebug) + index = module.items().index(item) + elif isinstance(item, SNop): + pass + elif isinstance(item, Instruction): + newItem = item + if moduleAndIndex: + newItem = FuseInstruction(item, moduleAndIndex, fuseDebug) + index = module.items().index(newItem) + if isinstance(newItem.dst, RegisterContainer): + # Update the dict + moduleAndIndex[newItem.dst] = newItem + index += 1 + +def RemoveEmptyBlocks(module): + for idx, item in enumerate(module.items()): + if isinstance(item, Module): + newItem = RemoveEmptyBlocks(item) + module.items()[idx] = newItem + if len(module.items()) == 1 and isinstance(module.items()[0], Module): + return module.items()[0] + return module + +def FuseInstruction(currentInst, moduleAndIndex, fuseDebug): + assert(isinstance(currentInst, Instruction)) + newInst = None + # Fuses if v_add_f16 to v_fma_f16 if v_add_f16 is a self adding instruction. + # Currently, we only fuse when the vgpr is add by 1 in both instructions. + # ex. v_add_f16 v0, 1.0, v0 + # + v_fma_f16 v0, -2.0, v0, 1.0 + # => v_fma_f16 v0, -2.0, v0, 2.0 + if isinstance(currentInst, VAddF16) or isinstance(currentInst, VAddPKF16) or \ + isinstance(currentInst, VAddF32): + isPK = isinstance(currentInst, VAddPKF16) + outVgpr = currentInst.dst + addConst = "" + isSelfAddConst = False + for param in currentInst.src: + if param == outVgpr: + isSelfAddConst = True + if (isinstance(param, float) or isinstance(param, int)): + if param == 1: + addConst = param + + if isSelfAddConst and addConst: + oldInst = moduleAndIndex.get(outVgpr) + if isinstance(oldInst, Instruction): + if currentInst.instType == InstType.INST_F16: + func = VFmaPKF16 if isPK else VFmaF16 + elif currentInst.instType == InstType.INST_F32: + func = VFmaF32 + else: + assert("You should not reach here.") + if isinstance(oldInst, func) and oldInst.src[2] == 1.0: + # Cannot fuse if the target instruction has any rvalue reassigned or its lvalue + # used before the current instruction + if not FindAssignAndUse(oldInst, currentInst, outVgpr, outVgpr): + newInst = deepcopy(oldInst) + newInst.src[2] = addConst + newInst.src[2] + newInst.comment += " ( + 1 (fused))" + replaceInst(currentInst, newInst, fuseDebug) + removeOldInst(oldInst, currentInst, newInst, fuseDebug) + # Fuses if v_mul_f16 to v_mul_f16 if the later one is a self multiplying instruction. + # Only fuses when both instructions multiply constant + elif isinstance(currentInst, VMulF16) or isinstance(currentInst, VMulPKF16) or \ + isinstance(currentInst, VMulF32) or isinstance(currentInst, VMulF64): + isPK = isinstance(currentInst, VMulPKF16) + outVgpr = currentInst.dst + mulConst = "" + newFuseInst = "" + isSelfMulConst = False + for param in currentInst.src: + if param == outVgpr: + isSelfMulConst = True + # The constant may be an sgpr + if isinstance(param, RegisterContainer) and param.regType == 's': + oldInst = moduleAndIndex.get(param) + if isinstance(oldInst, SMovB32): + oldparam = oldInst.src[0] + if oldInst.dst == param and (isinstance(oldparam, float) or isinstance(oldparam, int)): + # Cannot fuse if another instruction is using the same sgpr before a new assignment occurs + if not FindUse(oldInst, currentInst, param): + mulConst = oldparam + newFuseInst = oldInst + if (isinstance(param, float) or isinstance(param, int)): + mulConst = param + + if isSelfMulConst and mulConst: + oldInst = moduleAndIndex.get(outVgpr) + if isinstance(oldInst, Instruction): + if currentInst.instType == InstType.INST_F16: + func = VMulPKF16 if isPK else VMulF16 + elif currentInst.instType == InstType.INST_F32: + func = VMulF32 + elif currentInst.instType == InstType.INST_F64: + func = VMulF64 + else: + assert("You should not reach here.") + + if isinstance(oldInst, func): + # Cannot fuse if the target instruction has any rvalue reassigned or its lvalue + # used before the current instruction + if not FindAssignAndUse(oldInst, currentInst, outVgpr, outVgpr): + for paramIdx, param in enumerate(oldInst.src): + if (isinstance(param, float) or isinstance(param, int)): + newInst = deepcopy(oldInst) + newValue = param * mulConst + formatting = " (fused %f)" if isinstance(param, float) else " (fused %d)" + if newFuseInst: + newFuseInst.src[0] = newValue + newInst.src[paramIdx] = newFuseInst.dst + newFuseInst.comment += formatting%newValue + else: + newInst.src[paramIdx] = newValue + newInst.comment += formatting%newValue + replaceInst(currentInst, newInst, fuseDebug) + removeOldInst(oldInst, currentInst, newInst, fuseDebug) + break + return newInst if newInst else currentInst + +# This only works for Activation.py +def FindUse(startInst, targetInst, varTarget): + _, isUse = FindUseIter(startInst, targetInst, varTarget) + return isUse + +# This only works for Activation.py +def FindUseIter(startItem, targetInst, varTarget): + module = startItem + idx = -1 + isEnd = False + isUse = False + if isinstance(startItem, Instruction): + module = startItem.parent + idx = module.items().index(startItem) + assert(isinstance(module, Module)) + if idx + 1 < len(module.items()[idx + 1:]): + for item in module.items()[idx + 1:]: + if item is targetInst: + pass + elif isinstance(item, SNop): + pass + elif isinstance(item, Instruction): + if item.src: + for param in item.src: + if param == varTarget: + isEnd = True + isUse = True + break + elif item.dst == varTarget: + isEnd = True + isUse = False + elif isinstance(item, Module): + isEnd, isUse = FindUseIter(item, targetInst, varTarget) + if isEnd: + return isEnd, isUse + return False, isUse + +# This only works for Activation.py +def FindAssignAndUse(startInst, endInst, assignVar, useVar): + _, isUse = FindAssignAndUseIter(startInst, endInst, assignVar, useVar) + return isUse + +# This only works for Activation.py +def FindAssignAndUseIter(startItem, endInst, assignVar, useVar): + module = startItem + idx = -1 + isEnd = False + isUse = False + if isinstance(startItem, Instruction): + module = startItem.parent + idx = module.items().index(startItem) + assert(isinstance(module, Module)) + if idx + 1 < len(module.items()[idx + 1:]): + for item in module.items()[idx + 1:]: + # Use + if item is endInst: + isEnd = True + elif isinstance(item, SNop): + pass + elif isinstance(item, Instruction): + if item.dst == assignVar: + isEnd = True + isUse = True + # Check use + if item.src: + for param in item.src: + if param == useVar: + isEnd = True + isUse = True + break + elif isinstance(item, Module): + isEnd, isUse = FindAssignAndUseIter(item, endInst, assignVar, useVar) + if isEnd: + return isEnd, isUse + return isEnd, isUse + +def removeOldInst(removeInst, dstInst, fusedInst, debug): + module = removeInst.parent + targetIdx = -1 + for idx, item in enumerate(module.items()): + if item == removeInst: + if debug: + tb = TextBlock("\n/* Fused to block %s + %s -> %s */\n"%(str(removeInst), str(dstInst), str(fusedInst))) + tb.name = __FUSE_MAGIC_NAME__ + module.items()[idx] = tb + else: + targetIdx = idx + break + + if targetIdx > -1: + module.removeItemByIndex(targetIdx) + +def replaceInst(srcInst, dstInst, debug): + module = srcInst.parent + targetIdx = -1 + for idx, item in enumerate(module.items()): + if item == srcInst: + if debug: + dstInst.comment += " (Block replaced %s)"%(str(srcInst)) + targetIdx = idx + break + if targetIdx > -1: + module.replaceItemByIndex(targetIdx, dstInst) + +################################################################################ +################################################################################ +### +### Helper Functions +### +################################################################################ +################################################################################ + +def getMagic(cDataType, value, isPack=False): + if cDataType.isDouble(): + printExit("Currently magic does not support double.") + elif cDataType.isHalf(): + fu = value if isinstance(value, floatUnion) else floatUnion(f=value) + magicNum = struct.unpack('{width}}'.format(message=str, width=totalLength) + +class ActivationInline: + def __init__(self, dataType) -> None: + self.dataType = dataType + self.asmStr = "asm(" + + # Public Function + def generateInlineAssemblyFunction(self, activationType): + kStr = "" + if activationType == 'none': + return kStr + + enumName = "Tensile::ActivationType_%s"%self.dataType.toChar() + + ptrStr = self.dataType.toDevice("HIP") + names = "" + if activationType == 'all': + names += ",\n" + names += " %s const activationType"%enumName + for name in activationType.getAdditionalArgStringList(False): + names += ",\n" + names += " %s const %s"%(ptrStr, name) + changeLine = "\n " if names else "" + kStr += "__device__ inline %s activation(%s%s value%s)\n{\n"%(ptrStr, changeLine, ptrStr, names) + # function body + if activationType == 'all': + for index, enumStr in enumerate(ActivationType.getEnumStrList(self.dataType, includeNone=False)): + if index == 0: + kStr += " if (activationType == %s::%s) {\n"%(enumName, ActivationType(enumStr).toEnum()) + else: + kStr += " else if (activationType == %s::%s) {\n"%(enumName, ActivationType(enumStr).toEnum()) + kStr += self.generateInlineAssemblyBody(4, enumStr) + kStr += " }" + kStr += "\n" + else: + kStr += self.generateInlineAssemblyBody(2, activationType) + # function body end + kStr += " return value;\n" + kStr += "}\n" + return kStr + + def replaceGpr(self, module): + for item in module.items(): + if isinstance(item, SNop): + pass + elif isinstance(item, Instruction): + if isinstance(item.dst, RegisterContainer): + if not item.dst.regName: + item.dst.setInlineAsm(True) + if item.src: + for param in item.src: + if isinstance(param, RegisterContainer): + if not param.regName: + param.setInlineAsm(True) + + def getActivationAsmStr(self, activation, module, spaces): + module = activation.postProcess(self.dataType, module) + self.replaceGpr(module) + activation.assignGpr(module, 0, 0) + module.setInlineAsmPrintMode(True) + kStr = str(module) + newStr = "" + for instStr in kStr.split("\n"): + if instStr: + newStr += spaces + instStr + "\n" + return newStr + + # Internal Function + def generateInlineAssemblyBody(self, spaces, activationType): + ptrStr = self.dataType.toDevice("HIP") + activation = ActivationModule() + activation.setUsePK(False) + activation.resetGprCounter() + kStr = "" + padSpacesStr = ' ' * spaces + asm = padSpacesStr + self.asmStr + if (activationType == 'abs'): + if self.dataType.isHalf() or self.dataType.isBFloat16(): + unionDataTypeStr = "_Float16" if self.dataType.isHalf() else "BFloat16" + unionName = "f16_union" if self.dataType.isHalf() else "bf16_union" + kStr += (padSpacesStr + "union {\n") + kStr += (padSpacesStr + " %s f;\n"%unionDataTypeStr) + kStr += (padSpacesStr + " short s;\n") + kStr += (padSpacesStr + "} %s;\n"%unionName) + kStr += (padSpacesStr + "%s.f = value;\n"%unionName) + kStr += (padSpacesStr + "%s.s = %s.s & 0x7fff;\n"%(unionName, unionName)) + kStr += (padSpacesStr + "value = %s.f;\n"%unionName) + elif (self.dataType.isSingle() or self.dataType.isDouble() or self.dataType.isInt32()): + kStr += (padSpacesStr + "value = abs(value);\n") + else: + raise RuntimeError("Unrecognized data type %s."%self.dataType) + elif (activationType == 'clippedrelu'): + if (self.dataType.isSingle() or self.dataType.isHalf() or self.dataType.isDouble()): + kStr += (padSpacesStr + "value = (value >= alpha) ? min(value, beta) : 0.0;\n") + elif self.dataType.isInt32(): + kStr += (padSpacesStr + "value = (value >= alpha) ? min(value, beta) : 0;\n") + elif (activationType == 'exp'): + kStr += (asm + " // Exp\n") + module = activation.getExpModule(self.dataType, 0) + kStr += self.getActivationAsmStr(activation, module, (len(asm) * " ")) + kStr += addSpace(asm, ": \"+v\"(value) : \n") + kStr += self.getRequiredRegStr(asm, activation.vgprCounter, activation.sgprCounter) + elif (activationType == 'gelu'): + kStr += (asm + " // gelu\n") + module = activation.getGeluModule(self.dataType, 0) + kStr += self.getActivationAsmStr(activation, module, (len(asm) * " ")) + kStr += addSpace(asm, ": \"+v\"(value) : \n") + kStr += self.getRequiredRegStr(asm, activation.vgprCounter, activation.sgprCounter) + elif (activationType == 'leakyrelu'): + if (self.dataType.isSingle() or self.dataType.isHalf() or self.dataType.isDouble()): + kStr += (padSpacesStr + "value = (value >= 0.0) ? value : (value * alpha);\n") + elif self.dataType.isInt32(): + kStr += (padSpacesStr + "value = (value >= 0) ? value : (value * alpha);\n") + else: + raise RuntimeError("Unsupported data type %s."%ptrStr) + elif (activationType == 'relu'): + if (self.dataType.isSingle() or self.dataType.isHalf() or self.dataType.isDouble()): + kStr += (padSpacesStr + "value = max(0.0, value);\n") + elif self.dataType.isInt32(): + kStr += (padSpacesStr + "value = max(0, value);\n") + else: + raise RuntimeError("Unsupported data type %s."%ptrStr) + elif (activationType == 'sigmoid'): + kStr += (asm + " // Sigmoid\n") + module = activation.getSigmoidModule(self.dataType, 0) + kStr += self.getActivationAsmStr(activation, module, (len(asm) * " ")) + kStr += addSpace(asm, ": \"+v\"(value) : \n") + kStr += self.getRequiredRegStr(asm, activation.vgprCounter, activation.sgprCounter) + elif (activationType == 'tanh'): + kStr += (asm + " // tanh\n") + module = activation.getTanhModule(self.dataType, 0, 1, 2) + kStr += self.getActivationAsmStr(activation, module, (len(asm) * " ")) + kStr += addSpace(asm, ": \"+v\"(value) : \"s\"(alpha), \"s\"(beta)\n") + kStr += self.getRequiredRegStr(asm, activation.vgprCounter, activation.sgprCounter) + else: + if (activationType != 'none'): + raise RuntimeError("Unrecognized type %s."%activationType) + return kStr + # Internal use. Automatically gets the required vgprs and sgprs for inline assembly + def getRequiredRegStr(self, spaceAlignStr, numOfVgpr, numOfSgpr): + requiredReg = [] + for i in range(0, numOfVgpr): + requiredReg.append("\"v%d\""%i) + for i in range(0, numOfSgpr): + requiredReg.append("\"s%d\""%i) + requiredStr = "" + if (len(requiredReg) > 0): + requiredStr = requiredReg[0] + for i in range(1, len(requiredReg)): + requiredStr += ", %s"%requiredReg[i] + kStr = "" + kStr += addSpace(spaceAlignStr,":%s);\n"%requiredStr) + return kStr diff --git a/tensilelite/Tensile/AsmAddressCalculation.py b/tensilelite/Tensile/AsmAddressCalculation.py new file mode 100644 index 0000000000..c92e0a4490 --- /dev/null +++ b/tensilelite/Tensile/AsmAddressCalculation.py @@ -0,0 +1,501 @@ +################################################################################ +# Copyright 2019-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .TensileInstructions import Module, EXEC, vgpr, sgpr, log2 +from .TensileInstructions.Instructions import * +from .Common import globalParameters + +############################################################################## +# Fields associated with computing address +############################################################################## +class AddrCalculation: + # rowInc is number of rows to add to the base address + # coord0Vgpr : This is VGPR that holds coord0. Coord0 is element-space + # packed index for the 0 coordinate of the C/D matrix. + # coord1Vgpr : VGPR which tracks the last coord1 calculation. + # If this is new coord1, just overwrite it with latest calc. + def __init__(self, kernelWriter, ss, addrCVgpr, addrDVgpr, addrBiasVgpr, element, \ + coordOffset0, coord1Vgpr, coordOffset1, rowInc, newCoord1): + self.kernelWriter = kernelWriter + + # vgprs for address, could be more than one (for flat) + self.addrDVgpr = addrDVgpr + self.addrCVgpr = addrCVgpr + self.addrBiasVgpr = addrBiasVgpr + self.coord1Vgpr = coord1Vgpr # vgpr that stores coord1Vgpr + + self.element = element + self.coordOffset0 = coordOffset0 + self.coordOffset1 = coordOffset1 + self.rowInc = rowInc + self.rowIncDirtyRowPtr = 0 # rowInc was used to modify rowPtr, need to recompute addr + self.newCoord1 = newCoord1 # vgpr that stores newCoord1 + + if ss.optSingleColVgpr: + # optimized stores use the load offset for coordOffset0 calculations. + self.biasOffset = coordOffset0 * kernelWriter.bpeAB + self.globalOffset = coordOffset0 * kernelWriter.bpeCexternal + else: + # else non-opt stores include the coord0 offset into VGPR address calcs + self.biasOffset = 0 + self.globalOffset = 0 + + def addScaled(self, destV, src0, src1, scale1, tmpS01, comment=""): + """ + Use minimally efficient instructions to add stride*scale + """ + + module = Module("addScaled") + if scale1 == 1: + module.add(VAddU32(dst=destV, src0=src0, src1=src1, comment=comment)) + else: + module.add(SMulI32(dst=sgpr(tmpS01), src0=src1, src1=scale1, comment="scale stride")) + module.add(VAddU32(dst=destV, src0=src0, src1=sgpr(tmpS01), comment=comment)) + return module + + + def emitAddressCoordIncrement(self, kernel, ss, tmpVgpr, tmpS01, updateCoord1): + """ + Emit code that computes the coord0 and coord1 for this element + sets self.coord0Vgpr with the address that holds the coord0 value for this element. + Input: + - tmpVgpr is a 1 temporary VGPR used for coord0 calculation on edges + """ + + module = Module("emitAddressCoordIncrement") + kw = self.kernelWriter + (d1,d0,vc1,vc0) = self.element + self.coord0Vgpr = None # will set below + + # module.addComment0("store addr=v%u coordOffset0=%u"% \ + # (self.addr, self.coordOffset0)) + module.addComment0("(d1,vc1,d0,vc0)=(%u,%u,%u,%u)"\ + % (d1,vc1,d0,vc0)) + if ss.optSingleColVgpr: + self.coord0Vgpr = kw.vgprs.coord0 + elif not ss.optSharedColVgpr or (d1 == vc1 == 0): + # not share mode or first row always does the address calc math: + + if self.coordOffset0 == 0: + self.coord0Vgpr = kw.vgprs.coord0 + elif self.coordOffset0 <= 64: + self.coord0Vgpr = tmpVgpr + module.add(VAddCOU32(dst=vgpr(self.coord0Vgpr), dst1=VCC(), src0=vgpr(kw.vgprs.coord0), src1=self.coordOffset0, \ + comment="coord0.1: coord0 += d0*sg0*VW + vc0")) + else: + self.coord0Vgpr = tmpVgpr + module.add(SMovB32(dst=sgpr(tmpS01), src=self.coordOffset0, comment="coordOffset0 d0=%u vc0=%u"%(d0, vc0))) + module.add(VAddCOU32(dst=vgpr(self.coord0Vgpr), dst1=VCC(), src0=vgpr(kw.vgprs.coord0), src1=sgpr(tmpS01), \ + comment="coord0.2: coord0 += d0*sg0*VW + vc0")) + + if self.newCoord1: + if not kernel["BufferStore"] or updateCoord1: + if self.rowInc== 0: + None + elif self.rowInc <= 64: + # rowInc fits in instruction: + module.add(VAddCOU32(dst=vgpr(self.coord1Vgpr), dst1=VCC(), \ + src0=vgpr(self.kernelWriter.vgprs.coord1), src1=self.rowInc, \ + comment="coord1.1: coord1Vgpr += d1*sg1*VW + vc1")) + else: + module.add(SMovB32(dst=sgpr(tmpS01), src=self.rowInc, comment="rowInc d1=%u vc1=%u"%(d0, vc0))) + module.add(VAddCOU32(dst=vgpr(self.coord1Vgpr), dst1=VCC(), \ + src0=vgpr(self.kernelWriter.vgprs.coord1), src1=sgpr(tmpS01), \ + comment="coord1.2: coord1 += d1*sg1*VW + vc1")) + return module + + # storeChar is 'C' or 'D' + # elementVgpr is coord0Vgpr*strideCD0, or optimized to just coord0Vgpr if strideCD0 is unit const + def emitExtractAndScalePackedDims(self, kernel, ss, tmpVgpr, storeChar): + module = Module("emitExtractAndScalePackedDims") + kw = self.kernelWriter + packedIndices = kernel["PackedC0IndicesX"] + packedBits = self.coord0Vgpr # start with coord0, will move to temp below + rowPtr = kw.vgprs.cinRowPtr if (storeChar == 'C') else kw.vgprs.coutRowPtr + addrVgpr = self.addrCVgpr if (storeChar == 'C') else self.addrDVgpr + + for i,idx in enumerate(packedIndices[:-1]): + # vgprTmp assignments: + # - tmp+0 may be the incoming packed coordinate 0, used on replay too + # - tmp+1 is DIV output + # - tmp+2 is scratch + idxChar= globalParameters["IndexChars"][idx] + module.addComment0("extract %s"%kw.sizeRef(idx)) + assert(tmpVgpr+1 != packedBits) # bad since we still need packedBits below for remainder (can't overwrite here) + module.add(MacroInstruction("V_MAGIC_DIV", \ + args=[tmpVgpr+1, vgpr(packedBits), sgpr("MagicNumberSize%s"%idxChar), \ + sgpr("MagicShiftSize%s"%idxChar), sgpr("MagicAbitSize%s"%idxChar) if kernel["MagicDivAlg"]==2 else "0"])) + # tmpVgpr+1 returns the quotient, tmpVgpr+2 is overwritten + + # compute remainder, packedBits % sizeIdx - this is the 'extracted' index that must be scaled + # remainder is mul and sub + module.add(VMulLOU32(dst=vgpr(tmpVgpr+2), src0=vgpr(tmpVgpr+1), src1=kw.sizeRef(idx), \ + comment="remainder part 1")) + module.add(VSubU32(dst=vgpr(tmpVgpr+2), src0=vgpr(packedBits), src1=vgpr(tmpVgpr+2), + comment="remainder part 2")) + + if i==0: + module.add(VMulLOU32(dst=vgpr(addrVgpr), src0=vgpr(tmpVgpr+2), \ + src1=kw.strideRef(storeChar, idx), comment="addrCalc <- scaled extracted dim")) + else: + module.add(VMulLOU32(dst=vgpr(tmpVgpr+2), src0=vgpr(tmpVgpr+2), \ + src1=kw.strideRef(storeChar, idx), comment="scale extracted dim")) + module.add(VAddU32(dst=vgpr(addrVgpr), src0=vgpr(addrVgpr), \ + src1=vgpr(tmpVgpr+2), comment="addrCalc += scaled extracted dim ")) + + if i < len(packedIndices)-2: + # TODO - might be able to eliminate this + module.add(VMovB32(dst=vgpr(tmpVgpr+0), src=vgpr(tmpVgpr+1), \ + comment="Copy remaining bits for next divide")) + packedBits = tmpVgpr+0 + + if len(packedIndices)>1: + # if we unpacked something, then scale it to BPE + module.addComment0("extract final %s"%kw.sizeRef(packedIndices[-1])) + module.add(VMulLOU32(dst=vgpr(tmpVgpr+2), src0=vgpr(tmpVgpr+1), \ + src1=kw.strideRef(storeChar, packedIndices[-1]), comment="scale final extracted dim")) + module.add(VAddU32(dst=vgpr(addrVgpr), src0=vgpr(addrVgpr), \ + src1=vgpr(tmpVgpr+2), comment="addrCalc += scaled extracted dim ")) + + module.add(VAddLShiftLeftU32(dst=vgpr(addrVgpr), \ + src0=vgpr(rowPtr), \ + src1=vgpr(addrVgpr), \ + shiftHex=hex(log2(kw.bpeCexternal)), \ + comment="packed: add rowPtr and scaleToBpe")) + + return module + + def emitScaleToBpe(self, kernel, ss, tmpVgpr, singleUpdate, tc): + """ + Needs 3 temporary VGPRs + """ + + module = Module("emitScaleToBpe") + kw = self.kernelWriter + (d1,d0,vc1,vc0) = self.element + rowPtr = kw.vgprs.cinRowPtr if (tc == 'C') else kw.vgprs.coutRowPtr + addrVgpr = self.addrCVgpr if (tc == 'C') else self.addrDVgpr + # set when we generate code that updates the address + # optSingleColVgpr and optSharedColVgpr attempt to minimize these updates + updatedAddr = False + + # scale and set final address: + stride0 = kw.strideRef('D', 0) if tc == 'Bias' else kw.strideRef(tc, 0) + if kw.isConstUnitStride(stride0): + elementVgpr = self.coord0Vgpr + else: + module.add(VMulLOU32(dst=vgpr(addrVgpr), \ + src0=vgpr(self.coord0Vgpr), \ + src1=stride0, \ + comment="scale element by non-unit stride")) + elementVgpr = addrVgpr + + if ss.optSingleColVgpr: + # This is first element in the first batch, create a byte address that will + # be re-used by subsequent elements: + # if this element is firstInBatch - may need to set up a bpe-scaled row pointer for the batch: + # - need row-ptr start of each batch + assert (kw.vgprs.coord0 == self.coord0Vgpr) # elementAddr assignment above assumes these are the same + if singleUpdate: + updatedAddr = True + singleColAddrUpdated = ss.singleColCAddrUpdated if (tc == 'C') else ss.singleColDAddrUpdated + if not singleColAddrUpdated or not ss.optSrdIncForRow: + if tc == 'Bias' and kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + module.add(VLShiftLeftB32(dst=vgpr(self.addrBiasVgpr), \ + shiftHex=hex(log2(kw.bpeAB)), \ + src=vgpr(self.coord0Vgpr), \ + comment="Bias address scaled by BPE")) + return module + if tc == 'C': + ss.singleColCAddrUpdated = True + else: + ss.singleColDAddrUpdated = True + module.add(VAddLShiftLeftU32(dst=vgpr(addrVgpr), \ + src0=vgpr(rowPtr), \ + src1=vgpr(elementVgpr), \ + shiftHex=hex(log2(kw.bpeCexternal)), \ + comment="optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=%d, coord0Vgpr=%d"%(kw.vgprs.coord0, self.coord0Vgpr))) + elif ss.optSharedColVgpr: + # Need an address calculation for the first address in each row: + if d1==0 and vc1==0: + if tc == 'Bias' and kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + module.add(VLShiftLeftB32(dst=vgpr(self.addrBiasVgpr), shiftHex=hex(log2(kw.bpeAB)), src=vgpr(self.coord0Vgpr), \ + comment="Bias address scaled by BPE")) + return module + packedIndices = kernel["PackedC0IndicesX"] + if len(packedIndices) > 1: + updatedAddr = True + module.add(self.emitExtractAndScalePackedDims(kernel, ss, tmpVgpr, tc)) + else: + updatedAddr = True + module.add(VAddLShiftLeftU32(dst=vgpr(addrVgpr), \ + src0=vgpr(rowPtr), \ + src1=vgpr(elementVgpr), \ + shiftHex=hex(log2(kw.bpeCexternal)), \ + comment="optSharedColVgpr scaleToBpe for first row: col addr <- cinRowPtr + coord0, scaled by BPE")) + else: + # Generate final address calculation (to bytes) for each element + # The unpacking takes 8-10 instructions so could be worth optimizing someday : + # each col has same offset so could create a class to hold column-specific state including + # the byte address offset for that col and the mask in/out. + if tc == 'Bias' and kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + module.add(VLShiftLeftB32(dst=vgpr(self.addrBiasVgpr), shiftHex=hex(log2(kw.bpeAB)), src=vgpr(self.coord0Vgpr), \ + comment="Bias address scaled by BPE")) + return module + packedIndices = kernel["PackedC0IndicesX"] + if len(packedIndices) > 1: + updatedAddr = True + module.add(self.emitExtractAndScalePackedDims(kernel, ss, tmpVgpr, tc)) + else: + updatedAddr = True + module.add(VAddLShiftLeftU32(dst=vgpr(addrVgpr), \ + src0=vgpr(rowPtr), \ + src1=vgpr(elementVgpr), \ + shiftHex=hex(log2(kw.bpeCexternal)), \ + comment="scaleToBpe: accumulate d0 lower and *= bpe into Cin addr")) + + # if not optSrdIncForRow then we may have moved the row pointer + # and depending on paths above may not have refreshed addrVgpr already. + # if so - do it here: + if self.rowIncDirtyRowPtr and not updatedAddr: + module.add(VAddLShiftLeftU32(dst=vgpr(addrVgpr), \ + src0=vgpr(rowPtr), \ + src1=vgpr(kw.vgprs.coord0), \ + shiftHex=hex(log2(kw.bpeCexternal)), \ + comment="scaleToBpe: Update address with new rowPtr")) + + return module + + def edgeProtectCode(self, kernel, edge, beta, atomic, mask, tmpSgpr): + """ + Generate code to protect address offset in edge case + """ + + module = Module("edgeProtectCode") + kw = self.kernelWriter + tmpS01 = tmpSgpr + tmpS23 = tmpSgpr+self.kernelWriter.laneSGPRCount + + laneSGPRCount = self.kernelWriter.laneSGPRCount + wavefrontSize = kernel["WavefrontSize"] + + # Now do the edge check and compute the address in bytes: + if kernel["BufferStore"]: + if edge and (not kernel["StoreRemapVectorWidth"] or (kernel["StoreRemapVectorWidth"] and beta)): + # Set address to -1 if OOB on either dimension + # and only check the x/coord0 index here, save a couple inst + sizeBoundary = [0,0] + sizeBoundary[0] = \ + sgpr("PackedSize0") if len(kernel["PackedC0IndicesX"]) > 1 \ + else kw.sizeRef(kernel["ProblemType"]["Index0"]) + sizeBoundary[1] = \ + sgpr("PackedSize1") if len(kernel["PackedC1IndicesX"]) > 1 \ + else kw.sizeRef(kernel["ProblemType"]["Index1"]) + + module.add(VCmpLtU32(dst=sgpr(tmpS01,laneSGPRCount), src0=vgpr(self.coord0Vgpr), src1=sizeBoundary[0], comment="coord0 < size0" )) + module.add(VCmpLtU32(dst=sgpr(mask,laneSGPRCount), src0=vgpr(self.coord1Vgpr), src1=sizeBoundary[1], comment="coord1 < size1" )) + SAndX = SAndB64 if wavefrontSize == 64 else SAndB32 + module.add(SAndX(dst=sgpr(mask,laneSGPRCount), src0=sgpr(tmpS01,laneSGPRCount), src1=sgpr(mask,laneSGPRCount), comment="in0 && in1" )) + else: + module.add(VCmpLtU32(dst=sgpr(tmpS01,laneSGPRCount), src0=vgpr(self.coord0Vgpr), src1=sgpr("SizesFree+0"), comment="coord0 < size0" )) + module.add(VCmpLtU32(dst=sgpr(tmpS23,laneSGPRCount), src0=vgpr(self.coord1Vgpr), src1=sgpr("SizesFree+1"), comment="coord1 < size1" )) + SAndX = SAndB64 if wavefrontSize == 64 else SAndB32 + module.add(SAndX(dst=sgpr(mask,laneSGPRCount), src0=sgpr(tmpS01,laneSGPRCount), src1=sgpr(tmpS23,laneSGPRCount), comment="in0 && in1" )) + + if (beta or atomic): + SMovX = SMovB64 if wavefrontSize == 64 else SMovB32 + module.add(SMovX(dst=EXEC(), src=sgpr(mask,laneSGPRCount), comment="sgprs -> exec" )) + + return module + + # TODO - mask should be part of AddrCalc state not passed as parm + def emitAddressSetupCode(self, kernel, tPB, ss, tmpVgpr, tmpS01, edge, beta, atomic, elementIdx, addrVgpr): + """ + Generate code to set up the address vgpr + Input: + tmpVgpr : two temp vgprs + Output: + Returns kStr with appropriate setup code + Sets self.coord0Vgpr with vgpr that contains the coord0 for this element. This enables + optimization - if no setup code is required the coord0 can be the input. + """ + + module = Module("emitAddressSetupCode") + kw = self.kernelWriter + + updateCoord1 = (edge or len(kernel["PackedC1IndicesX"]) > 1) + module.add(self.emitAddressCoordIncrement(kernel, ss, tmpVgpr, tmpS01, updateCoord1)) + + # calculate flat load offset + if not kernel["BufferStore"]: + # flat: in-bounds exec mask + # global offset macro (requires 3 tmpVgpr) + # final address = C + index*bytes + params = ["%u" % addrVgpr] + for i in range(0, kernel["ProblemType"]["NumIndicesC"]): + if i == kernel["ProblemType"]["Index0"]: + params.append("%s" % (self.coord0Vgpr)) + elif i == kernel["ProblemType"]["Index1"]: + params.append("%s" % (self.coord1Vgpr)) + else: # just a group index + params.append("sgprWorkGroup%u"%i) + params.append("%s" % (tmpVgpr+2)) + module.add(MacroInstruction(name="GLOBAL_OFFSET_C", args=params)) + module.add(VMovB32(dst=vgpr(tmpVgpr+2), src=vgpr(addrVgpr+0), comment="temp store offset 0")) + module.add(VMovB32(dst=vgpr(tmpVgpr+3), src=vgpr(addrVgpr+1), comment="temp store offset 1")) + + # Move the row ptr VGPR + # optSrdIncForRow moves the SRD so don't move here + if not ss.optSrdIncForRow and kernel["BufferStore"]: + if self.rowInc > 0: + self.rowIncDirtyRowPtr = 1 + #assert (not kernel["ProblemType"]["UseInitialStridesCD"]) + module.addComment1("Fix for UseInitialStridesCD, emitAddressSetupCode") + + if len(kernel["PackedC1IndicesX"]) == 1: + strideChar = self.kernelWriter.indexChars[kernel["PackedC1IndicesX"][0]] + module.add(self.addScaled(vgpr(kw.vgprs.cinRowPtr), vgpr(kw.vgprs.cinRowPtr), \ + sgpr("StrideC%s"%strideChar), self.rowInc, tmpS01, "ROWINC- Move cinRowPtr to next row")) + module.add(self.addScaled(vgpr(kw.vgprs.coutRowPtr), vgpr(kw.vgprs.coutRowPtr), \ + sgpr("StrideD%s"%strideChar), self.rowInc, tmpS01, "Move coutRowPtr to next row")) + elif len(kernel["PackedC1IndicesX"]) > 1: + module.add(kw.extractPackedCoord1ToRowStart(kernel, kernel["PackedC1IndicesX"] , self.coord1Vgpr, 'D')) + + # Shift Pointer for MFMA: + # For MFMA shift pointer, correct data is stored in another thread. + # Therefore, MFMA cannot use v_mov to amend store data + # It needs to modify the coord1 of thread directly. + if (not kernel["SourceSwap"]) and (not kernel["GuaranteeNoPartialB"]) and tPB["rtv"] and kernel["EnableMatrixInstruction"] and edge: + (d1,d0,vc1,vc0) = self.element + if (d1 == vc1 == d0 == vc0 == 0) or self.newCoord1: + sgprCnt = self.kernelWriter.laneSGPRCount + waveSize = kernel["WavefrontSize"] + packedC1 = kernel["PackedC1IndicesX"] + strideC1 = "StrideC%s" % (kw.indexChars[packedC1[0]]) + strideD1 = "StrideD%s" % (kw.indexChars[packedC1[0]]) + + module.addComment1("shift vector components d1") + vw = kernel["GlobalLoadVectorWidthB"] + vTmp1 = tmpVgpr + vTmp2 = tmpVgpr+1 + sTmp1 = tmpS01 + sTmp2 = tmpS01+sgprCnt + # check conditions + module.add(VBfiB32(dst=vgpr(vTmp1), src0=vw-1, src1=0, src2=vgpr(self.coord1Vgpr), comment="coord1 & ~(vw-1)")) + module.add(VBfiB32(dst=vgpr(vTmp2), src0=vw-1, src1=0, src2=sgpr("SizesFree+%u"%tPB["idx"]), comment="sizeFree & ~(vw-1)")) + module.add(VCmpEQU32(dst=sgpr(sTmp1,sgprCnt), src0=vgpr(vTmp1), src1=vgpr(vTmp2), comment="if coord1 is in edge glvw")) + module.add(VAndB32(dst=vgpr(vTmp2), src0=sgpr("SizesFree+%u"%tPB["idx"]), src1=vw-1, comment="sizeFree mod VW")) + module.add(VCmpGtU32(dst=sgpr(sTmp2,sgprCnt), src0=vgpr(vTmp2), src1=0, comment="this problem is not multiple size of glvw")) + SAndBX = SAndB64 if waveSize == 64 else SAndB32 + module.add(SAndBX(dst=sgpr(sTmp1,sgprCnt), src0=sgpr(sTmp1,sgprCnt), src1=sgpr(sTmp2,sgprCnt), comment="AND both conditions")) + # calculate new coord + module.add(VAddU32(dst=vgpr(vTmp1), src0=vgpr(self.coord1Vgpr), src1=vgpr(vTmp2), comment="shift coord1")) + module.add(VBfiB32(dst=vgpr(vTmp1), src0=vw-1, src1=vgpr(vTmp1), src2=sgpr("SizesFree+%u"%tPB["idx"]), comment="new coord1 = (shift coord1 & (vw-1)) | (sizeFree & ~(vw-1))")) + module.add(VSubI32(dst=vgpr(vTmp2), src0=vgpr(vTmp1), src1=vgpr(self.coord1Vgpr), comment="shift how many column")) + module.add(VCndMaskB32(dst=vgpr(self.coord1Vgpr), src0=vgpr(self.coord1Vgpr), src1=vgpr(vTmp1), \ + src2=sgpr(sTmp1,sgprCnt), comment="set new coord1 if meet conditions" )) + + module.add(VMadI32I24(dst=vgpr(vTmp1), src0=sgpr(strideC1), src1=vgpr(vTmp2), src2=vgpr(kw.vgprs.cinRowPtr), \ + comment="new rowStart address += shift column * StridesC")) + module.add(VCndMaskB32(dst=vgpr(kw.vgprs.cinRowPtr), src0=vgpr(kw.vgprs.cinRowPtr), src1=vgpr(vTmp1), src2=sgpr(sTmp1,sgprCnt), \ + comment="set new rowStart if meet conditions" )) + module.add(VMadI32I24(dst=vgpr(vTmp1), src0=sgpr(strideD1), src1=vgpr(vTmp2), src2=vgpr(kw.vgprs.coutRowPtr), \ + comment="new rowStart address += shift column * StridesD")) + module.add(VCndMaskB32(dst=vgpr(kw.vgprs.coutRowPtr), src0=vgpr(kw.vgprs.coutRowPtr), src1=vgpr(vTmp1), src2=sgpr(sTmp1,sgprCnt), \ + comment="set new rowStart if meet conditions" )) + + if kernel["StoreRemapVectorWidth"]: + ldsPad = max(kernel["StoreRemapVectorWidth"],kernel["MIOutputVectorWidth"]) + module.add(VMovB32(dst=vgpr(vTmp1), src=hex((kernel["MacroTile0"]+ldsPad)*kw.bpeCexternal), \ + comment="lds byte stride = (MT0 + PAD) * bpe")) + module.add(VMadI32I24(dst=vgpr(vTmp1), src0=vgpr(vTmp1), src1=vgpr(vTmp2), src2=vgpr(kw.vgprs.storeRemapLW), \ + comment="new lds write address += shift column * Lds byte Stride")) + module.add(VCndMaskB32(dst=vgpr(kw.vgprs.storeRemapLW), src0=vgpr(kw.vgprs.storeRemapLW), src1=vgpr(vTmp1), \ + src2=sgpr(sTmp1,sgprCnt), comment="set new rowStart if meet conditions" )) + + module.addSpaceLine() + + return module + + def emitLdChange(self, kernel, ss, tc, edge, beta, mask, singleUpdate, tmpVgpr, addrVgpr, BufAddr): + """ + Generate code for final C read/D write address + """ + + laneSGPRCount = self.kernelWriter.laneSGPRCount + + module = Module("emitLdChange") + if kernel["BufferStore"]: + module.add(self.emitScaleToBpe(kernel, ss, tmpVgpr, singleUpdate, tc)) + if edge and (not kernel["StoreRemapVectorWidth"] or (kernel["StoreRemapVectorWidth"] and beta)) and \ + (tc != 'Bias'): + module.add(VCndMaskB32(dst=vgpr(addrVgpr), src0=-1, src1=vgpr(addrVgpr), \ + src2=sgpr(mask,laneSGPRCount), comment="LD%s clip if OOB. offset" % tc )) + else: + if tc == 'Bias' and kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + module.add(VLShiftLeftB32(dst=vgpr(self.addrBiasVgpr+0), shiftHex=hex(log2(self.kernelWriter.bpeAB)), src=vgpr(self.coord0Vgpr), \ + comment="Bias address scaled by BPE")) + else: + # store a copy of the offset in 2 of the tmpVgpr for D + module.add(VAddCOU32(dst=vgpr(addrVgpr+0), dst1=VCC(), src0=vgpr(BufAddr+0), src1=vgpr(tmpVgpr+2), \ + comment="addrVgpr = C(D) + index*bytes (lo)" )) + module.add(VAddCCOU32(dst=vgpr(addrVgpr+1), dst1=VCC(), src0=vgpr(BufAddr+1), src1=vgpr(tmpVgpr+3), \ + src2=VCC(), comment="addrVgpr = C(D) + index*bytes (hi)")) + return module + + def incrementToNextRow(self, kernel, tc, ss, stmp): + """ + Generate code to move to the next row(s) + If optSrdIncForRow, this will move the SRD forward + If not, this could generate some other instructions + """ + + module = Module("incrementToNextRow") + numRows = self.rowInc + tmpBpe = self.kernelWriter.bpeCexternal + if ss.optSrdIncForRow: + if numRows: + packedC1 = kernel["PackedC1IndicesX"] + assert(len(packedC1) == 1) # would need to extract each dim and scale + strideCD1 = "Stride%s%s"%(tc,self.kernelWriter.indexChars[packedC1[0]]) + if numRows > 1: + module.add(SMulI32(dst=sgpr(stmp), \ + src0=sgpr(strideCD1), \ + src1=numRows*tmpBpe, \ + comment="scale Stride%s *= numRows(%u) * bpe"%(tc,numRows))) + else: + module.add(SLShiftLeftB32(dst=sgpr(stmp), \ + src=sgpr(strideCD1), \ + shiftHex=log2(tmpBpe), \ + comment="incToNextRow: Scale by BPE")) + + module.add(SAddU32(dst=sgpr("Srd%s+0"%(tc)), \ + src0=sgpr("Srd%s+0"%(tc)), \ + src1=sgpr(stmp), \ + comment="incToNextRow: gra SRD += inc(lower)" )) + module.add(SAddCU32(dst=sgpr("Srd%s+1"%(tc)), \ + src0=sgpr("Srd%s+1"%(tc)), \ + src1=0, \ + comment="incToNextRow: gra SRD += inc(upper)" )) + + None + + return module diff --git a/tensilelite/Tensile/AsmMemoryInstruction.py b/tensilelite/Tensile/AsmMemoryInstruction.py new file mode 100644 index 0000000000..00f69a28f9 --- /dev/null +++ b/tensilelite/Tensile/AsmMemoryInstruction.py @@ -0,0 +1,63 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Common import printExit +from .TensileInstructions import DSStoreB8, DSStoreB8HID16, ReadWriteInstruction, \ + DSLoadD16HIU16, DSLoadD16HIU8, \ + DSLoadU8, DSLoadU16, DSStoreD16HIB16, \ + DSStoreB16 + +from dataclasses import dataclass, field +from typing import Type + +################################################################################ +# Memory Instruction +################################################################################ +@dataclass +class MemoryInstruction: + inst: Type[ReadWriteInstruction] + numAddresses: int + numOffsets: int + offsetMultiplier: int + blockWidth: float + numBlocks: int = field(init=False) + totalWidth: float = field(init=False) + issueLatency: int = field(init=False) + + def __post_init__(self): + self.numBlocks = 2 if self.numAddresses > 1 or self.numOffsets > 1 else 1 + self.totalWidth = self.blockWidth * self.numBlocks + self.issueLatency = self.inst.issueLatency() + + def getInst(self, highBits=0): + if highBits: + if self.inst is DSLoadU8: + return DSLoadD16HIU8 + elif self.inst is DSLoadU16: + return DSLoadD16HIU16 + elif self.inst is DSStoreB16: + return DSStoreD16HIB16 + elif self.inst is DSStoreB8: + return DSStoreB8HID16 + else: + printExit(str(self.inst) + " does not support high bits instructions.") + + return self.inst diff --git a/tensilelite/Tensile/AsmStoreState.py b/tensilelite/Tensile/AsmStoreState.py new file mode 100644 index 0000000000..48ad30c022 --- /dev/null +++ b/tensilelite/Tensile/AsmStoreState.py @@ -0,0 +1,445 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .AsmAddressCalculation import AddrCalculation + +from math import ceil, trunc, modf + +############################################################################## +# StoreState +# tracks state that is preserved across globalWriteBatch calls: +# init is called before globalWriteBatch +# the KernelWriter object +############################################################################## +class StoreState: + + ############################################################################## + # Setup store config for number of sgpr and vgpr needed + # These are set based on edge, atomic, etc - do not change during + # the generation of the store code. + ############################################################################## + class StoreConstConfig: + def __init__(self, kernelWriter, kernel, ss, gwvw, edge, beta, atomic): + self.gwvw = gwvw + + if ss.optSingleColVgpr: + # use one vgpr (allocated in ss.sharedColDVgprs) for all addressing + # - need 0 additional vgpr per element. + self.numVgprsPerAddr = 0 + else: + self.numVgprsPerAddr = kernelWriter.rpgo if kernel["BufferStore"] else kernelWriter.rpga + + if ss.optSGPRUsage == 'BufferLoad_Mask': + self.numMaskSgprPerElement = 0 + self.numMaskSgprPerBatch = 0 + self.numTempSgprPerBatch = kernelWriter.laneSGPRCount + elif ss.optSGPRUsage == 'BufferLoad_Edge_Mask': + self.numMaskSgprPerElement = 0 + self.numMaskSgprPerBatch = kernelWriter.laneSGPRCount + self.numTempSgprPerBatch = 2 * kernelWriter.laneSGPRCount + else: + self.numMaskSgprPerElement = kernelWriter.laneSGPRCount + self.numMaskSgprPerBatch = 0 + self.numTempSgprPerBatch = 2 * kernelWriter.laneSGPRCount + + if self.numMaskSgprPerElement: + numSgprAvailable = kernelWriter.maxSgprs - kernelWriter.sgprPool.size() + kernelWriter.sgprPool.availableBlockAtEnd() + numSgprAvailable = numSgprAvailable & ~0x1 # make sure it's aligned + #print("numSgprAvailable=", numSgprAvailable) + self.numElementsPerBatchLimitedBySgprs = (numSgprAvailable - self.numTempSgprPerBatch - self.numMaskSgprPerBatch) // self.numMaskSgprPerElement + else: + self.numElementsPerBatchLimitedBySgprs = 9999 # no limit + + if self.numElementsPerBatchLimitedBySgprs<=0: + kernelWriter.overflowedResources = 2 + self.numElementsPerBatchLimitedBySgprs = 1 # dummy value + #assert self.numElementsPerBatchLimitedBySgprs > 0, "numElementsPerBatchLimitedBySgprs=0 for %s"%self.kernelName + + if atomic: + # flat atomics have another VGPR to allow different data for return# + regsPerElement = 2 if kernel["BufferStore"] else (3 + 1) # + 1 for alignment + # The atomic loop processes multiple elements in single instruction + # so will use VGPR from consec elements? TODO + self.numVgprsPerDataPerVI = (1.0 * regsPerElement * kernelWriter.bpeCexternal) / kernelWriter.bpr + elif beta: + self.numVgprsPerDataPerVI = (1.0 * kernelWriter.bpeCexternal) / kernelWriter.bpr + else: + self.numVgprsPerDataPerVI = 0.0 + + if kernelWriter.serializedStore: + #self.numVgprPerValuC = kernel["MIRegPerOut"] + self.numVgprPerValuC = kernelWriter.bpeCinternal//kernelWriter.bpr # vgpr needed from register pool + else: + self.numVgprPerValuC = 0 # null since they are already declared in macro part of assembly kernel + + # indicates each vector element is actually half - + # changes vgpr allocation so two elements share a data vgpr + # Really only used if gwvw=1 - edge cases + # exception: data vgpr cannot be shared if UseInitialStridesCD is enabled and card enable EccHalf, + # since each buffer_load_short would overwrite undefined 16bit as zero. + self.halfDataRegPerVI = gwvw*self.numVgprsPerDataPerVI < 1.0 and not (kernel["ProblemType"]["UseInitialStridesCD"] and kernelWriter.archCaps["HasEccHalf"]) + + # StoreState constructor: + def __init__(self, kernelWriter, kernel, gwvw, edge, beta, atomic, elements): + self.kernelWriter = kernelWriter + self.kernel = kernel + + #-- + # Optimizations for coord0/column address calculations: + # + # optSingleColVgpr: + # - works in cases where the data is written row by row to memory. + # In this case we can use a single vgpr for addressing: + # - Use the load/store instruction offset (fixed at compile-time) + # - the horizontal addresses are fixed offsets from the base + # - as we move to a new row, increment the appropriate SRDs + + # optSharedColVgpr: + # - Each col gets it's own address, but elements in later rows with the same col will share VGPR. + # - allows cols to be non-adjacent + # - this is mutually exclusive with optSingleColVgpr - not as optimal but provides + # more flexibility. + + # optSrdIncForRow: optimize coord1/row address calculations: + # - Move the SRD between memory operations to get to new row + # atomic needs to reset the SRD to handle retry loop. Then might work. + + self.optSingleColVgpr = 0 + self.optSharedColVgpr = 0 + self.optSrdIncForRow = 0 + + # opt*ColVgpr doesn't work for edge since each element requires own addr VGPR so + # we can perform bounds check and set to -1 for OOB accesses. + # if optSingleColVgpr = optSharedColVgpr = 0, then each element gets + # 1-2 VGPRs to track address. Address calcs are performed independently + # for each element. + + # atomic contains multiple memory operations which need to preserve + # the address for each load. Memops in same row can use offsets + # and share a base register but Memops in different rows need + # different registers or need to inteligently reset the SRD. + if kernel["BufferStore"] and not edge and not atomic: + if len(kernel["PackedC0IndicesX"]) > 1: + # packed mode needs a unique VGPR address calc for each column. + self.optSharedColVgpr = 1 + elif len(kernel["PackedC1IndicesX"]) > 1: + self.optSharedColVgpr = 0 + self.optSingleColVgpr = 0 + else: + self.optSingleColVgpr = 1 + + if not atomic and len(kernel["PackedC1IndicesX"]) == 1: + self.optSrdIncForRow = 1 + + if kernel["StoreRemapVectorWidth"]: + self.optSrdIncForRow = 1 + + if kernel["ProblemType"]["UseInitialStridesCD"]: + self.optSingleColVgpr = 0 # BOZO, hack to disable this + self.optSharedColVgpr = 0# BOZO, hack to disable this + + self.optSGPRUsage = None + if kernel["BufferStore"] and (not atomic): + self.optSGPRUsage = 'BufferLoad_Edge_Mask' if edge else 'BufferLoad_Mask' + + # can't have both of these enabled: + assert (not (self.optSingleColVgpr and self.optSharedColVgpr)) + + + self.cfg = self.StoreConstConfig(kernelWriter, kernel, self, gwvw, edge, beta, atomic) + + # Use to detect new rows: + self.lastCoordOffset1 = 0 + + # vgpr holding current coord, setup initial state + self.coord1Vgpr = kernelWriter.vgprs.coord1 + + if self.optSharedColVgpr: + numCols = len([e for e in elements if e[0] == 0 and e[2] == 0]) # count #elements with row d1=v1==0 + self.numAddrVgpr = numCols + self.sharedColDVgprs = kernelWriter.vgprPool.checkOut(self.numAddrVgpr, "sharedColDVgprs for packed elements") + if kernel["GroupLoadStore"] and kernel["ProblemType"]["UseBeta"]: + self.sharedColCVgprs = kernelWriter.vgprPool.checkOut(self.numAddrVgpr, "sharedColCVgprs for packed elements") + else: + self.sharedColCVgprs = self.sharedColDVgprs + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + self.sharedColBiasVgprs = kernelWriter.vgprPool.checkOut(self.numAddrVgpr, "sharedColBiasVgprs for packed elements") + else: + self.sharedColBiasVgprs = None + elif self.optSingleColVgpr: + self.numAddrVgpr = 1 + self.sharedColDVgprs = kernelWriter.vgprPool.checkOut(1, "sharedColDVgprs") + self.singleColDAddrUpdated = False + self.singleColCAddrUpdated = False + if kernel["ProblemType"]["UseBeta"]: + self.sharedColCVgprs = kernelWriter.vgprPool.checkOut(1, "sharedColCVgprs") + else: + self.sharedColCVgprs = self.sharedColDVgprs + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + self.sharedColBiasVgprs = kernelWriter.vgprPool.checkOut(1, "sharedColBiasVgprs for packed elements") + else: + self.sharedColBiasVgprs = None + else: + self.numAddrVgpr = 0 + self.sharedColDVgprs = None + self.sharedColCVgprs = None + self.sharedColBiasVgprs = None + + # For detecting when we are running first batch + self.firstBatch = True + + # Calculate numVgprsPerElement + # print("numVgprsPerAddr=%u, numVgprsPerDataPerVI=%u, numVgprPerValuC=%u"%(self.cfg.numVgprsPerAddr, self.cfg.numVgprsPerDataPerVI, self.cfg.numVgprPerValuC)) + self.numVgprsPerElement = self.cfg.numVgprPerValuC*gwvw + self.cfg.numVgprsPerAddr + int(ceil(self.cfg.numVgprsPerDataPerVI * gwvw)) + if kernel["GroupLoadStore"] and kernel["ProblemType"]["UseBeta"]: + self.numVgprsPerElement += self.cfg.numVgprsPerAddr + if kernel["ProblemType"]["UseBias"]: + self.numVgprsPerElement += self.cfg.numVgprsPerAddr # Bias address + numVgprs = int(ceil(kernel["ProblemType"]["DataType"].numRegisters())) + self.numVgprsPerElement += numVgprs * gwvw # Loaded data + + # Calculate align + self.align = 1 + # align adjustment + if self.cfg.numVgprsPerAddr > 1: + self.align = max(self.align, self.cfg.numVgprsPerAddr) + if self.cfg.numVgprPerValuC*gwvw > 1: + self.align = max(self.align, self.cfg.numVgprPerValuC*gwvw) + if int(ceil(self.cfg.numVgprsPerDataPerVI * gwvw)) > 1: + self.align = max(self.align, int(ceil(self.cfg.numVgprsPerDataPerVI * gwvw))) + + ############################################################################## + # Setup data structures to feed store loops: + # self.elementAddr, self.elementData, self.elementMask, self.elementSumIdx + # batchElements is a list of (d0,d1,v0,v1) for which stores to perform + # batchElementSgprs is SGPRs to use for mask. If None, elementMask is + # not initialized. + # + # Also create an AddrCalc for each memory operation. + ############################################################################## + def setupStoreElementsForBatch(self, kernel, gwvw, batchElements, batchElementSgprs, isOptNLL, allowLRVWforTLUandMI, lrvwB): + + self.elementAddr = [] + self.elementData = [] # VGPR to use for element data, needed for atomic or beta + self.elementDataBias = [] + self.elementMask = [] # SGPR to use for element mask + self.elementSumIdx = [] + + kw = self.kernelWriter + + if kernel["EnableMatrixInstruction"]: + matrixInstM = (kernel["MatrixInstM"] * kernel["MatrixInstBM"]) if (kernel["MatrixInstM"] == 4) else kernel["MatrixInstM"] + matrixInstN = (kernel["MatrixInstN"] * kernel["MatrixInstBN"]) if (kernel["MatrixInstN"] == 4) else kernel["MatrixInstN"] + matrixInstBM = 1 if (kernel["MatrixInstM"] == 4) else kernel["MatrixInstBM"] + matrixInstBN = 1 if (kernel["MatrixInstN"] == 4) else kernel["MatrixInstBN"] + + biasVgprMap = {} + lastData = 0 + for elementIdx in range(0, len(batchElements)): + # Create the AddrCalc for each memory load/store + # This is the control code that sets up the dest, source, offsets, etc and + # identifies cases where the AddrCalc is a new row and therefore needs some + # additional math. Each AddrCalc contains isolated state sufficient to + # perform any needed range checks and address calculations for the element. + # + # The AddrCalc creation code here maintains state across elements (including + # across write batches) to remove replicated calculations. + # + # Later the AddrCalc::emitAddressSetupCode will emit the necessary code + # Also allocate VGPR resources here, if needed. + + element = batchElements[elementIdx] + (d1,d0,vc1,vc0) = element + + coordOffset1 = 0 + if kernel["EnableMatrixInstruction"]: + vc1Scale = lrvwB if allowLRVWforTLUandMI else 1 + MIOutputVectorWidth = kernel["MIOutputVectorWidth"] + MFMAContinuousOutputs = MIOutputVectorWidth if kernel["SourceSwap"] else 1 + OutputsPerMIMN = (matrixInstM * matrixInstN // self.kernel["WavefrontSize"]) if kernel["SourceSwap"] else 1 + + eIdx1 = d1 % (OutputsPerMIMN // MFMAContinuousOutputs) + remain_d1 = d1 // (OutputsPerMIMN // MFMAContinuousOutputs) + + bIdx1 = remain_d1 % matrixInstBN + remain_d1 = remain_d1 // matrixInstBN + wtIdex = remain_d1 % kernel["MIWaveTile"][1] + + coordOffset1 = eIdx1 * (self.kernel["WavefrontSize"] // matrixInstN) * MFMAContinuousOutputs + coordOffset1 += bIdx1 * matrixInstN + coordOffset1 += wtIdex * matrixInstN * matrixInstBN * kernel["MIWaveGroup"][1] + coordOffset1 = coordOffset1 * vc1Scale + vc1 + else: + if kernel["LocalSplitU"] > 1: + strideD1 = (kernel["NumThreads"]*kernel["VectorWidth"]//kernel["MacroTile0"]) + else: + strideD1 = (kernel["SubGroup1"] * kernel["VectorWidth"]) + coordOffset1 = d1 * strideD1 + vc1 + + newCoord1 = (self.firstBatch and elementIdx==0) or (coordOffset1 != self.lastCoordOffset1) + + # gpr and offset assignments for element + coordOffset0 = 0 + if kernel["EnableMatrixInstruction"]: + vectorWidth = kernel["VectorWidth"] if kernel["SourceSwap"] else 1 # TODO: nonSwap VectorWidth + MFMAContinuousOutputs = 1 if kernel["SourceSwap"] else kernel["MIOutputVectorWidth"] + OutputsPerMIMN = 1 if kernel["SourceSwap"] else matrixInstM * matrixInstN // self.kernel["WavefrontSize"] + + eIdx0 = d0 % (OutputsPerMIMN // MFMAContinuousOutputs) + remain_d0 = d0 // (OutputsPerMIMN // MFMAContinuousOutputs) + + bIdx0 = remain_d0 % matrixInstBM + remain_d0 = remain_d0 // matrixInstBM + wtIdex = remain_d0 % kernel["MIWaveTile"][0] + + coordOffset0 = eIdx0 * vectorWidth * (self.kernel["WavefrontSize"] // matrixInstM) * MFMAContinuousOutputs + coordOffset0 += bIdx0 * vectorWidth * matrixInstM + coordOffset0 += wtIdex * vectorWidth * matrixInstM * matrixInstBM * kernel["MIWaveGroup"][0] + coordOffset0 += vc0 + else: + coordOffset0 = d0 * kernel["SubGroup0"]*kernel["VectorWidth"] + vc0 + + if self.optSingleColVgpr: + # use same address vgpr for all + addrDVgpr = self.sharedColDVgprs + addrCVgpr = self.sharedColCVgprs + addrBiasVgpr = self.sharedColBiasVgprs + elif self.optSharedColVgpr: + if kernel["EnableMatrixInstruction"]: + elementCol = (d0 * kernel["MIOutputVectorWidth"] + vc0) / gwvw + else: + elementCol = (d0 * kernel["VectorWidth"] + vc0) / gwvw + assert (modf(elementCol)[0] < 0.001) + elementCol = trunc(elementCol) + addrDVgpr = self.sharedColDVgprs+elementCol + addrCVgpr = self.sharedColCVgprs+elementCol + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + addrBiasVgpr = self.sharedColBiasVgprs+elementCol + else: + addrBiasVgpr = None + #print ("d0=", d0, "vc0=", vc0, "elementCol=", elementCol) + else: + # allocate new VGPR for each element: + addrDVgpr = kw.vgprPool.checkOutAligned(self.cfg.numVgprsPerAddr, \ + int(ceil(self.cfg.numVgprsPerAddr)), "writeDBatch-addr for ei=%u"%(elementIdx), preventOverflow=not isOptNLL) + if kernel["GroupLoadStore"] and kernel["ProblemType"]["UseBeta"]: + addrCVgpr = kw.vgprPool.checkOutAligned(self.cfg.numVgprsPerAddr, \ + int(ceil(self.cfg.numVgprsPerAddr)), "loadCBatch-addr for ei=%u"%(elementIdx), preventOverflow=not isOptNLL) + else: + addrCVgpr = addrDVgpr + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + addrBiasVgpr = kw.vgprPool.checkOutAligned(self.cfg.numVgprsPerAddr, \ + int(ceil(self.cfg.numVgprsPerAddr)), "loadBiasBatch-addr for ei=%u"%(elementIdx), preventOverflow=not isOptNLL) + else: + addrBiasVgpr = None + + self.elementAddr.append(AddrCalculation(kw, self, addrCVgpr, addrDVgpr, addrBiasVgpr, element, coordOffset0, \ + self.kernelWriter.vgprs.coord1, coordOffset1, coordOffset1 - self.lastCoordOffset1, newCoord1)) + # if numVgprsPerDataPerVI == 0.5, then two consecutive elements + # should have same data pointer, next should move. + + if self.cfg.numVgprsPerDataPerVI > 0: + if self.cfg.halfDataRegPerVI: + # TODO- check (H,H,H,H,S,S) + if kernel["ProblemType"]["HighPrecisionAccumulate"] and \ + (kernel["ProblemType"]["DataType"].isBFloat16() or kernel["ProblemType"]["DataType"].isHalf()): + data = kw.vgprPool.checkOutAligned(int(2*self.cfg.numVgprsPerDataPerVI*self.cfg.gwvw), \ + int(ceil(int(2*self.cfg.numVgprsPerDataPerVI*self.cfg.gwvw))), "writeBatch-data for ei=%u and ei=%u"%(elementIdx,elementIdx+1), preventOverflow=not isOptNLL) + else: + if elementIdx%2 == 0: + # allocate for two elements: + data = kw.vgprPool.checkOutAligned(int(2*self.cfg.numVgprsPerDataPerVI*self.cfg.gwvw), \ + int(ceil(int(2*self.cfg.numVgprsPerDataPerVI*self.cfg.gwvw))), "writeBatch-data for ei=%u and ei=%u"%(elementIdx,elementIdx+1), preventOverflow=not isOptNLL) + lastData = data + else: + data = lastData + del lastData + else: + if self.cfg.numVgprsPerDataPerVI == 0.5: + data = kw.vgprPool.checkOutAligned(int(ceil(self.cfg.numVgprsPerDataPerVI*self.cfg.gwvw)), \ + int(ceil(self.cfg.numVgprsPerDataPerVI*self.cfg.gwvw)), "writeBatch-data for ei=%u"%elementIdx, preventOverflow=False) + else: + data = kw.vgprPool.checkOutAligned(int(self.cfg.numVgprsPerDataPerVI*self.cfg.gwvw), \ + int(ceil(self.cfg.numVgprsPerDataPerVI*self.cfg.gwvw)), "writeBatch-data for ei=%u"%elementIdx, preventOverflow=False) + #data = kw.vgprPool.checkOut(int(self.cfg.numVgprsPerDataPerVI*self.cfg.gwvw), \ + # "writeBatch-data for ei=%u"%elementIdx, preventOverflow=False) + else: + data = 0 + + self.elementData.append(data) + + if kernel["ProblemType"]["UseBias"]: + if coordOffset0 in biasVgprMap: + dataBias = biasVgprMap[coordOffset0] + else: + numVgprs = int(ceil(kernel["ProblemType"]["DataType"].numRegisters())) + dataBias = kw.vgprPool.checkOutAligned(int(numVgprs*self.cfg.gwvw), \ + int(ceil(numVgprs*self.cfg.gwvw)), "bias data for ei=%u"%elementIdx, preventOverflow=False) + biasVgprMap[coordOffset0] = dataBias + else: + dataBias = 0 + self.elementDataBias.append(dataBias) + + if batchElementSgprs != None: + if self.optSGPRUsage: + mask = batchElementSgprs + else: + mask = batchElementSgprs + self.cfg.numMaskSgprPerBatch + elementIdx * self.cfg.numMaskSgprPerElement + self.elementMask.append(mask) + + #print "Edge=", edge, element + sumIdx = 0 + if kernel["LocalSplitU"] > 1: + sumIdx = kw.startVgprValuC + vc0 + d1*kernel["VectorWidth"] + else: + bestVw = kernel["VectorWidth"] + elementsLoadedPerVw = kernel["NumThreads"] * bestVw + elementsLoadedPerbestVw = kernel["NumThreads"] * kernel["StoreVectorWidth"] + + if elementsLoadedPerVw < elementsLoadedPerbestVw: + bestVw = kernel["StoreVectorWidth"] + + if kernel["EnableMatrixInstruction"]: + alignment = self.cfg.numVgprPerValuC * self.cfg.gwvw + sumIdx = kw.vgprPool.checkOutAligned(self.cfg.numVgprPerValuC*self.cfg.gwvw, alignment, "vgprValuC") // self.cfg.numVgprPerValuC + else: + sumIdx = kw.startVgprValuC + vc0 + d0*kernel["VectorWidth"] + vc1*kernel["ThreadTile0"] + d1*kernel["VectorWidth"]*kernel["ThreadTile0"] + self.elementSumIdx.append(sumIdx) # sumIdx is an element idx, need to div/2 for half + self.lastCoordOffset1 = coordOffset1 + + def checkInTempVgprC(self): + if self.kernelWriter.serializedStore is False: + return # early exit; currently only serializedStore==True checks out C-tile from register pool + + if len(self.elementSumIdx) > 0: + for i in self.elementSumIdx: + self.kernelWriter.vgprPool.checkIn(i * self.cfg.numVgprPerValuC) + # print("checked in vgpr %u"%i) + self.elementSumIdx = [] + + def __del__(self): + if (self.sharedColDVgprs != None): + self.kernelWriter.vgprPool.checkIn(self.sharedColDVgprs) + if (self.sharedColCVgprs != self.sharedColDVgprs): + self.kernelWriter.vgprPool.checkIn(self.sharedColCVgprs) + if (self.sharedColBiasVgprs != None): + self.kernelWriter.vgprPool.checkIn(self.sharedColBiasVgprs) + self.checkInTempVgprC() diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py new file mode 100644 index 0000000000..9c0fd45395 --- /dev/null +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -0,0 +1,420 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +import os +import shutil +import sys +import time + +from copy import deepcopy + +from . import ClientExecutable +from . import SolutionLibrary +from . import LibraryIO +from . import Utils +from .BenchmarkStructs import BenchmarkProcess, checkParametersAreValid, constructForkPermutations +from .Contractions import ProblemType as ContractionsProblemType +from .ClientWriter import runClient, writeClientConfig, writeClientConfigIni +from .Common import globalParameters, HR, pushWorkingPath, popWorkingPath, print1, print2, \ + printExit, printWarning, ensurePath, startTime, validParameters +from .KernelWriterAssembly import KernelWriterAssembly +from .SolutionStructs import Solution, ProblemType, ProblemSizes +from .TensileCreateLibrary import copyStaticFiles, writeSolutionsAndKernels +from .CustomKernels import getCustomKernelConfig + + +def generateForkedSolutions(problemType, constantParams, forkPermutations): + """Creates a list with a Solution object for each parameter combination in forkPermutations""" + print1("# Enumerating Solutions") + + solutions = [] + solutionSet = set() + for perm in forkPermutations: + solution = {"ProblemType": deepcopy(problemType.state)} + solution.update(constantParams) + solution.update(perm) + + # TODO check if solution matches problem size for exact tile kernels + solutionObject = Solution(solution) + if solutionObject["Valid"]: + if solutionObject not in solutionSet: + solutionSet.add(solutionObject) + solutions.append(solutionObject) + elif globalParameters["PrintSolutionRejectionReason"]: + print1("rejecting solution " + str(solutionObject)) + + return solutions + + +def getCustomKernelSolutionObj(kernelName, directory=globalParameters["CustomKernelDirectory"]): + """Creates the Solution object for a custom kernel""" + kernelConfig = getCustomKernelConfig(kernelName, directory) + for k, v in kernelConfig.items(): + if k != "ProblemType": + checkParametersAreValid((k, [v]), validParameters) + kernelConfig["KernelLanguage"] = "Assembly" + kernelConfig["CustomKernelName"] = kernelName + + return Solution(kernelConfig) + + +def generateCustomKernelSolutions(problemType, customKernels, failOnMismatch): + """Creates a list with a Solution object for each name in customKernel""" + solutions = [] + for kernelName in customKernels: + print1("# Processing custom kernel {}".format(kernelName)) + solution = getCustomKernelSolutionObj(kernelName) + if solution["ProblemType"] != problemType: + # Raise error if this kernel was specifically requested and problem type doesn't match + if failOnMismatch: + benchmarkSet = set([(k,tuple(v)) if type(v) is list else (k,v) \ + for k,v in problemType.items()]) + customSet = set([(k,tuple(v)) if type(v) is list else (k,v) \ + for k,v in solution["ProblemType"].items()]) + + msg = "The problem type in the config file does not match " \ + "that of the custom kernel, {}.".format(kernelName) \ + + "\nDiffering parameters:\n" \ + + "\tConfig values:\n\t" \ + + str(sorted(benchmarkSet - (customSet & benchmarkSet))) \ + + "\n\tCustom kernel values:\n\t" \ + + str(sorted(customSet - (customSet & benchmarkSet))) + printExit(msg) + else: + print1("# Rejected {}: Problem Type doesn't match".format(kernelName)) + else: + print1("# Added {} to solutions".format(kernelName)) + if solution["Valid"]: + solutions.append(solution) + elif globalParameters["PrintSolutionRejectionReason"]: + print1("rejecting solution " + str(solution)) + + return solutions + +def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ + activationArgs, stepName, solutionSummationSizes): + """Write all the files needed for a given benchmarking step""" + if not globalParameters["MergeFiles"]: + ensurePath(os.path.join(globalParameters["WorkingPath"], "Solutions")) + ensurePath(os.path.join(globalParameters["WorkingPath"], "Kernels")) + + copyStaticFiles() + + kernels = [] + kernelHelperOjbs = [] + kernelNames = set() + kernelHelperNames = set() + + # get unique kernels and kernel helpers + for solution in Utils.tqdm(solutions, "Finding unique solutions"): + solutionKernels = solution.getKernels() + for kernel in solutionKernels: + kName = Solution.getNameFull(kernel) + if kName not in kernelNames: + kernels.append(kernel) + kernelNames.add(kName) + + solutionHelperKernels = solution.getHelperKernelObjects() + for ko in solutionHelperKernels: + kname = ko.getKernelName() + if kname not in kernelHelperNames: + kernelHelperOjbs.append(ko) + kernelHelperNames.add(kname) + + kernelSerialNaming = Solution.getSerialNaming(kernels) + kernelMinNaming = Solution.getMinNaming(kernels) + kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming) + + # write solution, kernels and CMake + problemType = solutions[0]["ProblemType"] + codeObjectFiles = writeSolutionsAndKernels( \ + globalParameters["WorkingPath"], globalParameters["CxxCompiler"], \ + [problemType], solutions, kernels, kernelHelperOjbs, \ + kernelWriterAssembly, errorTolerant=True ) + # ^ this is where solutions is mutated + + newLibraryDir = ensurePath(os.path.join(globalParameters["WorkingPath"], 'library')) + newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary") + newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions) + newLibrary.applyNaming(kernelMinNaming) + LibraryIO.write(newLibraryFile, Utils.state(newLibrary), globalParameters["LibraryFormat"]) + + codeObjectFiles = [os.path.relpath(f, globalParameters["WorkingPath"]) \ + for f in codeObjectFiles] + + if "TileAwareSelection" in problemType and problemType["TileAwareSelection"]: + maxMacroTile0 = 0 + maxMacroTile1 = 0 + for solution in solutions: + macroTile0 = solution["MacroTile0"] + macroTile1 = solution["MacroTile1"] + if macroTile0 > maxMacroTile0: + maxMacroTile0 = macroTile0 + if macroTile1 > maxMacroTile1: + maxMacroTile1 = macroTile1 + idealM = 36 * maxMacroTile0 + idealN = 36 * maxMacroTile1 + idealSizes = [] + if problemType["Batched"]: + for idealK in solutionSummationSizes: + idealSize = {"Exact": [idealM, idealN, 1, idealK]} + idealSizes.append(idealSize) + else: + for idealK in solutionSummationSizes: + idealSize = {"Exact": [idealM, idealN, idealK]} + idealSizes.append(idealSize) + idealProblemSizes = ProblemSizes(problemType, idealSizes) + writeClientConfig(True, solutions, idealProblemSizes, activationArgs, stepName, stepBaseDir, \ + newLibrary, codeObjectFiles, True) + else: + writeClientConfig(True, solutions, problemSizes, activationArgs, stepName, stepBaseDir, \ + newLibrary, codeObjectFiles, False) + + if len(solutions) == 0: + printExit("write solutions and kernels results 0 valid soultion.") + + return codeObjectFiles + + +def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache): + """Run the benchmarking for a single entry in the BenchmarkProblems of a Tensile config""" + benchmarkTestFails = 0 + + print1("") + print1(HR) + print1("# Converting Config to BenchmarkProcess Object") + print1(HR) + print1("") + benchmarkProcess = BenchmarkProcess(problemTypeConfig, problemSizeGroupConfig) + + enableTileSelection = benchmarkProcess.problemType["TileAwareSelection"] + groupName = "{}_{:02d}".format(str(benchmarkProcess.problemType), problemSizeGroupIdx) + pushWorkingPath(groupName) + ensurePath(os.path.join(globalParameters["WorkingPath"], "Data")) + + totalBenchmarkSteps = len(benchmarkProcess) + resultsFileBaseFinal = None + + print1("# NumBenchmarkSteps: {}".format(totalBenchmarkSteps)) + print1("") + print1(HR) + print1("# Done Creating BenchmarkProcess Object") + print1(HR) + + for benchmarkStepIdx in range(0, totalBenchmarkSteps): + benchmarkStep = benchmarkProcess[benchmarkStepIdx] + stepName = str(benchmarkStep) + shortName = stepName + + print1("\n") + print1(HR) + currentTime = time.time() + elapsedTime = currentTime - startTime + print1("# Benchmark Step: {} - {} {:.3f}s".format(groupName, stepName, elapsedTime)) + print1("# Num Sizes: {}".format(benchmarkStep.problemSizes.totalProblemSizes)) + print1("# Activation steps: {}".format(benchmarkStep.activationArgs.totalProblemSizes)) + print1("# Fork Parameters:") + for k, v in benchmarkStep.forkParams.items(): + print1("# {}: {}".format(k, v)) + + pushWorkingPath(shortName) + stepBaseDir = globalParameters["WorkingPath"] + + # file paths + resultsFileBase = os.path.normpath(os.path.join( \ + globalParameters["WorkingPath"], "../Data", shortName)) + if benchmarkStep.isFinal(): + resultsFileBaseFinal = resultsFileBase + resultsFileName = resultsFileBase + ".csv" + solutionsFileName = resultsFileBase + ".yaml" + + # check if a solution cache exists and if it matches our solution parameters + cachePath = os.path.join(stepBaseDir, "cache.yaml") + pushWorkingPath("source") + + cacheValid = False + if useCache and os.path.isfile(cachePath): + c = LibraryIO.readYAML(cachePath) + if c["ConstantParams"] == benchmarkStep.constantParams and \ + c["ForkParams"] == benchmarkStep.forkParams and \ + c["ParamGroups"] == benchmarkStep.paramGroups and \ + c["CustomKernels"] == benchmarkStep.customKernels and \ + c["CustomKernelWildcard"] == benchmarkStep.customKernelWildcard: + cacheValid = True + codeObjectFiles = c["CodeObjectFiles"] + else: + printWarning("Cache data does not match config: redoing solution generation") + + if not cacheValid: + # enumerate benchmark permutations and create resulting solution objects + forkPermutations = constructForkPermutations(benchmarkStep.forkParams, \ + benchmarkStep.paramGroups) + maxPossibleSolutions = len(forkPermutations) + + regSolutions = generateForkedSolutions(benchmarkProcess.problemType, \ + benchmarkStep.constantParams, forkPermutations) + kcSolutions = generateCustomKernelSolutions(benchmarkProcess.problemType, \ + benchmarkStep.customKernels, not benchmarkStep.customKernelWildcard) + + maxPossibleSolutions += len(kcSolutions) + solutions = regSolutions + kcSolutions + + print1("# Actual Solutions: {} / {} after SolutionStructs\n" \ + .format(len(solutions), maxPossibleSolutions)) + + # handle no valid solutions + if len(solutions) == 0: + msg = "Your parameters resulted in 0 valid solutions." + if globalParameters["PrintSolutionRejectionReason"]: + msg += "\nExamine reject and backtrace messages above to see why" \ + "and where solutions were rejected." + else: + msg += "\nYou should re-run with \"PrintSolutionRejectionReason: True\"" \ + "to see why each parameter combination was rejected." + printExit(msg) + + if globalParameters["PrintLevel"] >= 1: + for solution in solutions: + print2("# ({}:{}) {}".format(0, 0, Solution.getNameFull(solution))) + print2(HR) + + # write benchmarkFiles + prevCount = len(solutions) + codeObjectFiles = writeBenchmarkFiles(stepBaseDir, solutions, \ + benchmarkStep.problemSizes, benchmarkStep.activationArgs, shortName, []) + # ^ this mutates solutions + + # write cache data + cacheData = { + "CodeObjectFiles": codeObjectFiles, + "ConstantParams": benchmarkStep.constantParams, + "ForkParams": benchmarkStep.forkParams, + "ParamGroups": benchmarkStep.paramGroups, + "CustomKernels": benchmarkStep.customKernels, + "CustomKernelWildcard": benchmarkStep.customKernelWildcard + } + LibraryIO.writeYAML(cachePath, cacheData) + + print1("# Actual Solutions: {} / {} after KernelWriter\n" \ + .format(len(solutions), prevCount )) + else: + solutions = None + print1("# Using cached solution data") + + ssProblemType = ProblemType(problemTypeConfig) + conProblemType = ContractionsProblemType.FromOriginalState(ssProblemType) + outFile = os.path.join(globalParameters["WorkingPath"], "ClientParameters.ini") + + writeClientConfigIni(benchmarkStep.problemSizes, benchmarkStep.activationArgs, conProblemType, + globalParameters["WorkingPath"], codeObjectFiles, resultsFileName, + outFile) + + # I think the size portion of this yaml could be removed, + # but for now it's needed, so we update it even in the cache case + LibraryIO.writeSolutions(solutionsFileName, benchmarkStep.problemSizes, benchmarkStep.activationArgs, solutions, cacheValid) + + popWorkingPath() # source + + # run benchmarking client + if not os.path.exists(resultsFileName) or globalParameters["ForceRedoBenchmarkProblems"]: + libraryLogicPath = None + forBenchmark = True + returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection) + + if returncode: + benchmarkTestFails += 1 + printWarning("BenchmarkProblems: Benchmark Process exited with code {}" \ + .format(returncode)) + else: + print1("# Already benchmarked; skipping.") + + # End Iteration + popWorkingPath() # stepName + currentTime = time.time() + elapsedTime = currentTime - startTime + print1("{}\n# {}\n# {}: End - {:.3f}s\n{}\n" \ + .format(HR, groupName, shortName, elapsedTime, HR)) + + popWorkingPath() # ProblemType + return (resultsFileBaseFinal, benchmarkTestFails) + + +def main(config, useCache): + """Entry point for the "BenchmarkProblems" section of a Tensile config yaml""" + ClientExecutable.getClientExecutable() + + dataPath = os.path.join(globalParameters["WorkingPath"], globalParameters["BenchmarkDataPath"]) + pushWorkingPath(globalParameters["BenchmarkProblemsPath"]) + ensurePath(dataPath) + + totalTestFails = 0 + for benchmarkProblemTypeConfig in config: + problemTypeConfig = benchmarkProblemTypeConfig[0] + if len(benchmarkProblemTypeConfig) < 2: + problemSizeGroupConfigs = [{}] + else: + problemSizeGroupConfigs = benchmarkProblemTypeConfig[1:] + + for idx, sizeGroupConfig in enumerate(problemSizeGroupConfigs): + print2("ProblemTypeConfig: {}".format(problemTypeConfig)) + problemTypeObj = ProblemType(problemTypeConfig) + globalParameters["EnableHalf"] = problemTypeObj["DataType"].isHalf() + + # using a suffix to check the csv version (for later addFromCSV()) + csvSuffix = "_CSVWinner" if globalParameters["CSVExportWinner"] else "" + # results files will be named + newResultsFileName = os.path.join(dataPath, "{}_{:02d}{}.csv" \ + .format(str(problemTypeObj), idx, csvSuffix) ) + newSolutionsFileName = os.path.join(dataPath, "{}_{:02d}{}.yaml" \ + .format(str(problemTypeObj), idx, csvSuffix) ) + newGranularityFileName = os.path.join(dataPath, "{}_{:02d}{}.gsp" \ + .format(str(problemTypeObj), idx, csvSuffix) ) + + # skip if possible + if globalParameters["ForceRedoBenchmarkProblems"] \ + or not os.path.exists(newResultsFileName): + + # benchmark problem size group + (resultsFileBaseFinal, benchmarkErrors) = \ + benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache) + totalTestFails += benchmarkErrors + + print("clientExit={} {} for {}" \ + .format(totalTestFails, "(ERROR)" if totalTestFails else "(PASS)", \ + globalParameters["ConfigPath"]) ) + + # copy data + resultsFileBase = resultsFileBaseFinal + resultsFileName = resultsFileBase + ".csv" + solutionsFileName = resultsFileBase + ".yaml" + granularityFileName = resultsFileBase + "_Granularity.csv" + shutil.copy(resultsFileName, newResultsFileName) + shutil.copy(solutionsFileName, newSolutionsFileName) + if os.path.isfile(granularityFileName): + shutil.copy(granularityFileName, newGranularityFileName) + else: + print1("# {}_{:02d} already benchmarked; skipping." \ + .format(str(problemTypeObj), idx) ) + + popWorkingPath() + + if globalParameters["ExitOnFails"] and totalTestFails: + sys.exit(1) diff --git a/tensilelite/Tensile/BenchmarkSplitter.py b/tensilelite/Tensile/BenchmarkSplitter.py new file mode 100644 index 0000000000..35b035f634 --- /dev/null +++ b/tensilelite/Tensile/BenchmarkSplitter.py @@ -0,0 +1,170 @@ +import os +import copy +import yaml +import math + +class BenchmarkSplitter(object): + + """ + Benchmark splitter class + Loads in a benchmark yaml file and splits + it into several smaller benchmarks limited + by a number of problem sizes entries. + """ + + @staticmethod + def __readConfigFile(benchmarkConfigFile): + with open(benchmarkConfigFile) as f: + data = yaml.safe_load(f) + return data + + # data: a loaded .yaml file + # returns: a list of yaml files that + # are differentiated by the benchmark problem + @staticmethod + def __splitByProblem(data): + rv = [] + problemKey = "BenchmarkProblems" + for i in range(len(data[problemKey])): + result = {} + for k in data.keys(): + if k == problemKey: + result[k] = [copy.deepcopy(data[k][i])] + else: + result[k] = copy.deepcopy(data[k]) + rv.append(result) + return rv + + # data: a loaded .yaml file, containing one benchmark problem section + # returns: a list of yaml files that are differentiated by the + # benchmark groupings + @staticmethod + def __splitByBenchmarkGroup(data): + rv = [] + problemKey = "BenchmarkProblems" + + assert len(data[problemKey]) == 1, "Config file must have one BenchmarkProblems group" + + benchmarkProblems = data[problemKey][0] + + # Find the index of the problem type group + problemIdx = -1 + for i in range(len(benchmarkProblems)): + if("OperationType" in benchmarkProblems[i].keys()): + problemIdx = i + break + + assert problemIdx != -1, "Could not find problem type group" + + # Split files on the benchmark group + for i in range(len(benchmarkProblems)): + if i != problemIdx: + result = {} + for k in data.keys(): + if k == problemKey: + # Take only the problem group and one benchmarkgroup + result[k] = [[copy.deepcopy(benchmarkProblems[problemIdx]), \ + copy.deepcopy(benchmarkProblems[i])]] + else: + # copy other sections verbatim + result[k] = copy.deepcopy(data[k]) + rv.append(result) + return rv + + # data: a loaded .yaml file, containing one benchmark problem section + # and one benchmark group section + # returns: a list of yaml files that are differentiated by the + # benchmark sizes + @staticmethod + def __splitByBenchmarkSizes(data, numSizes=1): + rv = [] + problemKey = "BenchmarkProblems" + + assert len(data[problemKey]) == 1, "Config file must have one BenchmarkProblems group" + + benchmarkProblems = data[problemKey][0] + + # Find the index of the problem type group + # and the benchmark group + problemIdx = -1 + benchmarkIdx = -1 + for i in range(len(benchmarkProblems)): + groupKeys = benchmarkProblems[i].keys() + if("OperationType" in groupKeys): + problemIdx = i + elif("BenchmarkFinalParameters" in groupKeys): + benchmarkIdx = i + + assert len(benchmarkProblems) == 2 \ + and problemIdx is not -1 \ + and benchmarkIdx is not -1, \ + "Config file must have one ProblemType group and one Benchmark group" + + # Grab the problem sizes from the Benchmark group + benchmarkGroup = benchmarkProblems[benchmarkIdx] + assert "ProblemSizes" in benchmarkGroup["BenchmarkFinalParameters"][0] \ + and len(benchmarkGroup["BenchmarkFinalParameters"][0]["ProblemSizes"]), \ + "Benchmark group must have non-empty ProblemSizes" + + problemSizesGroup = benchmarkGroup["BenchmarkFinalParameters"][0]["ProblemSizes"] + problemSizesCount = len(problemSizesGroup) + + numFiles = math.ceil(problemSizesCount / numSizes) + + # Split files on the benchmark sizes + for i in range(numFiles): + result = {} + for k in data.keys(): + if k == problemKey: + + # Create a new benchmark group that has all the old keys but split the sizes. + newBenchmarkGroup = {} + for bk in benchmarkGroup.keys(): + if bk == "BenchmarkFinalParameters": + newBenchmarkGroup[bk] = [ {"ProblemSizes": [] } ] + for j in range(i*numSizes, min((i+1)*numSizes, problemSizesCount)): + newBenchmarkGroup[bk][0]["ProblemSizes"].append(copy.deepcopy(problemSizesGroup[j])) + else: + newBenchmarkGroup[bk] = copy.deepcopy(benchmarkGroup[bk]) + + result[k] = [[copy.deepcopy(benchmarkProblems[problemIdx]), copy.deepcopy(newBenchmarkGroup)]] + else: + result[k] = copy.deepcopy(data[k]) + rv.append(result) + return rv + + # filePath: Name of the file (can be a path) + # suffix: Add a suffix of _## to fileName by default + # formatting: How the stuffix string is to be treated. + # If more indices are needed, you can increase them on the formatting. + @staticmethod + def __appendFileNameSuffix(filePath, suffix, separator="_", formatting="{:02}"): + root, ext = os.path.splitext(filePath) + suffixString = (separator + formatting).format(suffix) + return root + suffixString + ext + + @staticmethod + def splitBenchmarkBySizes(configFile, outputDir, numSizes=1, baseFileName="", separator="_", suffixFormat="{:02}"): + + # Use the configFile as base name if none provided + if baseFileName == "": + baseFileName = os.path.basename(configFile) + + # If we want to split by sizes, then we must first split files + # into separate problems, then split those files into separate + # benchmark groups, then split those files into sizes + data = BenchmarkSplitter.__readConfigFile(configFile) + benchmarksByProblem = BenchmarkSplitter.__splitByProblem(data) + benchmarksByGroup = [] + benchmarksBySize = [] + for problem in benchmarksByProblem: + benchmarksByGroup += BenchmarkSplitter.__splitByBenchmarkGroup(problem) + for group in benchmarksByGroup: + benchmarksBySize += BenchmarkSplitter.__splitByBenchmarkSizes(group, numSizes) + + # outputDir/basefileName_XX.ext + outputFileBase = os.path.join(outputDir, baseFileName) + for i in range(len(benchmarksBySize)): + outFileName = BenchmarkSplitter.__appendFileNameSuffix(outputFileBase, i, separator, suffixFormat) + with open(outFileName, "w") as f: + yaml.safe_dump(benchmarksBySize[i], f) diff --git a/tensilelite/Tensile/BenchmarkStructs.py b/tensilelite/Tensile/BenchmarkStructs.py new file mode 100644 index 0000000000..e799022b99 --- /dev/null +++ b/tensilelite/Tensile/BenchmarkStructs.py @@ -0,0 +1,299 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from copy import deepcopy +import itertools +from .Common import print1, print2, hasParam, printExit, \ + defaultBenchmarkCommonParameters, validParameters, globalParameters, \ + defaultBatchedBenchmarkFinalProblemSizes, defaultBenchmarkFinalProblemSizes +from .CustomKernels import getAllCustomKernelNames +from .SolutionStructs import ProblemType, ProblemSizes, ActivationArgs + + +def getDefaultsForMissingParameters(paramList, defaultParams): + """Returns all parameters (with values) in defaultParams not present in paramList""" + benchmarkParams = {} + for paramDict in defaultParams: + for name, value in paramDict.items(): + if not hasParam(name, paramList) \ + or name == "ProblemSizes": + benchmarkParams[name] = value + return benchmarkParams + + +def checkParametersAreValid(param, validParams): + """Ensures paramaters in params exist and have valid values as specified by validParames""" + (name, values) = param + if name == "ProblemSizes": + return + + if name not in validParams: + printExit("Invalid parameter name: {}\nValid parameters are {}." \ + .format(name, sorted(validParameters.keys()))) + + for value in values: + if validParams[name] != -1 and value not in validParams[name]: + msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." + msgExt = " (only first 32 combos printed)\nRefer to Common.py for more info" \ + if len(validParams[name])>32 else "" + printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) + + +def separateParameters(paramSetList): + """Separates paramSetList into parameters with single and multiple values""" + singleVaules = {} + multiValues = {} + for name, values in paramSetList.items(): + if values == None: + printExit("You must specify value(s) for parameter \"{}\"".format(name)) + if len(values) == 1 and name != "ProblemSizes": + singleVaules[name] = values[0] + elif len(values) > 1 and name != "ProblemSizes": + multiValues[name] = values + + return singleVaules, multiValues + + +def checkCDBufferAndStrides(problemType, problemSizes, isCEqualD): + """Ensures ldd == ldc when CEqualD""" + if isCEqualD and problemType["OperationType"] == "GEMM": + for problem in problemSizes.problems: + ldd = problem.sizes[problemType["IndexAssignmentsLD"][0]] + ldc = problem.sizes[problemType["IndexAssignmentsLD"][1]] + if ldd != ldc: + printExit("LDD({}) != LDC({}) causes unpredictable result when CEqualD(True)" \ + .format(ldd, ldc)) + + +class BenchmarkProcess: + """Representation of benchmarking parameters and resulting steps""" + + def __init__(self, problemTypeConfig, problemSizeGroupConfig): + """Create from the two sections of a config for a BenchmarkProblem""" + self.problemType = ProblemType(problemTypeConfig) + self.isBatched = "Batched" in problemTypeConfig and problemTypeConfig["Batched"] + print2("# BenchmarkProcess beginning {}".format(self.problemType)) + + # fill parameter values from config + self.singleValueParams = {} + self.multiValueParams = {} + self.customKernels = [] + self.sizes = None + self.getConfigParameters(self.isBatched, problemSizeGroupConfig) + + # convert parameter lists to steps + # previously, multiple benchmark steps were possible + # currently only 1 benchmark step is possible; more may be added back later + self.benchmarkSteps = [] + self.benchmarkStepIdx = 0 + self.convertParametersToSteps() + + def getConfigParameters(self, isbatched, config): + """Parse and validate benchmarking parameters in config""" + print2("") + print2("####################################################################") + print1("# Filling in Parameters With Defaults") + print2("####################################################################") + print2("") + + # check for no longer supported legacy benchmark steps + badParams = ["InitialSolutionParameters", "BenchmarkForkParameters", \ + "JoinParameters", "BenchmarkJoinParameters"] + badsInConfig = [] + + for p in badParams: + if config.get(p) is not None: + badsInConfig.append(p) + + if len(badsInConfig) == 1: + printExit("Benchmark step {} is no longer supported".format("'" + badsInConfig[0] + + "'")) + elif len(badsInConfig) > 1: + printExit("Benchmark steps {} are no longer supported".format(badsInConfig)) + + # get supported configurations + # value in config file may be "None", which we should ignore + def getNonNoneFromConfig(key, default): + if config.get(key) is not None: + return config[key] + else: + return default + + # converts list of dicts into a flat dict + benchmarkCommonParams = dict(itertools.chain(*[x.items() \ + for x in getNonNoneFromConfig("BenchmarkCommonParameters", [])])) + forkParams = dict(itertools.chain(*[x.items() \ + for x in getNonNoneFromConfig("ForkParameters", [])])) + self.paramGroups = forkParams.pop("Groups") if "Groups" in forkParams else [] + self.customKernels = getNonNoneFromConfig("CustomKernels", []) + + activationConf = "" + if "BenchmarkFinalParameters" in config: + sizes = config["BenchmarkFinalParameters"][0]["ProblemSizes"] + if len(config["BenchmarkFinalParameters"]) == 2: + activationConf = config["BenchmarkFinalParameters"][1]["ActivationArgs"] + else: + sizes = defaultBatchedBenchmarkFinalProblemSizes if isbatched \ + else defaultBenchmarkFinalProblemSizes + + self.problemSizes = ProblemSizes(self.problemType, sizes) + checkCDBufferAndStrides(self.problemType, self.problemSizes, globalParameters["CEqualD"]) + + self.activationArgs = ActivationArgs(self.problemType, activationConf) + + # validate parameter values + configParams = {**benchmarkCommonParams, **forkParams} + for param in configParams.items(): + checkParametersAreValid(param, validParameters) + + # TODO other checks on groups (same params for each entry? no dups between groups?) + for list in self.paramGroups: + for group in list: + for k, v in group.items(): + checkParametersAreValid((k, [v]), validParameters) + + params = dict(itertools.chain(*[x.items() for x in defaultBenchmarkCommonParameters])) + params.update(configParams) + self.singleValueParams, self.multiValueParams = separateParameters(params) + + # print summary of parameter values + print2("Single Value Parameters:") + for k, v in self.singleValueParams.items(): + print2(" {}: {}".format(k, v)) + + print2("Multi-Value Parameters:") + for k, v in self.multiValueParams.items(): + print2(" {}: {}".format(k, v)) + + if len(self.paramGroups) > 0: + print2("{} Parameter Group(s):".format(len(self.paramGroups))) + for i, group in enumerate(self.paramGroups): + print2(" {} entries is group {}".format(len(group), i + 1)) + + def convertParametersToSteps(self): + """Create benchmark steps based on parsed parameters""" + print2("") + print2("####################################################################") + print1("# Convert Parameters to Benchmark Step(s)") + print2("####################################################################") + print2("") + + # currently only a single step is supported + print2("") + print2("####################################################################") + print1("# Benchmark Final") + benchmarkStep = BenchmarkStep( \ + self.multiValueParams, \ + self.singleValueParams, \ + self.paramGroups, \ + self.customKernels, \ + self.problemSizes, \ + self.activationArgs, \ + self.benchmarkStepIdx) + self.benchmarkSteps.append(benchmarkStep) + self.benchmarkStepIdx += 1 + + def __len__(self): + return len(self.benchmarkSteps) + + def __getitem__(self, key): + return self.benchmarkSteps[key] + + def __str__(self): + string = "BenchmarkProcess:\n" + for step in self.benchmarkSteps: + string += str(step) + return string + + def __repr__(self): + return self.__str__() + + +def constructForkPermutations(forkParams, paramGroups): + """Constructs cartesian product of parameter values in forkParams and paramGroups""" + + myParams = {} + myParams.update(forkParams) + + totalPermutations = 1 + for _, values in forkParams.items(): + totalPermutations *= len(values) + + # add groups to parameters to fork on + for i, group in enumerate(paramGroups): + myParams["_group" + str(i)] = group + totalPermutations *= len(group) + + forkPermutations = [] + for i in range(0, totalPermutations): + permutation = {} + pIdx = i + for name, v in myParams.items(): + values = deepcopy(v) + valueIdx = pIdx % len(v) + + # groups have multiple parameters to update + if "_group" in name: + entry = values[valueIdx] + for n2, v2 in entry.items(): + permutation[n2] = v2 + else: + permutation[name] = values[valueIdx] + + pIdx //= len(values) + forkPermutations.append(permutation) + + return forkPermutations + + +class BenchmarkStep: + """A single benchmark step which consists of constant and fork parameters and a set of sizes""" + + def __init__(self, forkParams, constantParams, paramGroups, customKernels, problemSizes, activationArgs, idx): + """Basic constructor storing each argument""" + self.forkParams = forkParams + self.constantParams = constantParams + self.paramGroups = paramGroups + self.customKernels = customKernels + self.problemSizes = problemSizes + self.activationArgs = activationArgs + self.stepIdx = idx + + self.customKernelWildcard = False + if self.customKernels == ["*"]: + self.customKernels = getAllCustomKernelNames() + self.customKernelWildcard = True + + print2("# Creating BenchmarkStep: {} fork params and {} sizes" \ + .format( len(forkParams), problemSizes.totalProblemSizes)) + + def isFinal(self): + """Legacy. Currently always returns true since only one benchmark step is possible""" + return True + + def __str__(self): + string = "{:02d}".format(self.stepIdx) + if self.isFinal(): + string += "_Final" + return string + + def __repr__(self): + return self.__str__() diff --git a/tensilelite/Tensile/ClientExecutable.py b/tensilelite/Tensile/ClientExecutable.py new file mode 100644 index 0000000000..ca3809f2f8 --- /dev/null +++ b/tensilelite/Tensile/ClientExecutable.py @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +import itertools +import os +import subprocess + +from . import Common +from .Common import globalParameters + +class CMakeEnvironment: + def __init__(self, sourceDir, buildDir, **options): + self.sourceDir = sourceDir + self.buildDir = buildDir + self.options = options + + def generate(self): + + args = ['cmake'] + args += itertools.chain.from_iterable([ ['-D', '{}={}'.format(key, value)] for key,value in self.options.items()]) + args += [self.sourceDir] + + Common.print2(' '.join(args)) + with Common.ClientExecutionLock(): + subprocess.check_call(args, cwd=Common.ensurePath(self.buildDir)) + + def build(self): + args = ['make', '-j'] + Common.print2(' '.join(args)) + with Common.ClientExecutionLock(): + subprocess.check_call(args, cwd=self.buildDir) + + def builtPath(self, path, *paths): + return os.path.join(self.buildDir, path, *paths) + +def clientExecutableEnvironment(builddir=None): + sourcedir = globalParameters["SourcePath"] + if builddir is None: + builddir = os.path.join(globalParameters["OutputPath"], globalParameters["ClientBuildPath"]) + builddir = Common.ensurePath(builddir) + + options = {'CMAKE_BUILD_TYPE': globalParameters["CMakeBuildType"], + 'TENSILE_USE_MSGPACK': 'ON', + 'TENSILE_USE_LLVM': 'ON', + 'Tensile_LIBRARY_FORMAT': globalParameters["LibraryFormat"], + 'CMAKE_CXX_COMPILER': os.path.join(globalParameters["ROCmBinPath"], globalParameters['CxxCompiler'])} + + return CMakeEnvironment(sourcedir, builddir, **options) + + +buildEnv = None + +def getClientExecutable(builddir=None): + if "PrebuiltClient" in globalParameters: + return globalParameters["PrebuiltClient"] + + global buildEnv + + if buildEnv is None: + buildEnv = clientExecutableEnvironment(builddir) + buildEnv.generate() + buildEnv.build() + + return buildEnv.builtPath("client/tensile_client") + diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py new file mode 100644 index 0000000000..caa0965b8d --- /dev/null +++ b/tensilelite/Tensile/ClientWriter.py @@ -0,0 +1,1574 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from . import ClientExecutable +from . import LibraryIO +from .TensileInstructions import getGfxName +from .Common import globalParameters, pushWorkingPath, popWorkingPath, print1, printExit, CHeader, printWarning, listToInitializer, ClientExecutionLock +from .SolutionStructs import ProblemType, ProblemSizesMock, ActivationArgs +from .TensileCreateLibrary import copyStaticFiles + +import os +import subprocess +import shlex +import shutil +from enum import Enum + +from .Contractions import FreeIndex +from .Contractions import ProblemType as ContractionsProblemType + +class DataInitName(Enum): + Zero = 0 + One = 1 + Two = 2 + Random = 3 + NaN = 4 + Inf = 5 + BadInput = 6 + BadOutput = 7 + SerialIdx = 8 + SerialDim0 = 9 + SerialDim1 = 10 + Identity = 11 + TrigSin = 12 + TrigCos = 13 + TrigAbsSin = 14 + TrigAbsCos = 15 + RandomNarrow = 16 + NegOne = 17 + Max = 18 + DenormMin = 19 + DenormMax = 20 + RandomNegPosLimited = 21 + +class ClientLogLevel(Enum): + Error = 0 + Terse = 1 + Verbose = 2 + Debug = 3 + + +################################################################################ +# Main +################################################################################ +def main( config ): + libraryLogicPath = os.path.join(globalParameters["WorkingPath"], \ + globalParameters["LibraryLogicPath"]) + stepBaseDir = pushWorkingPath(globalParameters["LibraryClientPath"]) + + pushWorkingPath("source") + copyStaticFiles() + + ############################################################################## + # Read Logic Files + ############################################################################## + logicFiles = [os.path.join(libraryLogicPath, f) for f \ + in os.listdir(libraryLogicPath) \ + if (os.path.isfile(os.path.join(libraryLogicPath, f)) \ + and os.path.splitext(f)[1]==".yaml")] + print1("LogicFiles: %s" % logicFiles) + functions = [] + functionNames = [] + enableHalf = False + + createLibraryScript = getBuildClientLibraryScript(stepBaseDir, libraryLogicPath) + subprocess.run(shlex.split(createLibraryScript), cwd=stepBaseDir) + coList = [] + yamlList = [] + with open(os.path.join(stepBaseDir,"library","TensileManifest.txt"), "r") as f: + lines = f.read().split("\n") + coList = [line for line in lines if "co" in line] + yamlList = [line for line in lines if "yaml" in line] + + clientParametersPaths = [] + for logicFileName in logicFiles: + (scheduleName, deviceNames, problemType, solutionsForType, \ + indexOrder, exactLogic, rangeLogic, newLibrary, architectureName) \ + = LibraryIO.parseLibraryLogicFile(logicFileName) + if problemType["DataType"].isHalf(): + enableHalf = True + functions.append((scheduleName, problemType)) + functionNames.append("tensile_%s" % (problemType)) + problemSizes = ProblemSizesMock(exactLogic) + activationArgs = ActivationArgs(problemType, [[{'Enum': 'relu'}]]) if problemType["ActivationType"] == 'all' else "" + clientParametersPaths.append(writeClientConfig( + forBenchmark=False, + solutions=None, + problemSizes=problemSizes, + activationArgs=activationArgs, + stepName=str(ProblemType(problemType)), + stepBaseDir=globalParameters["WorkingPath"], + newLibrary=newLibrary, + configBase="ClientParameters_%s"%str(ProblemType(problemType)), + codeObjectFiles=coList, + tileAwareSelection=False, + libraryFile=yamlList[0])) + globalParameters["EnableHalf"] = enableHalf + + ############################################################################## + # Write Generated Header + ############################################################################## + forBenchmark = False + problemSizes = None + popWorkingPath() # source + + ############################################################################## + # Run Build Script + ############################################################################## + # if redo=true, clobber the build directory + if globalParameters["ForceRedoLibraryClient"]: + shutil.rmtree(os.path.join(globalParameters["WorkingPath"], "build"), \ + ignore_errors=True) + + forBenchmark = False + enableTileSelection = False + returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, clientParametersPaths) + + popWorkingPath() # LibraryClient + + return returncode + +################################################################################ +# Write Run Script +################################################################################ +def runNewClient(scriptPath, clientParametersPath, clientBuildDir=None): + + clientExe = ClientExecutable.getClientExecutable(clientBuildDir) + iniFile = "--config-file={}".format(clientParametersPath) + args = [clientExe, iniFile] + + try: + subprocess.run(args, check=True) + except (subprocess.CalledProcessError, OSError) as e: + printWarning("ClientWriter Benchmark Process exited with error: {}".format(e)) + + +def runClient(libraryLogicPath, forBenchmark, enableTileSelection, configPaths=None): + # write runScript + pushWorkingPath("build") + path = globalParameters["WorkingPath"] + + runScriptName = writeRunScript(path, forBenchmark, enableTileSelection, configPaths) + with ClientExecutionLock(): + process = subprocess.Popen(runScriptName, cwd=path) + process.communicate() + + if process.returncode: + printWarning("ClientWriter Benchmark Process exited with code %u" % process.returncode) + popWorkingPath() # build + + return process.returncode + +def getBuildClientLibraryScript(buildPath, libraryLogicPath): + import io + runScriptFile = io.StringIO() + + callCreateLibraryCmd = globalParameters["ScriptPath"] + "/bin/TensileCreateLibrary" + + + if globalParameters["MergeFiles"]: + callCreateLibraryCmd += " --merge-files" + else: + callCreateLibraryCmd += " --no-merge-files" + + if globalParameters["ShortNames"]: + callCreateLibraryCmd += " --short-file-names" + else: + callCreateLibraryCmd += " --no-short-file-names" + + if globalParameters["LibraryPrintDebug"]: + callCreateLibraryCmd += " --library-print-debug" + else: + callCreateLibraryCmd += " --no-library-print-debug" + + if globalParameters["GenerateManifestAndExit"]: + callCreateLibraryCmd += " --generate-manifest-and-exit" + + callCreateLibraryCmd += " --architecture=" + globalParameters["Architecture"] + callCreateLibraryCmd += " --code-object-version=" + globalParameters["CodeObjectVersion"] + callCreateLibraryCmd += " --cxx-compiler=" + globalParameters["CxxCompiler"] + callCreateLibraryCmd += " --library-format=" + globalParameters["LibraryFormat"] + + callCreateLibraryCmd += " %s" % libraryLogicPath + callCreateLibraryCmd += " %s" % buildPath #" ../source" + callCreateLibraryCmd += " %s\n" % globalParameters["RuntimeLanguage"] + + runScriptFile.write(callCreateLibraryCmd) + + return runScriptFile.getvalue() + +def writeBuildClientLibraryScript(path, libraryLogicPath): + filename = os.path.join(path, \ + "build.%s" % ("bat" if os.name == "nt" else "sh") ) + with open(filename, "w") as file: + file.write("#!/bin/bash\n\n") + file.write("set -ex\n") + file.write(getBuildClientLibraryScript(path, libraryLogicPath)) + + if os.name != "nt": + os.chmod(filename, 0o777) + return filename + +def writeRunScript(path, forBenchmark, enableTileSelection, configPaths=None): + if configPaths is None: + configPaths = [] + configPaths.append(os.path.join(globalParameters["WorkingPath"], "../source/ClientParameters.ini")) + if enableTileSelection is True: + configPaths.append(os.path.join(globalParameters["WorkingPath"], "../source/ClientParameters_Granularity.ini")) + + # create run.bat or run.sh which builds and runs + runScriptName = os.path.join(path, \ + "run.%s" % ("bat" if os.name == "nt" else "sh") ) + runScriptFile = open(runScriptName, "w") + if os.name != "nt": + runScriptFile.write("#!/bin/bash\n\n") + + runScriptFile.write("set -ex\n") + + + if forBenchmark: + if os.name == "nt": + runScriptFile.write(os.path.join(globalParameters["CMakeBuildType"], \ + "client.exe") ) + else: + if globalParameters["PinClocks"] and globalParameters["ROCmSMIPath"]: + runScriptFile.write("%s -d 0 --setfan 255 --setsclk 7\n" % globalParameters["ROCmSMIPath"]) + runScriptFile.write("sleep 1\n") + runScriptFile.write("%s -d 0 -a\n" % globalParameters["ROCmSMIPath"]) + + runScriptFile.write("set +e\n") + + + if globalParameters["DataInitTypeA"] == -1 : + globalParameters["DataInitTypeA"] = globalParameters["DataInitTypeAB"] + if globalParameters["DataInitTypeB"] == -1 : + globalParameters["DataInitTypeB"] = globalParameters["DataInitTypeAB"] + + runScriptFile.write("ERR1=0\n") + + clientExe = ClientExecutable.getClientExecutable() + for configFile in configPaths: + runScriptFile.write("{} --config-file {} {}\n".format(clientExe, configFile, globalParameters["ClientArgs"])) + runScriptFile.write("ERR2=$?\n\n") + + runScriptFile.write(""" +ERR=0 +if [[ $ERR1 -ne 0 ]] +then + echo one + ERR=$ERR1 +fi +if [[ $ERR2 -ne 0 ]] +then + echo two + ERR=$ERR2 +fi +""") + + if os.name != "nt": + if globalParameters["PinClocks"] and globalParameters["ROCmSMIPath"]: + runScriptFile.write("%s -d 0 --resetclocks\n" % globalParameters["ROCmSMIPath"]) + runScriptFile.write("%s -d 0 --setfan 50\n" % globalParameters["ROCmSMIPath"]) + else: + for configFile in configPaths: + runScriptFile.write("{} --config-file {} {} --best-solution 1\n".format(ClientExecutable.getClientExecutable(), configFile, globalParameters["ClientArgs"])) + if os.name != "nt": + runScriptFile.write("exit $ERR\n") + runScriptFile.close() + if os.name != "nt": + os.chmod(runScriptName, 0o777) + return runScriptName + + +def toCppBool(yamlBool): + return "true" if yamlBool else "false" + +def getMaxSolutionSizes(solutions, solutionSummationSizes): + + maxK = max(solutionSummationSizes) + maxMT0 = 0 + maxMT1 = 0 + for solution in solutions: + + wg = solution["WorkGroup"] + tt = solution["ThreadTile"] + mt0 = wg[0] * tt[0] + mt1 = wg[1] * tt[1] + + if (mt0 > maxMT0): + maxMT0 = mt0 + + if (mt1 > maxMT1): + maxMT1 = mt1 + + return [maxMT0, maxMT1, maxK] + +def checkConstStride(constStrideMap, keyIdx): + finalVal = None + for (mapIdx, val) in constStrideMap: + if keyIdx == mapIdx: + finalVal = val + #print ("idx=", keyIdx, "=", finalVal) + return finalVal + + +def problemSizeParams(problemType, problem): + + numIndices = len(problemType.indices) + rv = [] + + if problem.stridesA: + astrides = list(problem.stridesA) + else: + astrides = [-1] * problemType.aDims + for sc in problemType.setConstStrideA: + index = problemType.indices[sc[0]] + if type(index) == FreeIndex: + assert(index.isA) + astrides[index.i] = sc[1] + else: + astrides[index.a] = sc[1] + + if problem.stridesB: + bstrides = list(problem.stridesB) + else: + bstrides = [-1] * problemType.bDims + for sc in problemType.setConstStrideB: + index = problemType.indices[sc[0]] + if type(index) == FreeIndex: + assert(not index.isA) + bstrides[index.i] = sc[1] + else: + bstrides[index.b] = sc[1] + + if problem.stridesC: + cstrides = list(problem.stridesC) + else: + cstrides = [-1] * problemType.cDims + + if problem.stridesD: + dstrides = list(problem.stridesD) + else: + dstrides = [-1] * problemType.dDims + + if len(problem.sizes) == numIndices: + None + elif len(problem.sizes) == numIndices + 4: + # FIXME-problem, this is Exact format with strides tacked onto sizes as 4 extra pams + # should just set problem.stride* appropriately when reading the Yaml and not deal with extra fields here + if astrides[1] == -1: + astrides[1] = problem.sizes[numIndices+2] + elif astrides[1] != problem.sizes[numIndices+2]: + raise RuntimeError("problem-specified lda(%u) conflicts with setConstStrideA(%u)" % \ + (astrides[1], problem.sizes[numIndices+2])) + + if bstrides[1] == -1: + bstrides[1] = problem.sizes[numIndices+3] + elif bstrides[1] != problem.sizes[numIndices+3]: + raise RuntimeError("problem-specified ldb(%u) conflicts with setConstStrideB(%u)" % \ + (bstrides[1], problem.sizes[numIndices+3])) + + if cstrides[1] == -1: + cstrides[1] = problem.sizes[numIndices+1] + + if dstrides[1] == -1: + dstrides[1] = problem.sizes[numIndices+0] + + else: + raise RuntimeError( + "Invalid number of problem type indices: {0} - Indices: {1}, problemSize: {2}".format(len(problem.sizes), numIndices, + ', '.join(map(str, problem.sizes)))) + + problemSizeArg = ('problem-size', ','.join(map(str, problem.sizes[:numIndices]))) + rv.insert(0, problemSizeArg) + + rv.append(('a-strides', ",".join(map(str, astrides)))) + rv.append(('b-strides', ",".join(map(str, bstrides)))) + if cstrides: + rv.append(('c-strides', ",".join(map(str, cstrides)))) + if dstrides: + rv.append(('d-strides', ",".join(map(str, dstrides)))) + + return rv + +def dataInitParams(problemType): + initA = globalParameters['DataInitTypeA'] + initB = globalParameters['DataInitTypeB'] + initC = globalParameters['DataInitTypeC'] + initD = globalParameters['DataInitTypeD'] + initAlpha = globalParameters['DataInitTypeAlpha'] + initBeta = globalParameters['DataInitTypeBeta'] + initBias = globalParameters['DataInitTypeBias'] + + if not problemType.useBeta: + initBeta = 0 + + if initA == -1: initA = globalParameters['DataInitTypeAB'] + if initB == -1: initB = globalParameters['DataInitTypeAB'] + + return [('init-a', DataInitName(initA).name), + ('init-b', DataInitName(initB).name), + ('init-c', DataInitName(initC).name), + ('init-d', DataInitName(initD).name), + ('init-alpha', DataInitName(initAlpha).name), + ('init-beta', DataInitName(initBeta).name), + ('init-bias', DataInitName(initBias).name)] + +def boundsCheckName(mode): + if mode == 0: return 'Disable' + if mode == 1: return 'NaN' + if mode == 2: return 'GuardPageFront' + if mode == 3: return 'GuardPageBack' + if mode == 4: return 'GuardPageAll' + + +def writeClientConfigIni(problemSizes, activationArgs, problemType, sourceDir, codeObjectFiles, resultsFileName, parametersFilePath, libraryFile=None): + + with open(parametersFilePath, "w") as f: + def param(key, value): + f.write("{}={}\n".format(key, value)) + + if libraryFile is None: + libraryFilename = "TensileLibrary.yaml" if globalParameters["LibraryFormat"] == "yaml" else "TensileLibrary.dat" + libraryFile = os.path.join(sourceDir, "library", libraryFilename) + param("library-file", libraryFile) + + currentGFXName = getGfxName(globalParameters["CurrentISA"]) + for coFile in codeObjectFiles: + if 'gfx' not in coFile or currentGFXName in coFile: + param("code-object", os.path.join(sourceDir,coFile)) + + param('results-file', resultsFileName) + param('performance-metric', globalParameters["PerformanceMetric"]) + param('problem-identifier', problemType.operationIdentifier) + param('a-type', problemType.aType.toEnum()) + param('b-type', problemType.bType.toEnum()) + param('c-type', problemType.cType.toEnum()) + param('d-type', problemType.dType.toEnum()) + param('alpha-type', problemType.alphaType.toEnum()) + param('beta-type', problemType.betaType.toEnum()) + param('use-bias', problemType.useBias) + + param('high-precision-accumulate', problemType.highPrecisionAccumulate) + param('strided-batched', problemType.stridedBatched) + + for problem in problemSizes.problems: + for key,value in problemSizeParams(problemType, problem): + param(key,value) + + if activationArgs: + for setting in activationArgs.settingList: + param('activation-enum-args', setting.activationEnum.toEnum()) + param('activation-type', problemType.activationType.toEnum()) + param('activation-hpa', problemType.activationHPA) + if globalParameters["DataInitValueActivationArgs"]: + param('activation-additional-args', ','.join(map(str, globalParameters["DataInitValueActivationArgs"]))) + + param("device-idx", globalParameters["Device"]) + + for key,value in dataInitParams(problemType): + param(key, value) + + param("c-equal-d", globalParameters["CEqualD"]) + + param("offset-a", globalParameters["BufferOffsetA"]) + param("offset-b", globalParameters["BufferOffsetB"]) + param("offset-c", globalParameters["BufferOffsetC"]) + param("offset-d", globalParameters["BufferOffsetD"]) + + if globalParameters["PrintTensorA"]: + param("print-tensor-a", 1) + if globalParameters["PrintTensorB"]: + param("print-tensor-b", 1) + if globalParameters["PrintTensorC"]: + param("print-tensor-c", 1) + if globalParameters["PrintTensorD"]: + param("print-tensor-d", 1) + if globalParameters["PrintTensorRef"]: + param("print-tensor-ref", 1) + if globalParameters["DumpTensors"]: + param("dump-tensors", 1) + if globalParameters["ExitOnFails"] > 1: + param("exit-on-error", 1) + + param("bounds-check", boundsCheckName(int(globalParameters["BoundsCheck"]))) + param("print-valids", globalParameters["ValidationPrintValids"]) + param("print-max", globalParameters["ValidationMaxToPrint"]) + param("num-benchmarks", globalParameters["NumBenchmarks"]) + param("num-elements-to-validate", globalParameters["NumElementsToValidate"]) + param("num-enqueues-per-sync", globalParameters["EnqueuesPerSync"]) + param("num-syncs-per-benchmark", globalParameters["SyncsPerBenchmark"]) + param("use-gpu-timer", globalParameters["KernelTime"]) + param("hardware-monitor", globalParameters["HardwareMonitor"]) + if not globalParameters["KernelTime"]: + param("num-warmups", 1) + param("sleep-percent", globalParameters["SleepPercent"]) + param("perf-l2-read-hits", globalParameters["PerfModelL2ReadHits"]) + param("perf-l2-write-hits", globalParameters["PerfModelL2WriteHits"]) + param("perf-l2-read-bw-mul", globalParameters["PerfModelL2ReadBwMul"]) + param("perf-read-efficiency", globalParameters["PerfModelReadEfficiency"]) + param("csv-export-extra-cols", globalParameters["CSVExportWinner"]) + param("csv-merge-same-problems", globalParameters["CSVMergeSameProblemID"]) + param("log-level", ClientLogLevel(globalParameters["ClientLogLevel"]).name) + param("max-workspace-size", globalParameters["MaxWorkspaceSize"]) + param("granularity-threshold", globalParameters["GranularityThreshold"]) + param("pristine-on-gpu", globalParameters["PristineOnGPU"]) + + param("library-update-file", globalParameters["LibraryUpdateFile"]) + param("library-update-comment", globalParameters["LibraryUpdateComment"]) + + +def writeClientConfig(forBenchmark, solutions, problemSizes, activationArgs, stepName, stepBaseDir, newLibrary, codeObjectFiles, tileAwareSelection, configBase = "ClientParameters", libraryFile = None): + + if tileAwareSelection: + filename = os.path.join(globalParameters["WorkingPath"], "%s_Granularity.ini"%configBase) + else: + filename = os.path.join(globalParameters["WorkingPath"], "%s.ini"%configBase) + + if len(newLibrary.solutions)==0: + raise RuntimeError ("No valid solutions found") + + resultsFileName = None + if tileAwareSelection: + resultsFileName = os.path.join(stepBaseDir, "../Data", stepName+"_Granularity.csv") + else: + resultsFileName = os.path.join(stepBaseDir, "../Data", stepName+".csv") + + newSolution = next(iter(newLibrary.solutions.values())) + sourceDir = os.path.join(stepBaseDir, "source") + writeClientConfigIni(problemSizes, activationArgs, newSolution.problemType, sourceDir, codeObjectFiles, resultsFileName, filename, libraryFile) + + return filename + +def CreateBenchmarkClientParametersForSizes(libraryRootPath, problemSizes, dataFilePath, configFile, problemTypeDict=None): + + libraryPath = os.path.join(libraryRootPath, "library") + libraryFiles = [os.path.join(libraryPath, f) for f in os.listdir(libraryPath)] + codeObjectFiles = [f for f in libraryFiles if f.endswith("co")] + + if problemTypeDict: + problemType = ContractionsProblemType.FromOriginalState(problemTypeDict) + else: + # if the we can library contains meta data then we can get the problem type this data + metaDataFilePath = os.path.join(libraryPath, "metadata.yaml") + if not os.path.exists(metaDataFilePath): + printExit ("meta data file %s does not exist" % metaDataFilePath) + metaData = LibraryIO.readYAML(metaDataFilePath) + problemTypeDict = metaData["ProblemType"] + problemType = ContractionsProblemType.FromOriginalState(problemTypeDict) + + writeClientConfigIni(problemSizes, "", problemType, libraryRootPath, codeObjectFiles, dataFilePath, configFile) + + +################################################################################ +# Write Generated Benchmark Parameters +################################################################################ +def writeClientParameters(forBenchmark, solutions, problemSizes, stepName, \ + functionList, stepBaseDir, solutionSummationSizes, solutionWriter = None): + h = "" + + ############################################################################## + # Min Naming + ############################################################################## + """ + if forBenchmark: + kernels = [] + for solution in solutions: + solutionKernels = solution.getKernels() + for kernel in solutionKernels: + if kernel not in kernels: + kernels.append(kernel) + + solutionSerialNaming = Solution.getSerialNaming(solutions) + kernelSerialNaming = Solution.getSerialNaming(kernels) + solutionMinNaming = Solution.getMinNaming(solutions) + kernelMinNaming = Solution.getMinNaming(kernels) + """ + + if forBenchmark: + if globalParameters["MergeFiles"]: + h += "#include \"Solutions.h\"\n" + else: + for solution in solutions: + solutionName = solutionWriter.getSolutionName(solution) + h += "#include \"" + solutionName + ".h\"\n" + h += "#include \"Solutions.h\"\n" + h += "#include \"ReferenceCPU.h\"\n" + h += "\n" + else: + h += "#include \"Solutions.h\"\n" + h += "#include \"Tensile.h\"\n" + + + h += "typedef enum {\n" + h += " enum_float,\n" + h += " enum_double,\n" + h += " enum_TensileComplexFloat,\n" + h += " enum_TensileComplexDouble\n" + h += "#ifdef Tensile_ENABLE_HALF\n" + h += " ,enum_TensileHalf\n" + h += "#endif\n" + h += " ,enum_TensileInt8x4\n" + h += " ,enum_TensileInt32\n" + h += " ,enum_tensile_bfloat16\n" + h += "} DataTypeEnum;\n" + h += "\n" + + h += "// Debug Params\n" + h += "const unsigned printTensorA=%x;\n" % int(globalParameters["PrintTensorA"]) + h += "const unsigned printTensorB=%x;\n" % int(globalParameters["PrintTensorB"]) + h += "const unsigned printTensorC=%x;\n" % int(globalParameters["PrintTensorC"]) + h += "const unsigned printTensorD=%x;\n" % int(globalParameters["PrintTensorD"]) + + h += "const bool printWinnersOnly=%s;\n" % toCppBool(globalParameters["PrintWinnersOnly"]) + h += "\n" + + h += "const char indexChars[%u] = \"%s" \ + % (len(globalParameters["IndexChars"])+1, \ + globalParameters["IndexChars"][0]) + for i in range(1, len(globalParameters["IndexChars"])): + h += globalParameters["IndexChars"][i] + h += "\";\n" + + h += "unsigned int functionIdx;\n" + h += "unsigned int dataTypeIdx;\n" + h += "unsigned int problemTypeIdx;\n" + h += "\n" + + ############################################################################## + # Problem Types + ############################################################################## + #dataTypes = [] + #problemTypes = [] + #functionSerialToDataTypeAndIdx = [] + dataTypes = [] + problemTypes = [] + destDataTypes = {} + computeDataTypes = {} + problemTypesForDataType = {} # for data type + schedulesForProblemType = {} # for problem type + functionInfo = [] # dataTypeIdx, problemTypeIdx, idxWithinDataType, idxWithinProblemType + #tileSelection = False + + if forBenchmark: + problemType = solutions[0]["ProblemType"] + dataType = problemType["DataType"] + #tileSelection = problemType["TileAwareSelection"] + + destDataType = problemType["DestDataType"] + destDataTypes[dataType] = destDataType + + computeDataType = problemType["ComputeDataType"] + computeDataTypes[dataType] = computeDataType + + dataTypes.append(dataType) + + problemTypes.append(problemType) + problemTypesForDataType[dataType] = [problemType] + schedulesForProblemType[problemType] = solutions + numProblemTypes = 1 + for solution in solutions: + functionInfo.append([ 0, 0, 0, 0, 0, 0 ]) + else: + for functionIdx in range(0, len(functionList)): + function = functionList[functionIdx] + scheduleName = function[0] + problemType = function[1] + dataType = problemType["DataType"] + destDataType = problemType["DestDataType"] + computeDataType = problemType["ComputeDataType"] + if dataType not in dataTypes: + dataTypes.append(dataType) + destDataTypes[dataType] = destDataType + computeDataTypes[dataType] = computeDataType + problemTypesForDataType[dataType] = [] + if problemType not in problemTypesForDataType[dataType]: + problemTypesForDataType[dataType].append(problemType) + schedulesForProblemType[problemType] = [] + schedulesForProblemType[problemType].append(scheduleName) + + # sort + dataTypes = sorted(dataTypes) + for dataType in dataTypes: + problemTypesForDataType[dataType] = \ + sorted(problemTypesForDataType[dataType],key=str) + for problemType in problemTypesForDataType[dataType]: + schedulesForProblemType[problemType] = \ + sorted(schedulesForProblemType[problemType],key=str) + + # assign info + functionIdxSerial = 0 + problemTypeIdxSerial = 0 + for dataTypeIdxSerial in range(0, len(dataTypes)): + dataType = dataTypes[dataTypeIdxSerial] + functionIdxForDataType = 0 + for problemTypeIdxForDataType in range(0, \ + len(problemTypesForDataType[dataType])): + problemType = \ + problemTypesForDataType[dataType][problemTypeIdxForDataType] + problemTypes.append(problemType) + functionIdxForProblemType = 0 + for functionIdxForProblemType in range(0, \ + len(schedulesForProblemType[problemType])): + functionInfo.append([ \ + dataTypeIdxSerial, \ + problemTypeIdxForDataType, \ + problemTypeIdxSerial, \ + functionIdxSerial,\ + functionIdxForDataType,\ + functionIdxForProblemType, \ + ]) + functionIdxForProblemType += 1 + functionIdxForDataType += 1 + functionIdxSerial += 1 + problemTypeIdxSerial += 1 + numProblemTypes = problemTypeIdxSerial + numFunctions = functionIdxSerial + h += "const unsigned int numFunctions = %u;\n" % numFunctions + + ############################################################################## + # Data Types + ############################################################################## + h += "/* data types */\n" + numDataTypes = len(dataTypes) + h += "const unsigned int numDataTypes = %u;\n" % numDataTypes + h += "const DataTypeEnum dataTypeEnums[numDataTypes] = { enum_%s" \ + % dataTypes[0].toCpp() + for dataTypeIdx in range(1, numDataTypes): + h += ", enum_%s" % dataTypes[dataTypeIdx].toCpp() + h += " };\n" + # bytes per elements + h += "const unsigned int bytesPerElement[numDataTypes] = { %u" \ + % (dataTypes[0].numBytes()) + for dataTypeIdx in range(1, numDataTypes): + dataType = dataTypes[dataTypeIdx] + h += ", %u" % dataType.numBytes() + h += " };\n" + # flops per mac + if dataTypes[0].isInt8x4(): + h += "const unsigned int numFlopsPerMac[numDataTypes] = { %u" % (8 if dataTypes[0].isReal() else 32) + else: + h += "const unsigned int numFlopsPerMac[numDataTypes] = { %u" % (2 if dataTypes[0].isReal() else 8) + for dataTypeIdx in range(1, numDataTypes): + dataType = dataTypes[dataTypeIdx] + h += ", %u" % (2 if dataType.isReal() else 8) + h += " };\n" + for dataTypeIdx in range(0, numDataTypes): + h += "#define Tensile_DATA_TYPE_%s\n" \ + % dataTypes[dataTypeIdx].toCpp().upper() + + ############################################################################## + # Problem Types + ############################################################################## + h += "/* problem types */\n" + h += "const unsigned int numProblemTypes = %u;\n" % numProblemTypes + # Num C Indices + h += "const unsigned int numIndicesC[numProblemTypes] = { %u" \ + % problemTypes[0]["NumIndicesC"] + for problemTypeIdx in range(1, numProblemTypes): + problemType = problemTypes[problemTypeIdx] + h += ", %u" % problemType["NumIndicesC"] + h += " };\n" + + # Num AB Indices + maxNumIndicesA = len(problemTypes[0]["IndexAssignmentsA"]) + maxNumIndicesB = len(problemTypes[0]["IndexAssignmentsB"]) + h += "const unsigned int numIndicesA[numProblemTypes] = { %u" \ + % len(problemTypes[0]["IndexAssignmentsA"]) + for problemTypeIdx in range(1, numProblemTypes): + problemType = problemTypes[problemTypeIdx] + numIndicesA = len(problemType["IndexAssignmentsA"]) + h += ", %u" % numIndicesA + maxNumIndicesA = max(numIndicesA, maxNumIndicesA) + h += " };\n" + h += "const unsigned int maxNumIndicesA = %u;\n" % maxNumIndicesA + + h += "const unsigned int numIndicesB[numProblemTypes] = { %u" \ + % len(problemTypes[0]["IndexAssignmentsB"]) + for problemTypeIdx in range(1, numProblemTypes): + problemType = problemTypes[problemTypeIdx] + numIndicesB = len(problemType["IndexAssignmentsB"]) + h += ", %u" % numIndicesB + maxNumIndicesB = max(numIndicesB, maxNumIndicesB) + h += " };\n" + h += "const unsigned int maxNumIndicesB = %u;\n" % maxNumIndicesB + + # Index Assignments A + h += "const unsigned int indexAssignmentsA[numProblemTypes][maxNumIndicesA] = {\n" + for problemTypeIdx in range(0, numProblemTypes): + problemType = problemTypes[problemTypeIdx] + indices = problemType["IndexAssignmentsA"] + h += " { %u" % indices[0] + for i in range(1, maxNumIndicesA): + if i < len(indices): + h += ", %u" % indices[i] + else: + h += ", static_cast(-1)" + if problemTypeIdx < numProblemTypes-1: + h += " },\n" + else: + h += " }\n" + h += "};\n" + # Index Assignments B + h += "const unsigned int indexAssignmentsB[numProblemTypes][maxNumIndicesB] = {\n" + for problemTypeIdx in range(0, numProblemTypes): + problemType = problemTypes[problemTypeIdx] + indices = problemType["IndexAssignmentsB"] + h += " { %u" % indices[0] + for i in range(1, maxNumIndicesB): + if i < len(indices): + h += ", %u" % indices[i] + else: + h += ", static_cast(-1)" + if problemTypeIdx < numProblemTypes-1: + h += " },\n" + else: + h += " }\n" + h += "};\n" + # Index Assignments LD + h += "const unsigned int numIndicesLD = %u;\n" % problemType["NumIndicesLD"] + h += "const unsigned int indexAssignmentsLD[numIndicesLD] = {" + if problemType["NumIndicesLD"] > 0: + h += " %u" % problemType["IndexAssignmentsLD"][0] + for ldIdx in range(1, len(problemType["IndexAssignmentsLD"])): + h += ", %u" % problemType["IndexAssignmentsLD"][ldIdx] + h += "};\n" + # beta + h += "bool useBeta[numProblemTypes] = { %s" \ + % ("true" if problemTypes[0]["UseBeta"] else "false") + for problemTypeIdx in range(1, numProblemTypes): + problemType = problemTypes[problemTypeIdx] + h += ", %s" % ("true" if problemType["UseBeta"] else "false") + h += " };\n" + # Complex Conjugates + h += "const bool complexConjugateA[numProblemTypes] = { %s" \ + % ("true" if problemTypes[0]["ComplexConjugateA"] else "false" ) + for problemTypeIdx in range(1, numProblemTypes): + problemType = problemTypes[problemTypeIdx] + h += ", %s" % ("true" if problemTypes[0]["ComplexConjugateA"] else "false" ) + h += " };\n" + h += "const bool complexConjugateB[numProblemTypes] = { %s" \ + % ("true" if problemTypes[0]["ComplexConjugateB"] else "false" ) + for problemTypeIdx in range(1, numProblemTypes): + problemType = problemTypes[problemTypeIdx] + h += ", %s" % ("true" if problemTypes[0]["ComplexConjugateB"] else "false" ) + h += " };\n" + h += "\n" + + if not forBenchmark: + h += "// dataTypeIdxSerial, problemTypeIdxForDataType, problemTypeIdxSerial, functionIdxSerial, functionIdxForDataType, functionIdxForProblemType\n" + first = True + h += "const unsigned int functionInfo[numFunctions][6] = {\n" + for info in functionInfo: + h += "%s{ %u, %u, %u, %u, %u, %u }" % (" " if first else ",\n ", \ + info[0], info[1], info[2], info[3], info[4], info[5] ) + first = False + h += " };\n" + + + ############################################################################## + # Problem Sizes + ############################################################################## + maxNumIndices = problemTypes[0]["TotalIndices"] + if not forBenchmark: + for problemType in problemTypes: + maxNumIndices = max(problemType["TotalIndices"], maxNumIndices) + h += "const unsigned int maxNumIndices = %u;\n" % maxNumIndices + h += "const unsigned int totalIndices[numProblemTypes] = { %u" \ + % problemTypes[0]["TotalIndices"] + for problemTypeIdx in range(1, numProblemTypes): + h += ", %u" % problemTypes[problemTypeIdx]["TotalIndices"] + h += " };\n" + if forBenchmark: + h += "const unsigned int numProblems = %u;\n" \ + % problemSizes.totalProblemSizes + h += "const unsigned int problemSizes[numProblems][%u] = {\n" \ + % (problemTypes[0]["TotalIndices"] + problemType["NumIndicesLD"]) + for i in range(problemSizes.totalProblemSizes): + #assert problemSizes.problems[i].stridesA == None # new stride functionality only supported on new client, not here + problemSize = problemSizes.problems[i].sizes + line = " {%5u" %problemSize[0] + for j in range(1, problemTypes[0]["TotalIndices"] + problemType["NumIndicesLD"]): + line += ",%5u" % problemSize[j] + line += " }" + h += line + if i < problemSizes.totalProblemSizes-1: + h += "," + else: + h += "" + h += "};\n" + h += "const unsigned int minStrides[%u] = {" \ + % problemTypes[0]["TotalIndices"] + for i in range(0, len(problemSizes.minStrides)): + if (i!=0): + h += ", " + h += str(problemSizes.minStrides[i]) + h += "};\n" + else: + h += "unsigned int userSizes[maxNumIndices];\n" + h += "unsigned int minStrides[%u] = {" \ + % maxNumIndices + for i in range(0, maxNumIndices): + if (i!=0): + h += ", " + h += str(0); # always use 0 for minStrides in benchmark mode + h += "};\n" + + if forBenchmark: + h += "/* problem sizes */\n" + """ + h += "const bool indexIsSized[maxNumIndices] = {" + for i in range(0, problemSizes.totalIndices): + h += " %s" % ("true" if problemSizes.indexIsSized[i] else "false") + if i < problemSizes.totalIndices-1: + h += "," + h += " };\n" + + h += "const unsigned int numIndicesSized = %u;\n" \ + % len(problemSizes.indicesSized) + h += "const unsigned int indicesSized[numIndicesSized][4] = {\n" + h += "// { min, stride, stride_incr, max }\n" + for i in range(0, len(problemSizes.indicesSized)): + r = problemSizes.indicesSized[i] + h += " { %u, %u, %u, %u }" % (r[0], r[1], r[2], r[3]) + if i < len(problemSizes.indicesSized)-1: + h += "," + h += "\n" + h += " };\n" + + numIndicesMapped = len(problemSizes.indicesMapped) + h += "const unsigned int numIndicesMapped = %u;\n" % numIndicesMapped + if numIndicesMapped > 0: + h += "#define Tensile_INDICES_MAPPED 1\n" + h += "const unsigned int indicesMapped[numIndicesMapped] = {" + for i in range(0, numIndicesMapped): + h += " %u" % problemSizes.indicesMapped[i] + if i < numIndicesMapped-1: + h += "," + h += " };\n" + else: + h += "#define Tensile_INDICES_MAPPED 0\n" + """ + + ############################################################################## + # Max Problem Sizes + ############################################################################## + if forBenchmark: + maximumD = problemSizes.maxD + maximumC = problemSizes.maxC + maximumA = problemSizes.maxA + maximumB = problemSizes.maxB + maximumW = problemSizes.maxD * 32; + + maxMT = getMaxSolutionSizes(solutions, solutionSummationSizes) + + maxMN = 1296 * maxMT[0] * maxMT[1] + maxMK = 36 * maxMT[0] * maxMT[2] + maxNK = 36 * maxMT[1] * maxMT[2] + + maximumA = max(maximumA, maxMK) + maximumB = max(maximumB, maxNK) + maximumC = max(maximumC, maxMN) + maximumD = max(maximumD, maxMN) + maximumW = max(maximumW, maxMN) + + h += "size_t maxSizeD = %u;\n" % (maximumD) + h += "size_t maxSizeC = %u;\n" % (maximumC) + h += "size_t maxSizeA = %u;\n" % (maximumA) + h += "size_t maxSizeB = %u;\n" % (maximumB) + h += "size_t maxSizeW = %u;\n" % (maximumW) + h += "\n" + else: + h += "size_t maxSizeD;\n" + h += "size_t maxSizeC;\n" + h += "size_t maxSizeA;\n" + h += "size_t maxSizeB;\n" + h += "size_t maxSizeW;\n" + h += "\n" + + ############################################################################## + # Current Problem Size + ############################################################################## + h += "/* current problem size */\n" + #h += "unsigned int fullSizes[maxNumIndices];\n" + #h += "unsigned int currentSizedIndexSizes[numIndicesSized];\n" + #h += "unsigned int currentSizedIndexIncrements[numIndicesSized];\n" + h += "\n" + + ############################################################################## + # Solutions + ############################################################################## + if forBenchmark: + # Solution Ptrs + h += "/* solutions */\n" + # Problem Type Indices + h += "const unsigned int maxNumSolutions = %u;\n" % len(solutions) + h += "float solutionPerf[numProblems][maxNumSolutions]; // milliseconds\n" + h += "\n" + + h += "static const SolutionInfo solutions[maxNumSolutions] = {\n" + for i in range(0, len(solutions)): + solution = solutions[i] + solutionName = solutionWriter.getSolutionName(solution) + h += " {(void*)%s, \"%s\", {%d, %d, %d, %d, %s} }" % \ + (solutionName, solutionName, + solution["AssertSummationElementMultiple"], + solution["AssertFree0ElementMultiple"], + solution["AssertFree1ElementMultiple"], + "false" + ) + if i < len(solutions)-1: + h += "," + h += "\n" + h += " };\n" + h += "\n" + + numSummations = len(solutionSummationSizes) + h += "const unsigned int numSummations = %d;\n" % (numSummations) + + h += "const unsigned int summations[numSummations] = {%d" % (solutionSummationSizes[0]) + for i in range(1, numSummations): + h += ", %d" % (solutionSummationSizes[i]) + h += "};\n" + + ############################################################################## + # Solution meta data + ############################################################################## + + transA = solutions[0]["ProblemType"]["TransposeA"] + transB = solutions[0]["ProblemType"]["TransposeB"] + h += "const unsigned int solutionMetaData[maxNumSolutions][10] = {\n" + for i in range(0, len(solutions)): + solution = solutions[i] + + wg = solution["WorkGroup"] + tt = solution["ThreadTile"] + mt0 = wg[0] * tt[0] + mt1 = wg[1] * tt[1] + gsu = solution["GlobalSplitU"] + lsu = wg[2] + + h += " {%d, %d, %d, %d, %d, %d, %d, %d, %d, %d}" % (mt0,mt1,tt[0],tt[1],wg[0],wg[1],transA,transB,gsu,lsu) + + if (i < len(solutions) - 1): + h += ",\n" + else: + h += "\n" + h += " };\n" + h += "\n" + + + + else: + # Function Names + functionNames = [] + for dataType in dataTypes: + for problemType in problemTypesForDataType[dataType]: + # example scheduleName is fiji, vega10, etc + for scheduleName in schedulesForProblemType[problemType]: + functionNames.append("tensile_%s" % (problemType)) + h += "const char *functionNames[numFunctions] = {\n" + for functionIdx in range(0, len(functionNames)): + functionName = functionNames[functionIdx] + h += " \"%s\"%s\n" % (functionName, \ + "," if functionIdx < len(functionNames)-1 else "" ) + h += " };\n" + + ############################################################################## + # Runtime Structures + ############################################################################## + h += "/* runtime structures */\n" + h += "TensileStatus status;\n" + if globalParameters["RuntimeLanguage"] == "OCL": + h += "cl_platform_id platform;\n" + h += "cl_device_id device;\n" + h += "cl_context context;\n" + h += "cl_command_queue stream;\n" + else: + h += "hipStream_t stream;\n" + #h += "int deviceIdx = %u;\n" \ + # % (globalParameters["Device"]) + h += "\n" + h += "void *deviceWS;\n" + h += "void *deviceD;\n" + h += "void *deviceC;\n" + h += "void *deviceA;\n" + h += "void *deviceB;\n" + + ############################################################################## + # Benchmarking and Validation Parameters + ############################################################################## + h += "\n/* benchmarking parameters */\n" + #h += "const bool measureKernelTime = %s;\n" \ + # % ("true" if globalParameters["KernelTime"] else "false") + #h += "const unsigned int numEnqueuesPerSync = %u;\n" \ + # % (globalParameters["EnqueuesPerSync"]) + #h += "const unsigned int numSyncsPerBenchmark = %u;\n" \ + # % (globalParameters["SyncsPerBenchmark"]) + #h += "unsigned int numElementsToValidate = %s;\n" \ + # % (str(globalParameters["NumElementsToValidate"]) \ + # if globalParameters["NumElementsToValidate"] >= 0 \ + # else "0xFFFFFFFF" ) + #h += "unsigned int validationMaxToPrint = %u;\n" \ + # % globalParameters["ValidationMaxToPrint"] + #h += "bool validationPrintValids = %s;\n" \ + # % ("true" if globalParameters["ValidationPrintValids"] else "false") + h += "size_t validationStride;\n" + if problemType["HighPrecisionAccumulate"]: + h += "static bool useHighPrecisionAccumulate = true;\n" + else: + h += "static bool useHighPrecisionAccumulate = false;\n" + #h += "unsigned int dataInitTypeC = %s;\n" % globalParameters["DataInitTypeC"] + #h += "unsigned int dataInitTypeAB = %s;\n" % globalParameters["DataInitTypeAB"] + h += "\n" + + ############################################################################## + # Generated Call to Reference + ############################################################################## + h += "/* generated call to reference */\n" + h += "template\n" + h += "TensileStatus generatedCallToReferenceCPU(\n" + h += " const unsigned int *sizes,\n" + h += " const unsigned int *minStrides,\n" + h += " DestDataType *referenceD,\n" + h += " DestDataType *referenceC,\n" + h += " DataType *initialA,\n" + h += " DataType *initialB,\n" + h += " const unsigned int lda,\n" + h += " const unsigned int ldb,\n" + h += " const unsigned int ldc,\n" + h += " const unsigned int ldd,\n" + h += " const unsigned int stride_a,\n" + h += " const unsigned int stride_b,\n" + h += " const unsigned int stride_c,\n" + h += " const unsigned int stride_d,\n" + h += " ComputeDataType alpha,\n" + h += " ComputeDataType beta,\n" + h += " bool useHighPrecisionAccumulate) {\n" + h += " return tensileReferenceCPU(\n" + h += " referenceD,\n" + h += " referenceC,\n" + h += " initialA,\n" + h += " initialB,\n" + h += " lda,\n" + h += " ldb,\n" + h += " ldc,\n" + h += " ldd,\n" + h += " stride_a,\n" + h += " stride_b,\n" + h += " stride_c,\n" + h += " stride_d,\n" + h += " alpha,\n" + h += " beta,\n" + h += " totalIndices[problemTypeIdx],\n" + h += " sizes,\n" + h += " minStrides,\n" + h += " numIndicesC[problemTypeIdx],\n" + h += " numIndicesA[problemTypeIdx],\n" + h += " numIndicesB[problemTypeIdx],\n" + h += " indexAssignmentsA[problemTypeIdx],\n" + h += " indexAssignmentsB[problemTypeIdx],\n" + h += " complexConjugateA[problemTypeIdx],\n" + h += " complexConjugateB[problemTypeIdx],\n" + h += " validationStride,\n" + h += " useHighPrecisionAccumulate);\n" + h += "};\n" + h += "\n" + + ############################################################################## + # Generated Call to Solution + ############################################################################## + if forBenchmark: + problemType = solutions[0]["ProblemType"] + h += "/* generated call to solution */\n" + h += "template\n" + h += "TensileStatus generatedCallToSolution(\n" + h += " const SolutionInfoType &solution,\n" + h += " SolutionLock *solutionLock,\n" + h += " const unsigned int *sizes,\n" + h += " const unsigned int *minStrides,\n" + h += " const unsigned int lda,\n" + h += " const unsigned int ldb,\n" + h += " const unsigned int ldc,\n" + h += " const unsigned int ldd,\n" + h += " const unsigned int stride_a,\n" + h += " const unsigned int stride_b,\n" + h += " const unsigned int stride_c,\n" + h += " const unsigned int stride_d,\n" + h += " ComputeDataType alpha,\n" + h += " ComputeDataType beta,\n" + h += " unsigned int numEvents = 0,\n" + if globalParameters["RuntimeLanguage"] == "OCL": + h += " cl_event *event_wait_list = NULL,\n" + h += " cl_event *outputEvent = NULL ) {\n" + else: + h += " hipEvent_t *startEvent = NULL,\n" + h += " hipEvent_t *stopEvent = NULL ) {\n" + + h += " // calculate parameters assuming packed data\n" + # strides + indexChars = globalParameters["IndexChars"] + firstStride = 1 + #assert(not problemType["UseInitialStridesCD"]) # not supported in old client + if problemType["UseInitialStridesAB"]: + firstStride = 0 + lastStrideD = problemType["NumIndicesC"] + lastStrideC = problemType["NumIndicesC"] + lastStrideA = len(problemType["IndexAssignmentsA"]) + lastStrideB = len(problemType["IndexAssignmentsB"]) + + # calculate strides + for i in range(0,lastStrideD): + h += " unsigned int strideD%u%s = 1" % (i, indexChars[i]) + for j in range(0, i): + h += " * (" + if j == 0: + h += "(ldd != std::numeric_limits::max()) ? ldd : " + h += "std::max(minStrides[%i], sizes[%i]))" % (j,j) + h += ";\n" + h += " if (stride_d != std::numeric_limits::max()) strideD%u%s = stride_d;\n" % (lastStrideD-1, indexChars[lastStrideD-1]) + for i in range(0,lastStrideC): + h += " unsigned int strideC%u%s = 1 " % (i, indexChars[i]) + for j in range(0, i): + h += " * (" + if j == 0: + h += "(ldc != std::numeric_limits::max()) ? ldc : " + h+= "std::max(minStrides[%i], sizes[%i]))" % (j,j) + h += ";\n" + h += " if (stride_c != std::numeric_limits::max()) strideC%u%s = stride_c;\n" % (lastStrideC-1, indexChars[lastStrideC-1]) + + constStride = None + for i in range(0,lastStrideA): + idx = problemType["IndexAssignmentsA"][i] + constStride = checkConstStride(problemType["SetConstStrideA"], idx) + if constStride != None: + h += " unsigned int strideA%u%s = %d; //SetConstStrideA\n" % (i, + indexChars[problemType["IndexAssignmentsA"][i]], + constStride) + else: + h += " unsigned int strideA%u%s = 1" % (i, \ + indexChars[problemType["IndexAssignmentsA"][i]]) + for j in range(0, i): + h += " * (" + if j == 0: + h += "(lda != std::numeric_limits::max()) ? lda : " + h += "std::max(minStrides[%i], sizes[%i]))" % \ + (problemType["IndexAssignmentsA"][j], + problemType["IndexAssignmentsA"][j]) + h += ";\n" + if constStride == None: + h += " if (stride_a != std::numeric_limits::max()) strideA%u%s = stride_a;\n" % (lastStrideA-1, indexChars[problemType["IndexAssignmentsA"][lastStrideA-1]]) + + for i in range(0,lastStrideB): + idx = problemType["IndexAssignmentsB"][i] + constStride = checkConstStride(problemType["SetConstStrideB"], idx) + if constStride != None: + h += " unsigned int strideB%u%s = %d; //SetConstStrideB\n" % (i, + indexChars[problemType["IndexAssignmentsB"][i]], + constStride) + else: + h += " unsigned int strideB%u%s = 1" % (i, \ + indexChars[problemType["IndexAssignmentsB"][i]]) + for j in range(0, i): + h += " * (" + if j == 0: + h += "(ldb != std::numeric_limits::max()) ? ldb : " + h+= "std::max(minStrides[%i], sizes[%i]))" % \ + (problemType["IndexAssignmentsB"][j], + problemType["IndexAssignmentsB"][j]) + h += ";\n" + h += " if (stride_b != std::numeric_limits::max()) strideB%u%s = stride_b;\n" % (lastStrideB-1, indexChars[problemType["IndexAssignmentsB"][lastStrideB-1]]) + + for i in range(0, problemType["TotalIndices"]): + h += " unsigned int size%s = sizes[%u];\n" % (indexChars[i], i) + h += "\n" + + + # function call + h += " // Check assertions,\n" + assert(not problemType["UseInitialStridesCD"]) # not supported in old client + firstStride = 0 if problemType["UseInitialStridesAB"] else 1 + lastStrideD = problemType["NumIndicesC"] + lastStrideC = problemType["NumIndicesC"] + lastStrideA = len(problemType["IndexAssignmentsA"]) + lastStrideB = len(problemType["IndexAssignmentsB"]) + numSizes = problemType["TotalIndices"] + h += " typedef ProblemDims<%u,%u,%u,%u,%u,%u> ProblemDims_%s;\n" \ + % (firstStride, lastStrideD, lastStrideC, lastStrideA, lastStrideB, numSizes, problemType) + # TODO - this should be initialized somewhere once? + h += " static const ProblemType problemType( " + h += listToInitializer(problemType["IndicesFree"]) + ", " + h += listToInitializer(problemType["IndicesSummation"]) + ", " + h += listToInitializer(problemType["IndicesBatch"]) + ', ' + h += listToInitializer(problemType["IndexAssignmentsA"]) + ', ' + h += listToInitializer(problemType["IndexAssignmentsB"]) + h += ");\n" + # create problem size - TODO could move this up to the caller + h += " ProblemDims_%s pdims(" % problemType + indexChars = globalParameters["IndexChars"] + for i in range(firstStride,lastStrideD): + if i != firstStride: h += ", " + h += "strideD%u%s" % (i, indexChars[i]) + for i in range(firstStride,lastStrideC): + h += ", strideC%u%s" % (i, indexChars[i]) + for i in range(firstStride,lastStrideA): + h += ", strideA%u%s" % (i, \ + indexChars[problemType["IndexAssignmentsA"][i]]) + for i in range(firstStride,lastStrideB): + h += ", strideB%u%s" % (i, \ + indexChars[problemType["IndexAssignmentsB"][i]]) + for i in range(0, problemType["TotalIndices"]): + h += ", size%s" % indexChars[i] + h += ");\n" + h += " if (!ProblemProperties(pdims,&problemType).validForSolution(solution._assertionRequirements))\n" + h += " return tensileStatusAssertFailure; // problem dims did not meet requirements for solution\n" + h += "\n" + + h += " // call solution function\n" + h += " TensileSolutionPointer_%s f = reinterpret_cast (solution._functionPtr);\n" \ + % (problemType, problemType) + if globalParameters["RuntimeLanguage"] == "OCL": + h += " return f(solutionLock, static_cast(deviceD), static_cast(deviceC), static_cast(deviceA), static_cast(deviceB),\n" + else: + typeName = dataTypes[0].toCpp() + destTypeName = destDataTypes[dataType].toCpp() + computeTypeName = computeDataTypes[dataType].toCpp() + h += " return f(solutionLock,\n" + h += " static_cast<%s *>(deviceD),\n" % destTypeName + h += " static_cast<%s *>(deviceC),\n" % destTypeName + h += " static_cast<%s *>(deviceA),\n" % typeName + h += " static_cast<%s *>(deviceB),\n" % typeName + h += " alpha,\n" + if problemType["UseBeta"]: + h += " beta,\n" + for i in range(firstStride,lastStrideD): + h += " strideD%u%s,\n" % (i, indexChars[i]) + for i in range(firstStride,lastStrideC): + h += " strideC%u%s,\n" % (i, indexChars[i]) + for i in range(firstStride,lastStrideA): + h += " strideA%u%s,\n" % (i, \ + indexChars[problemType["IndexAssignmentsA"][i]]) + for i in range(firstStride,lastStrideB): + h += " strideB%u%s,\n" % (i, \ + indexChars[problemType["IndexAssignmentsB"][i]]) + for i in range(0, problemType["TotalIndices"]): + h += " size%s,\n" % indexChars[i] + h += " stream,\n" + if globalParameters["RuntimeLanguage"] == "OCL": + h += " numEvents, event_wait_list, outputEvent ); // events\n" + else: + h += " numEvents,\n" + h += " startEvent,\n" + h += " stopEvent,\n" + h += " static_cast(deviceWS)); // events\n" + + h += "};\n" + h += "\n" + else: + ############################################################################ + # Generated Call to Function + ############################################################################ + for enqueue in [True, False]: + functionName = "tensile" if enqueue else "tensileGetSolutionName" + returnName = "TensileStatus" if enqueue else "const char *" + h += "/* generated call to function */\n" + h += "template\n" + h += "%s generatedCallTo_%s(\n" % (returnName, functionName) + h += " unsigned int *sizes,\n" + h += " unsigned int *minStrides,\n" + h += " ComputeDataType alpha,\n" + h += " ComputeDataType beta,\n" + h += " unsigned int lda,\n" + h += " unsigned int ldb,\n" + h += " unsigned int ldc,\n" + h += " unsigned int ldd,\n" + h += " unsigned int strideA,\n" + h += " unsigned int strideB,\n" + h += " unsigned int strideC,\n" + h += " unsigned int strideD,\n" + h += " unsigned int numEvents = 0,\n" + + if globalParameters["RuntimeLanguage"] == "OCL": + h += " cl_event *event_wait_list = NULL,\n" + h += " cl_event *outputEvent = NULL );\n\n" + else: + h += " hipEvent_t *startEvent = NULL,\n" + h += " hipEvent_t *stopEvent = NULL );\n\n" + + +#need to get DestDataType in here + for dataType in dataTypes: + typeName = dataType.toCpp() + destDataType = destDataTypes[dataType] + destTypeName = destDataType.toCpp() + computeDataType = computeDataTypes[dataType] + computeTypeName = computeDataType.toCpp() + functionsForDataType = [] + for problemType in problemTypesForDataType[dataType]: + for scheduleName in schedulesForProblemType[problemType]: + functionsForDataType.append([scheduleName, problemType]) + h += "template<>\n" + h += "inline %s generatedCallTo_%s<%s, %s, %s>(\n" \ + % (returnName, functionName, typeName, destTypeName, computeTypeName) + h += " unsigned int *sizes,\n" + h += " unsigned int *minStrides,\n" + h += " %s alpha,\n" % computeTypeName + h += " %s beta,\n" % computeTypeName + h += " unsigned int lda,\n" + h += " unsigned int ldb,\n" + h += " unsigned int ldc,\n" + h += " unsigned int ldd,\n" + h += " unsigned int strideA,\n" + h += " unsigned int strideB,\n" + h += " unsigned int strideC,\n" + h += " unsigned int strideD,\n" + h += " unsigned int numEvents, \n" + + if globalParameters["RuntimeLanguage"] == "OCL": + h += " cl_event *event_wait_list,\n" + h += " cl_event *outputEvent ) {\n\n" + else: + h += " hipEvent_t *startEvent,\n" + h += " hipEvent_t *stopEvent ) {\n\n" + + h += " unsigned int functionIdxForDataType = functionInfo[functionIdx][4];\n" + + for functionIdx in range(0, len(list(functionsForDataType))): + function = functionsForDataType[functionIdx] + scheduleName = function[0] + problemType = function[1] + if len(list(functionsForDataType))> 1: + if functionIdx == 0: + h += " if (functionIdxForDataType == %u) {\n" % functionIdx + elif functionIdx == len(list(functionsForDataType))-1: + h += " } else {\n" + else: + h += " } else if (functionIdxForDataType == %u) {\n" \ + % functionIdx + + # strides + indexChars = globalParameters["IndexChars"] + firstStride = 1 + assert(not problemType["UseInitialStridesCD"]) # not supported in old client + if problemType["UseInitialStridesAB"]: + firstStride = 0 + lastStrideD = problemType["NumIndicesC"] + lastStrideC = problemType["NumIndicesC"] + lastStrideA = len(problemType["IndexAssignmentsA"]) + lastStrideB = len(problemType["IndexAssignmentsB"]) + + # calculate strides + for i in range(0,lastStrideD): + h += " unsigned int strideD%u%s = 1" % (i, indexChars[i]) + for j in range(0, i): + h += "*sizes[%i]" % j + h += ";\n" + h += " if (strideD != std::numeric_limits::max()) strideD%u%s = strideD;\n" % (lastStrideD-1, indexChars[lastStrideD-1]) + for i in range(0,lastStrideC): + h += " unsigned int strideC%u%s = 1" % (i, indexChars[i]) + for j in range(0, i): + h += "*sizes[%i]" % j + h += ";\n" + h += " if (strideC != std::numeric_limits::max()) strideC%u%s = strideC;\n" % (lastStrideC-1, indexChars[lastStrideC-1]) + + for i in range(0,lastStrideA): + h += " unsigned int strideA%u%s = 1" % (i, \ + indexChars[problemType["IndexAssignmentsA"][i]]) + for j in range(0, i): + h += "*sizes[%i]" % \ + problemType["IndexAssignmentsA"][j] + h += ";\n" + h += " if (strideA != std::numeric_limits::max()) strideA%u%s = strideA;\n" % (lastStrideA-1, indexChars[problemType["IndexAssignmentsA"][lastStrideA-1]]) + for i in range(0,lastStrideB): + h += " unsigned int strideB%u%s = 1" % (i, \ + indexChars[problemType["IndexAssignmentsB"][i]]) + for j in range(0, i): + h += "*sizes[%i]" % \ + problemType["IndexAssignmentsB"][j] + h += ";\n" + h += " if (strideB != std::numeric_limits::max()) strideB%u%s = strideB;\n" % (lastStrideB-1, indexChars[problemType["IndexAssignmentsB"][lastStrideB-1]]) + for i in range(0, problemType["TotalIndices"]): + h += " unsigned int size%s = sizes[%u];\n" % (indexChars[i], i) + + # function call + h += " // call solution function\n" + h += " return %s_%s(\n" % (functionName, problemType) + if enqueue: + if globalParameters["RuntimeLanguage"] == "OCL": + h += " static_cast(deviceD),\n" + h += " static_cast(deviceC),\n" + h += " static_cast(deviceA),\n" + h += " static_cast(deviceB),\n" + else: + h += " static_cast<%s *>(deviceD),\n" % destTypeName + h += " static_cast<%s *>(deviceC),\n" % destTypeName + h += " static_cast<%s *>(deviceA),\n" % typeName + h += " static_cast<%s *>(deviceB),\n" % typeName + h += " alpha,\n" + if problemType["UseBeta"]: + h += " beta,\n" + for i in range(firstStride,lastStrideD): + h += " strideD%u%s,\n" % (i, indexChars[i]) + for i in range(firstStride,lastStrideC): + h += " strideC%u%s,\n" % (i, indexChars[i]) + for i in range(firstStride,lastStrideA): + h += " strideA%u%s,\n" % (i, \ + indexChars[problemType["IndexAssignmentsA"][i]]) + for i in range(firstStride,lastStrideB): + h += " strideB%u%s,\n" % (i, \ + indexChars[problemType["IndexAssignmentsB"][i]]) + for i in range(0, problemType["TotalIndices"]): + h += " size%s%s\n" % (indexChars[i], "," if i != problemType["TotalIndices"]-1 else "") + if enqueue: + if globalParameters["RuntimeLanguage"] == "OCL": + h += ", stream, numEvents, event_wait_list, outputEvent" + else: + h += ", stream, numEvents, startEvent, stopEvent, static_cast(deviceWS)" + h += ");\n" + + if len(functionsForDataType) > 1: + h += " }\n" # close last if + h += "};\n" # close callToFunction + + ############################################################################## + # Results File Name + ############################################################################## + if forBenchmark: + h += "/* results file name */\n" + resultsFileName = os.path.join(stepBaseDir, \ + "../Data","%s.csv" % stepName) + resultsFileName = resultsFileName.replace("\\", "\\\\") + h += "const char *resultsFileName = \"%s\";\n" % resultsFileName + + granularityFileName = os.path.join(stepBaseDir, \ + "../Data","%s_Granularity.csv" % stepName) + + granularityFileName = granularityFileName.replace("\\", "\\\\") + h += "const char *granularityFileName = \"%s\";\n" % granularityFileName + + ############################################################################## + # Write File + ############################################################################## + clientParametersFile = open(os.path.join(globalParameters["WorkingPath"], \ + "ClientParameters.h"), "w") + clientParametersFile.write(CHeader) + clientParametersFile.write(h) + clientParametersFile.close() diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py new file mode 100644 index 0000000000..4f9db63117 --- /dev/null +++ b/tensilelite/Tensile/Common.py @@ -0,0 +1,1662 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from . import __version__ +from . import Parallel +from .TensileInstructions import getGfxName, TensileInstructions +from collections import OrderedDict +from copy import deepcopy + + +import math +import os.path +import subprocess +import sys +import time + +startTime = time.time() + +ParallelMap = Parallel.ParallelMap + +# print level +# 0 - user wants no printing +# 1 - user wants limited prints +# 2 - user wants full prints + +################################################################################ +# Global Parameters +################################################################################ +globalParameters = OrderedDict() +workingDirectoryStack = [] + +######################################## +# common +######################################## +globalParameters["MinimumRequiredVersion"] = "0.0.0" # which version of tensile is required to handle all the features required by this configuration file +globalParameters["PerformanceMetric"] = "DeviceEfficiency" # performance metric for benchmarking; one of {DeviceEfficiency, CUEfficiency} +globalParameters["PrintLevel"] = 1 # how much info to print in generator. 0=none, 1=standard, 2=verbose +globalParameters["ClientLogLevel"] = 3 # the log level of client. 0=Error, 1=Terse, 2=Verbose, 3=Debug (Aligned with ResultReporter.hpp) +# benchmarking +globalParameters["KernelTime"] = False # T=use device timers, F=use host timers +globalParameters["PreciseKernelTime"] = True # T=On hip, use the timestamps for kernel start and stop rather than separate events. Can provide more accurate kernel timing. For GlobalSplitU kernels, recommend disabling this to provide consistent +# timing between GSU / non-GSU kernels +globalParameters["CodeFromFiles"] = True # if False byte arrays will be generated during Benchmarking phase as before +globalParameters["SortProblems"] = False # sort problems by size; else use order in YAML file +globalParameters["PinClocks"] = False # T=pin gpu clocks and fan, F=don't +globalParameters["HardwareMonitor"] = True # False: disable benchmarking client monitoring clocks using rocm-smi. +globalParameters["NumBenchmarks"] = 1 # how many benchmark data points to collect per problem/solution +globalParameters["SyncsPerBenchmark"] = 1 # how iterations of the stream synchronization for-loop to do per benchmark data point +globalParameters["EnqueuesPerSync"] = 1 # how many solution enqueues to perform per synchronization +globalParameters["SleepPercent"] = 300 # how long to sleep after every data point: 25 means 25% of solution time. Sleeping lets gpu cool down more. +# validation +globalParameters["NumElementsToValidate"] = 128 # number of elements to validate, 128 will be evenly spaced out (with prime number stride) across C tensor +globalParameters["BoundsCheck"] = 0 # Bounds check +#1: Perform bounds check to find out of bounds reads/writes. NumElementsToValidate must be -1. +#2: Perform bounds check by front side guard page +#3: Perform bounds check by back side guard page +#4: Perform bounds check by both back and front side guard page + +globalParameters["ValidationMaxToPrint"] = 4 # maximum number of mismatches to print +globalParameters["ValidationPrintValids"] = False # print matches too +# steps +globalParameters["ForceRedoBenchmarkProblems"] = True # if False and benchmarking already complete, then benchmarking will be skipped when tensile is re-run +globalParameters["ForceRedoLibraryLogic"] = True # if False and library logic already analyzed, then library logic will be skipped when tensile is re-run +globalParameters["ForceRedoLibraryClient"] = True # if False and library client already built, then building library client will be skipped when tensile is re-run + +globalParameters["ShowProgressBar"] = True # if False and library client already built, then building library client will be skipped when tensile is re-run +globalParameters["SolutionSelectionAlg"] = 1 # algorithm to detetermine which solutions to keep. 0=removeLeastImportantSolutions, 1=keepWinnerSolutions (faster) +globalParameters["ExpandRanges"] = True # expand ranges into exact configs before writing logic file. False ignores ranges. +globalParameters["ExitAfterKernelGen"] = False # Exit after generating kernels +globalParameters["GenerateSourcesAndExit"] = False # Exit after kernel source generation. +globalParameters["ShowProgressBar"] = True # if False and library client already built, then building library client will be skipped when tensile is re-run +globalParameters["WavefrontWidth"] = 64 # if False and library client already built, then building library client will be skipped when tensile is re-run +globalParameters["ExitOnFails"] = 1 # 1: Exit after benchmark run if failures detected. 2: Exit during benchmark run. +globalParameters["CpuThreads"] = -1 # How many CPU threads to use for kernel generation. 0=no threading, -1 == nproc, N=min(nproc,N). TODO - 0 sometimes fails with a kernel name error? 0 does not check error codes correctly +# FROM MERGE +#globalParameters["CpuThreads"] = -4 # How many CPU threads to use for kernel generation. 0=no threading, <0 == nproc*abs(CpuThreads), N=min(nproc,N) + +# even if error occurs in kernel generation (ie due to resource overflow), +# generate the kernel source anyway. Tensile will also attempt to run +# the kernel. Useful to examine and debug overflow errors. +globalParameters["ForceGenerateKernel"] = 0 + +######################################## +# optimization knob controls +######################################## + +globalParameters["UnrollLoopEfficiencyEnable"] = False # if True split(S) MAC&LDS in each unroll iteration into n smaller groups.. + +######################################## +# less common +######################################## +globalParameters["CMakeBuildType"] = "Release" # whether benchmark clients and library client should be release or debug +globalParameters["PrintSolutionRejectionReason"] = False # when a solution is marked as invalid, print why +globalParameters["LibraryFormat"] = "yaml" # set library backend (either yaml or msgpack) +globalParameters["EmbedLibrary"] = None # whether library should be embedded or not + +# True/False: CSV will/won't export WinnerGFlops, WinnerTimeUS, WinnerIdx, WinnerName. +# TODO - if no side-effect, we can set default to True. This can make analyzing "LibraryLogic" (AddFromCSV) faster +globalParameters["CSVExportWinner"] = False + +# (When NumBenchmarks > 1). True: CSV will merge the rows of same Problem-ID. False: Each problem will write out "NumBenchmarks" rows +# In old client - No effect, since in old client, CSV file only exports the last benchmark, somehow is not correct because the previous benchmarks are discarded +# In new client - csv file exports "NumBenchmarks" rows for every problem. This also make the later analyzing slower +# Set this to "True" can merge the rows for same problem, hence can reduce the csv file size and speed up the later analyzing +# TODO - if side-effect, we can set default to True. This can make "getResults()" / "AddFromCSV()" faster +globalParameters["CSVMergeSameProblemID"] = False + +# how to initialize tensor data +# serial-in-u will use a sequence that increments in the K dimension +# This is a predictable patterns that can be checked as the kernel runs to detect +# when the wrong data is being used. +# trig_float initializes with the sin function to have non-zero values in the mantissa +# and exponent. It cannot be used for int8 or int32. Need to use tensileAlmostEqual +# not tensileEqual for checking the result. +# See ClientWriter.py, the DataInitName(Enum) for a list of initialization patterns +# - Problem-Independent: 0=0, 1=1, 2=2, 3=rand, 4=Nan, 5=Infinity, 6=BadInput(Nan), 7=BadOutput(Inf), 16=RandomNarrow +# - Problem-dependent: 8=SerialID, 9=SerialDim0, 10=SerialDim1, 11=Identity, 12~15= Cos/Sin, Abs or Not +# For A, B, C, D: All the InitMode (0~16) can be used +# For Alpha/Beta: Only problem-independent init (0~7, 16) can be used, +# problem-dependent init (8~15) would cause a exception (Invalid InitMode) in New Client +globalParameters["DataInitTypeAB"] = 3 +globalParameters["DataInitTypeA"] = -1 +globalParameters["DataInitTypeB"] = -1 +globalParameters["DataInitTypeC"] = 3 +globalParameters["DataInitTypeD"] = 0 +globalParameters["DataInitTypeAlpha"] = 2 +globalParameters["DataInitTypeBeta"] = 2 +globalParameters["DataInitTypeBias"] = 1 +globalParameters["DataInitValueActivationArgs"] = [2.0, 2.0] +globalParameters["CEqualD"] = False # Set to true if testing for the case where the pointer to C is the same as D. +globalParameters["BufferOffsetA"] = 0 # data offset of buffer A +globalParameters["BufferOffsetB"] = 0 # data offset of buffer B +globalParameters["BufferOffsetC"] = 0 # data offset of buffer C +globalParameters["BufferOffsetD"] = 0 # data offset of buffer D + +# build parameters +globalParameters["CMakeCXXFlags"] = "" # pass flags to cmake +globalParameters["CMakeCFlags"] = "" # pass flags to cmake +globalParameters["DebugKernel"] = False # assembly only, kernel gets buffer for debug "printing"; kernel writes data to memory, gets coppied to host and printed +globalParameters["LibraryPrintDebug"] = False # solutions will print enqueue info when enqueueing a kernel + +# debug for assembly +globalParameters["EnableAsserts"] = False # Enable assembly debug assert +globalParameters["EnableDebugA"] = False # Enable / Disable CheckValue1A +globalParameters["EnableDebugB"] = False # Enable / Disable CheckValue1B +globalParameters["EnableDebugC"] = False # Enable / Disable CheckValueC +globalParameters["ExpectedValueC"] = 16.0 # Expected C Value when CheckValueC, debug for Alpha*A*B +globalParameters["ForceCExpectedValue"] = False # Force C to "DebugExpectedValueC", debug for global write + +# Tensor printing controls: +globalParameters["PrintTensorA"] = 0 # Print TensorA after initialization +globalParameters["PrintTensorB"] = 0 # Print TensorB after initialization +globalParameters["PrintTensorC"] = 0 # Print TensorC. 0x1=after init; 0x2=after copy-back; 0x3=both +globalParameters["PrintTensorD"] = 0 # Print TensorD. 0x1=after init; 0x2=after copy-back; 0x3=both +globalParameters["PrintTensorRef"] = 0 # Print reference tensor. 0x1=after init; 0x2=after copy-back; 0x3=both +globalParameters["PrintIndexAssignments"] = 0 # Print the tensor index assignment info +globalParameters["PrintWinnersOnly"] = False # Only print the solutions which become the fastest +globalParameters["PrintCodeCommands"] = False # print the commands used to generate the code objects (asm,link,hip-clang, etc) +globalParameters["DumpTensors"] = False # If True, dump tensors to binary files instead of printing them. + +# If PrintMax* is greater than the dimension, the middle elements will be repaced with "..." + + +# device selection +globalParameters["Platform"] = 0 # select opencl platform +globalParameters["Device"] = 0 # select hip device or opencl device within platform + +# shouldn't need to change +globalParameters["DeviceLDS"] = 65536 # LDS bytes per CU, for computing occupancy +globalParameters["MaxLDS"] = 65536 # max LDS a kernel should attempt to use +globalParameters["MaxDepthU"] = 256 # max DepthU value to allow +globalParameters["ShortNames"] = False # on windows kernel names can get too long; =True will convert solution/kernel names to serial ids +globalParameters["MergeFiles"] = True # F=store every solution and kernel in separate file; T=store all solutions in single file +globalParameters["NumMergedFiles"] = 1 # The number of files that kernels should be split between when merging + +globalParameters["MaxFileName"] = 64 # If a file name would be longer than this, shorten it with a hash. +globalParameters["SupportedISA"] = [(8,0,3), (9,0,0), (9,0,6), (9,0,8), (9,0,10), (10,1,0), (10,1,1), (10,1,2), (10,3,0), (11,0,0), (11,0,1), (11,0,2)] # assembly kernels writer supports these architectures + +globalParameters["GenerateManifestAndExit"] = False # Output manifest file with list of expected library objects and exit +globalParameters["NewClient"] = 2 # Old client deprecated: NewClient must be set to 2. +globalParameters["ClientBuildPath"] = "0_Build" # subdirectory for host code build directory +globalParameters["BenchmarkProblemsPath"] = "1_BenchmarkProblems" # subdirectory for benchmarking phases +globalParameters["BenchmarkDataPath"] = "2_BenchmarkData" # subdirectory for storing final benchmarking data +globalParameters["LibraryLogicPath"] = "3_LibraryLogic" # subdirectory for library logic produced by analysis +globalParameters["LibraryClientPath"] = "4_LibraryClient" # subdirectory for building example library client +globalParameters["ClientExecutionLockPath"] = None # Path for a file lock to ensure only one client is executed at once. filelock module is required if this is enabled. +globalParameters["LibraryUpdateFile"] = "" # File name for writing indices and speeds suitable for updating an existing library logic file +globalParameters["LibraryUpdateComment"] = False # Include solution name as a comment in the library update file + +# internal, i.e., gets set during startup +globalParameters["CurrentISA"] = (0,0,0) +globalParameters["ROCmAgentEnumeratorPath"] = None # /opt/rocm/bin/rocm_agent_enumerator +globalParameters["ROCmSMIPath"] = None # /opt/rocm/bin/rocm-smi +globalParameters["AssemblerPath"] = None # /opt/rocm/hip/bin/hipcc +globalParameters["WorkingPath"] = os.getcwd() # path where tensile called from +globalParameters["IndexChars"] = "IJKLMNOPQRSTUVWXYZ" # which characters to use for C[ij]=Sum[k] A[ik]*B[jk] +globalParameters["ScriptPath"] = os.path.dirname(os.path.realpath(__file__)) # path to Tensile/Tensile.py +globalParameters["SourcePath"] = os.path.join(globalParameters["ScriptPath"], "Source") # path to Tensile/Source/ +globalParameters["HipClangVersion"] = "0.0.0" + +# default runtime is selected based on operating system, user can override +if os.name == "nt": + globalParameters["RuntimeLanguage"] = "HIP" #"OCL" +else: + globalParameters["RuntimeLanguage"] = "HIP" + +globalParameters["CodeObjectVersion"] = "V3" +globalParameters["CxxCompiler"] = "hipcc" +globalParameters["Architecture"] = "all" + +# might be deprecated +globalParameters["EnableHalf"] = False +globalParameters["ClientArgs"] = "" +globalParameters["PackageLibrary"] = False + +# perf model +globalParameters["PerfModelL2ReadHits"] = 0.0 +globalParameters["PerfModelL2WriteHits"] = 0.15 +globalParameters["PerfModelL2ReadBwMul"] = 2 +globalParameters["PerfModelReadEfficiency"] = 0.85 + +# limitation for training +globalParameters["MaxWorkspaceSize"] = 32 * 1024 * 1024 # max workspace for training (32M) +globalParameters["MinKForGSU"] = 256 # min K size to use GlobalSplitU algorithm (only for HPA now) + +# control if a solution is run for a given problem +globalParameters["GranularityThreshold"] = 0.0 + +# directory where custom kernels are located +globalParameters["CustomKernelDirectory"] = os.path.join(os.path.dirname(os.path.realpath(__file__)), "CustomKernels") + +globalParameters["PristineOnGPU"] = True # use Pristine memory on Tensile trainning verification or not + +globalParameters["SeparateArchitectures"] = False # write Tensile library metadata to separate files for each architecture + +# Save a copy - since pytest doesn't re-run this initialization code and YAML files can override global settings - odd things can happen +defaultGlobalParameters = deepcopy(globalParameters) + +# Translate GPU targets to filter filenames in Tensile_LOGIC directory +architectureMap = { + 'all':'_','gfx000':'none', 'gfx803':'r9nano', 'gfx900':'vega10', + 'gfx906':'vega20', 'gfx906:xnack+':'vega20', 'gfx906:xnack-':'vega20', + 'gfx908':'arcturus','gfx908:xnack+':'arcturus', 'gfx908:xnack-':'arcturus', + 'gfx90a':'aldebaran', 'gfx90a:xnack+':'aldebaran', 'gfx90a:xnack-':'aldebaran', + 'gfx1010':'navi10', 'gfx1011':'navi12', 'gfx1012':'navi14', 'gfx1030':'navi21', + 'gfx1100':'navi31', 'gfx1101':'navi32', 'gfx1102':'navi33' +} + +def getArchitectureName(gfxName): + if gfxName in architectureMap: + return architectureMap[gfxName] + else: + for archKey in architectureMap: + if gfxName in archKey: + return architectureMap[archKey] + return None + +################################################################################ +# Enumerate Valid Solution Parameters +################################################################################ +validWorkGroups = [] +for numThreads in range(64, 1025, 64): + for nsg in [ 1, 2, 4, 8, 16, 32, 64, 96, 128, 256 ]: + for sg0 in range(1, numThreads//nsg+1): + sg1 = numThreads//nsg//sg0 + if sg0*sg1*nsg == numThreads: + workGroup = [sg0, sg1, nsg] + validWorkGroups.append(workGroup) + +validThreadTileSides = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + list(range(20, 256, 4)) +validThreadTiles = [] +for i in validThreadTileSides: + for j in validThreadTileSides: + validThreadTiles.append([i, j]) + +validActivationFormats = ('NCHW', 'NHWC', 'CNHW', 'NCDHW', 'NDHWC', 'CNDHW') +validWeightFormats = ('KCYX', "KYXC", "CKYX", "CYXK", 'KCZYX', 'CKZYX', 'CZYXK') +validMacroTileSides = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 6, 12, 24, 48, 96, 192, 384, 768 ] +validMacroTiles = [] +validISA = [(0,0,0)] +validISA.extend(globalParameters["SupportedISA"]) +depthUs = list(range(-16, 0)) +depthUs.extend(list(range(2,512+1,1))) +for i in validMacroTileSides: + for j in validMacroTileSides: + validMacroTiles.append([i, j]) + +validMFMA = {} +validMFMA["H"] = [[32,32,4,2], [32,32,8,1], [16,16,4,4], [16,16,16,1], [4,4,4,16]] +validMFMA["S"] = [[32,32,1,2], [32,32,2,1], [16,16,1,4], [16,16,4,1], [4,4,1,16]] +validMFMA["B"] = [[32,32,2,2], [32,32,4,1], [16,16,2,4], [16,16,8,1], [4,4,2,16]] +validMFMA["D"] = [[16,16,4,1], [4,4,4,4]] +validMFMA["B1k"] = [[32,32,4,2], [32,32,8,1], [16,16,4,4], [16,16,16,1], [4,4,4,16]] +validMFMA["C"] = validMFMA["S"] +validMFMA["Z"] = validMFMA["D"] +validMFMA["I8"] = [[32,32,4,2], [32,32,8,1], [16,16,4,4], [16,16,16,1], [4,4,4,16]] +validTT = 16 +validMFMA["_format9"] = [] + +for MFMA in [validMFMA["H"], validMFMA["S"], validMFMA["B"], validMFMA["D"]]: + for MI in MFMA: + for bm in range(int(math.log(MI[3],2))+1): + for tt0 in range(1,validTT+1): + for tt1 in range(1,validTT+1): + for wave_m in range (3): + for wave_n in range(3): + validMFMA["_format9"].append([MI[0],MI[1],MI[2],MI[3],2**bm,tt0,tt1,2**wave_m, 2**wave_n]) +validMatrixInstructions = [[], [-1]] + validMFMA["H"] + validMFMA["S"] + validMFMA["B"] + validMFMA["D"] + validMFMA["B1k"] +validMatrixInstructions = validMatrixInstructions + validMFMA["_format9"] + +# The supported typed GEMM, each entry is (Ti, To, Tc). +# DataType (Ti) = The data-type of the input matrices: A/B +# DestDataType (To) = The data-type of the output matrices: C/D +# ComputeDataType (Tc) = The data-type of computaiton: alpha/beta: +# Cinternal: basically should == ComputeDataType + +# Align the supported GEMM type with rocBLAS: [A/B/ C/D/ alpha/beta] +# (rocblas/library/include/internal/rocblas_functions.h) +# GEMM (HPA=F, the data type of input, output, and computation are all the same.) +# - HGEMM: [H/H/ H/H/ H/H] +# - SGEMM: [S/S/ S/S/ S/S] +# - DGEMM: [D/D/ D/D/ D/D] +# - CGEMM: [C/C/ C/C/ C/C] +# - ZGEMM: [Z/Z/ Z/Z/ Z/Z] +# GEMM_Ex: (HPA=T, Computation is in a higher precision data-type) +# - GEMM_EX (HHS): [H/H/ H/H/ S/S] +# - GEMM_EX (HSS): [H/H/ S/S/ S/S] +# - GEMM_EX (BBS): [B/B/ B/B/ S/S] +# - GEMM_EX (BSS): [B/B/ S/S/ S/S] +# - GEMM_EX (I8II): [I8/I8/ I/I/ I/I] +# - GEMM_EX (4xi8II): [4xi8/4xi8/ I/I/ I/I], tensile packs 4 i8 to 4xi8 with some restrictions +# This is used in SolutionStruct.py::checkIfSupportedGEMMType() +validGEMMTypes = [ ('H','H','H'), ('S','S','S'), ('D','D','D'), ('C','C','C'), ('Z','Z','Z'), \ + ('H','H','S'), ('H','S','S'), \ + ('B','B','S'), ('B','S','S'), \ + ('I8','I','I'), ('4xi8','I','I'), ('I8','I8','I')] + +# All HPA types are listed here (HPA=T). The name of the library logic files for these types is: +# *_TiToTc_BH*.yaml where Ti, Tc, and To are the data types of A/B, C/D, and computation, respectively. +# The name of the library logic files for non-HPA (HPA=F) types is: *_TiB*.yaml. +HPATypes = [ ('H','S','S'), ('H','H','S'), ('B','B','S'), ('B','S','S'), ('I8','I','I'), ('4xi8','I','I')] + +validParameters = { + # original global read to lds is interlace, [w0,w1,w2,w3,w0,w1,w2,w3,w0,w1,w2,w3,w0,w1,w2,w3] + # when WaveSeparateGlobalRead is enabled, LDS is divided to number of waves part. + # each wave load a block memory to lds, [w0,w0,w0,w0,w1,w1,w1,w1,w2,w2,w2,w2,w3,w3,w3,w3] + # -1 is selected by logic, 0 disable, 1 enable. + "WaveSeparateGlobalReadA": [ 0, 1 ], + "WaveSeparateGlobalReadB": [ 0, 1 ], + + # PrefetchGlobalRead = 1: + # Requires 2X LDS space, and VGPRs for buffering data on way into LDS + # prefetch / double-buffer reads from global memory -> vgprs -> lds. + # + # PrefetchGlobalRead = 2: + # Do another prefetch while writing data from vgpr to lds. + # prefetch / double-buffer reads from global memory -> vgprs --> lds. + # |-> prefetch reads + "PrefetchGlobalRead": [ 0, 1, 2 ], + + # number of iteration prefetch local reads from lds to VGPRs buffer = PLR % LoopIter + # number of VGPRs buffer = min(PLR+1,LoopIters) + # LoopIters = DepthU / LocalSplitU + # (LoopIters /= MatrixInstruction_K) + # ex. MT64x128x16_MI32x32x4x2_PLR1, we'll have 4 LoopIters, prefetch read 1 iteration, with 2 VGPRs buffer (2=min(1+1,4)) + # before loop: plr[0] + # loop: iter0:plr[1] MAC_r[0], iter1:plr[0] MAC_r[1], iter2:plr[1] MAC_r[0], iter3:plr[0] MAC_r[1] + # no load loop: iter0:plr[1] MAC_r[0], iter1:plr[0] MAC_r[1], iter2:plr[1] MAC_r[0], iter3: MAC_r[1] + # + # ex. MT64x128x16_MI32x32x4x2_PLR3, we'll have 4 LoopIters, prefetch read 3 iteration, with 4 VGPRs buffer (4=min(3+1,4)) + # before loop: plr[0] plr[1] plr[2] + # loop: iter0:plr[3] MAC_r[0], iter1:plr[0] MAC_r[1], iter2:plr[1] MAC_r[2], iter3:plr[2] MAC_r[3] + # no load loop: iter0:plr[3] MAC_r[0], iter1: MAC_r[1], iter2: MAC_r[2], iter3: MAC_r[3] + # + # ex. MT64x128x16_MI32x32x4x2_PLR5, we'll have 4 LoopIters, prefetch read 5%4=1 iteration, with 4 VGPRs buffer (4=min(5+1,4)) + # before loop: plr[0] + # loop: iter0:plr[1] MAC_r[0], iter1:plr[2] MAC_r[1], iter2:plr[3] MAC_r[2], iter3:plr[0] MAC_r[3] + # no load loop: iter0:plr[1] MAC_r[0], iter1:plr[2] MAC_r[1], iter2:plr[3] MAC_r[2], iter3: MAC_r[3] + # + # ex. MT64x128x16_MI32x32x4x2_PLR5_LRVW8, we'll have 4 LoopIters, prefetch read 5%4=1 iteration, with 4 VGPRs buffer (4=min(5+1,4)) , each read read 2 iterations + # before loop: plr[0:1] + # loop: iter0:plr[2:3] MAC_r[0], iter1: MAC_r[1], iter2: MAC_r[2], iter3:plr[0:1] MAC_r[3] + # no load loop: iter0:plr[2:3] MAC_r[0], iter1: MAC_r[1], iter2: MAC_r[2], iter3: MAC_r[3] + # + # ex. MT64x128x16_MI32x32x4x2_PLR7, we'll have 4 LoopIters, prefetch read 7%4=3 iteration, with 4 VGPRs buffer (=min(7+1,4)) --> Exactly the same as PLR3 + # before loop: plr[0] + # loop: iter0:plr[1] MAC_r[0], iter1:plr[2] MAC_r[1], iter2:plr[3] MAC_r[2], iter3:plr[0] MAC_r[3] + # no load loop: iter0:plr[1] MAC_r[0], iter1:plr[2] MAC_r[1], iter2:plr[3] MAC_r[2], iter3: MAC_r[3] + "PrefetchLocalRead": list(range(128+1)), + + # We use double LDS buffer when PrefetchGlobalRead. + # While it reads data from LDS[0]/[1], it prefetch global data and writes to LDS[1]/[0] + # If we can make sure all data are read from LDS to register before writing data to LDS, we can use 1 LDS buffer to save LDS memory. + # this can help to generate Kernel that LDS usage originally exceed MaxLDS if using double LDS buffer, + # or help to increase Occupancy. + # 1 means: Force to use 1 LDS Buffer even with PrefetchGlobalRead + # -1 means: generator will use 1 LDS buffer only when LDS exceed MaxLDS + # Use case: + # SIA2: 1LDSBuffer is set to 1 natively + # SIA3: 1LDSBuffer works only when PGR=True + # TODO: optimize scheduling to support more cases. + "1LDSBuffer": [-1 ,0, 1], + + # Split the unroll summation into multiple sections and combine the sections + # GSU applies only to the unroll summation dimension + "GlobalSplitU": list(range(1, 1024+1)), + + # choose how to do GlobalSplitU + # 1: use atomic operation to accumlate on one buffer + # 2: each GSU group write to each own buffer and accumulate by another kernel + "GlobalSplitUAlgorithm": ["SingleBuffer", "MultipleBuffer"], + + # don't create a whole copy of the Unroll loop with loads removed - instead + # use buffer limits to suppress global loads and ignore unnecessary ds_reads + "SuppressNoLoadLoop": [False, True], + + # For PrefetchGlobalRead=1, create a second copy of the unroll loop with + # the LDS pointer swaps expanded into inline constants for LDS read and write instructions + # This eliminates 4 vector XOR instructions used for pointer swap + "ExpandPointerSwap": [False, True], + + # Schedule global reads and global read incrementsinto LocalRead iterations + # Can reduce pressure on local read instruction dispatch queue + # 0=perform global reads at start of instruction loop + # 1=schedule into the local read instruction iterations + "ScheduleGlobalRead": [0, 1], + + # Schedule local writes into LocalRead iterations. + # Can reduce pressure on local read instruction dispatch queue + "ScheduleLocalWrite": [0, 1], + + # Scheduling algorithm to use for each iteration: + # 0 = minimal/no scheduling. Global Read and increments, followed by local reads, + # followed by local writes, followed by MACs + "ScheduleIterAlg": [0, 1, 2, 3], + + # For MatrixInstruction and SIA3, number of GlobalReadInstruction between mfma + # the purpose of this parameter is to control density of global read instruction scheduling + # Scheduling global read back to back can have better memory efficiency + # However, when full of vmem FIFO, it will block other instruction to be issued + # Range from 0.01 to 32 + # 0.1 means 1 GR per 10 mfma + # 5 means 5 GR per 1 mfma + "GlobalReadPerMfma": [ i/100 for i in range(1,3200)], + # + # For MatrixInstruction and SIA3, number of LocalWriteInstruction between mfma + # the purpose of this parameter is to control density of local write instruction scheduling + # In PGR1, we want to schedule local write more denser, so we can have more + # latency to hide global read + # In PGR2, since LW is followed by GR, every LW has same whole loop latecy + # to hide global read. We want to schedule LW less denser, can + # avoid full of vmem FIFO. + # Range from 0.01 to 32 + # 0.1 means 1 LW per 10 mfma + # 5 means 5 LW per 1 mfma + # -1 will derived an optimized value internally + # -2 will derived an optimized value and override LWPM silently (debug only, not recommended) + "LocalWritePerMfma": [ i/100 for i in range(1,3200)] + [ -1 ], + + # Interleave alpha scale calculation with beta loads and address calcs - rather + # than as a separate block of instructions + "InterleaveAlpha": [0, 1], + + # Create a copy of NoLoadLoop which interleaves the stores with the final mac + # calculation and may perform other optimizations + # 0 = no interleave + # 1 = interleave one stores after required macs have completed execution + # 2 = interleave two stores after required macs have completed execution + "OptNoLoadLoop": [0, 1, 2], + + "BufferLoad": [ False, True ], + "BufferStore": [ False, True ], + + # Attempt to load directly from global memory into Vgpr. + # Assembly only + "DirectToVgprA": [ False, True ], + "DirectToVgprB": [ False, True ], + + # Attempt to load directly from global memory into LDS. + # Assembly only + # Requires BufferLoad, assembler support for lds modifier on buffer + # loads (checked automatically), GlobalVectorWidth=1 (this is hw + # requirement) and A/B must not require any transpose. + # DirectToLds reduces load latency and eliminates the + # G2L registers used to stage data. Also replaces the + # local write offset with an SGPR. + # For an 8x8 TT with PrefetchGlobalRead=1 this can save 33 VGPRs. + # - Requirements for DirectToLds=1: + # GlobalLoadVectorWidth = 1/2/4 + # TransposeLDS = 1 for TLU=0 case + # DirectToLds support for x2/x4 (1st part of async_copy() support) + "DirectToLds": [ False, True ], + + # Load options: + # (GRO = Global Read Offset) + # BufferLoad=0: + # = Use flat instructions with 64 bit GRO for each load + # + supports sizes up to 2^64 + # - uses many VGPR for addressing + # - uses execmask+compares for edge detection + # - generates extra LDS traffic (could convert flat->global load) + # BufferLoad=1: + # = Use buffer load instructions with 32-bit offset + # + Less VGPRS (32b offset vs 64-bit) needed for addressing + # + Uses hardware buffer limit for edge detection + # - Limited range - the bot-right corner of macro-tile (plus padding=GRVW + # for shift-pointer, if ShiftPtr is required) must be within 2^32. + # ShiftPtrPad = MayShift ? GRWV*BPE : 0 + # For TLU=1: Unroll*StrideA1 + ShiftPtrPad <= 2^32 + # For TLU=0: MT*StrideA1 + ShiftPtrPad <= 2^32 + # These conditions should be checked using Assert - TODO + # = UseSgprForGRO=1: + # + Attempt to use SGPR for Global Read Offsets. + # + Use one VGPR base GRO + many SGPR GRO rather than many VGPR GRO. + # + Each SGPR stores an offset from base GlobalReadOffset+0. + # - Requirements for UseSgprForGRO=1: + # - BufferLoad=1 + # - Use appropriate Assert*ElementMultiple or GRVW=1 to eliminate need for ShifPtr + # (UseSgprForGRO does not support ShiftPtr since ShiftPtr needs to potentially shift GRO) + # = KernelWriterAssembly also supports 64-bit 2D buffer size (see use64bPbcLimit) + # - Requires 4 instructions to move scalar limit and a couple SGPR + # - Enabled by default. If the overhead matters we can add asserts/YAML parm to specialize + # = UseInstOffsetForGRO=1: + # + Attempt to use Instruction offset for Global Read Offsets. + # + This feature avoid updating m0 for subsequent GRO(s) for directToLds feature + # - Requirements for UseInstOffsetForGRO=1: + # - BufferLoad=1 + # - DirectToLds=1 + + # converting m0 update from LocalWriteAddrSGpr using is usually win + # -1 attempt to use a hueristic to determine when the tile size will use too many SGPR and fall back to VGPR + "UseInstOffsetForGRO": [ -1, 0, 1], + + + # Converting VGPR GRO into SGPR GRO is usually a win + # However, the mode may exhaust all available SGPR, in particular for large unroll + # -1 attempt to use a hueristic to determine when the tile size will use too many SGPR and fall back to VGPR + "UseSgprForGRO": [ -1, 0, 1], + + # Use a 64-bit shadow limit register to allow buffers larger than 2^32 bytes + "Use64bShadowLimit": [ True, False], + + # Assertion properties + # These provide information or assertions that the problem size meets certain requirements + # for sizes or alignments. The kernel generator can use this information to produce + # a kernel which uses those assertions to produce a faster kernel. + # + # If modifying or adding Assertions also change ProblemProperties class in TensileTypes.h + + # Kernel generator will assume that the summation size is some multiple of the element size + # and uses this to optimize the kernel. + # This can result in more efficient kernels, but requires runtime checking to ensure the specified + # summation value meets the requirements. + # (Recommended AF1EM value is 8 for half, 4 for single, 2 for double) + # + # Optimizations enabled by AssertSummationElementMultiple>1: + # - If >=2 for half: + # - Tail loop loads can be vectorized 2X to use dword + # - Enables asm kernels on V20 + # - Can use DirectToLds for both unroll and tail loops + # - Tail loop can be unrolled up to InnerUnroll amount if AssertSummationElementMultiple%InnerUnroll==0 + # + # 1 indicates no assertion (since all sizes are multiples of 1) + "AssertSummationElementMultiple": [1,2,4,8,16,32,64], + + # Kernel generator will assume that the FreeIndex[0] size is some multiple of the element size + # and uses this to optimize the kernel. + # FreeIndex[0] is usually letter "I" + # (Recommended AF0EM value is 8 for half, 4 for single, 2 for double) + # + # Optimizations enabled by AssertFree0ElementMultiple>1: + # Load optimizations: + # - For TLU=1 matrix, if AF1WM>=GLVW then can enable UseSgprForGRO + # - Reduces registers used for address calculations + # - Removes address shift/unshift code + # - UseSgprForGRO will only be enabled if all matrices meet assertion requirements. + # + # Store Optimizations: + # - Can vectorize stores in edge tiles. Vector width can be up to AF0EM. + # (since C matrix is always coalesced in Free0 index direction and this assertion guarantees the index element multiple) + # + # 1 indicates no assertion (since all sizes are multiples of 1) + "AssertFree0ElementMultiple" : [1,2,4,8], + + # Kernel generator will assume that the FreeIndex[1] size is some multiple of the element size + # and uses this to optimize the kernel. + # FreeIndex[1] is usually letter "J" + # (Recommended AF1EM value is 8 for half, 4 for single, 2 for double) + + # Optimizations enabled by AssertFree1ElementMultiple>1: + # - See above AssertFree0ElementMultiple "Load optimizations" + + # 1 indicates no assertion (since all sizes are multiples of 1) + "AssertFree1ElementMultiple" : [1,2,4,8], + + # Stagger the start summation position of the tiles. + # Elements from the summation dimension are loaded at offsets rather than all starting at 0. + # StaggerU is the max 'clicks' of StaggerUStride bytes where each wg starts ; see StaggerUMapping + # for how the specific stagger for a given wg is determined. + # + # The tile assignment C are same as with StaggerOffset=0 ; the difference is the + # order that the summation elements are added. + # GRO will wrap back to the row start when the edge is reached. + # + # This can be effective for TLU=0 style matrices where the K dimension is a large power-of-2. + # In this case the start of each row of the tile is separated by an exact power-of-2 + # which causes poor dram, cache, and tlb behavior. V20 has 16 channels each 256 bytes wide. + + # StaggerU adjusts the start position in the summation (aka 'U') dimension + # to avoid these conflicts. Both A and B matrix start at the adjusted position. + # If >0 specifies the offset in multiples of the macro-tile "unroll" dim + # - Higher values will spread traffic to more channels but provide less L2 re-use. + # - StaggerU and WorkGroupMapping interact and should be tuned together - + # The WGM controls how tiles are assigned in C matrix, while StaggerU controls where those + # tiles start reading their summation dim parms. + # - StaggerU requires BufferLoad==1 and is silently ignored if BufferLoad==0 + "StaggerU": [0,2,4,8,16,32,64], + + # Stride in bytes for each staggeru 'click'. + # 256 is recommended since this is the width of memory channel (on gfx803,gfx900,gf906) - so + # each click will start in a new memory channel and spread traffic among the 16 available channels. + # For example StaggerUStride=256 and StaggerU=8 will use 8 unique starting points + # in summation dimension, each offset by 256-bytes - provided the tensor dims are large + # enough to support this. + # StaggerUStride will be internally increased so it is an integer multiple of DepthU*BpeAB. + # (the implementation requires this - the unroll iteration accesses data in steps of + # DepthU*BPE + "StaggerUStride": [16,32,64,128,256,512,1024], + + # How the tile assignment (wg0, wg1, wg2) controls the initial StaggerU offset: + # 0: Use wg0 + # 1: Use wg1 + # 2: Use wg2 + # 3: Use wgSerial, wgSerial = wg0 + wg1 * nwg0 + wg2 * (nwg0 * nwg1) + # 4: Debug mode, offset each tile max allowed StaggerU. This just moves hotspot + # to a different bank since all workgroups still start at same point. + "StaggerUMapping": [0,1,2,3,4], + + + # 0=don't use magic div (source only) + # 1=magic div alg #1. Slightly faster but limited range (if magic number is 2^32) + # 2=magic div alg#2. Slightly slower but handles all unsigned ints up to 2^32 + "MagicDivAlg": [0,1,2], + + # For Block Mapping type: + # 0 : Use hardware-assigned wg number with no remapping. + # N : WG block width. "Wrap" to a new wg1 "row" assignment after N WGs assigned in that row. + # Tensor C always mapped with first free coord as fastest moving + # (Elements in this dimension are sequential in memory. + # + # For 2D nonbatched Matrix this means index order is I, then J + # For 2D batched Matrix this means index order is I, then J, then K. + # + # Then for 2D case: + # - If drawn in row-major format, I is the width and J is the height. + # - WGM determines dimensions of the box used to assign tiles from C + # - WGM is the height of the box (in the J dimension) + # - Given WGM, the box width (in I dim) is determined by number of CUs + # - The box always moves across matrixC in the fastest-moving "I" dim, then + # wraps to next J. TODO - might be useful to change this? + # + # Examples for 2D matrix: + # WGM=8: on CU64 machine this is a square box + # WGM=1: Short/Fat - this will cover maximum width in I dimension of C. This matches hardware assigned mapping. + # WGM=64: Tall/Skinny - this will cover maximum width in J dimention of C. + # + # Formula for wgSerial: + # wgSerial = wg0 + (wg1 % WorkGroupMapping) * nwg0 + "WorkGroupMapping": list(range(0,1024+1)), # change a workgroup's id so that the all the workgroups on the gpu at a time are hitting L2 cache the best + "MaxOccupancy": list(range(1, 40+1)), # wg / CU; if cache thrashing is hurting performance, this allocates extra lds to artificially limit occupancy + "WorkGroup": validWorkGroups, # ( wg0 x wg1 x LocalSplitU ) dimensions of the workgroup which will operate on a tile and share lds + + #ThreadTile: ( tt0 x tt1 ) dimensions of the C tile that each thread works on, + # TT=4 and VW=4 means a thread will work on a tight 4x4 tile of C, where VW=1 means the tile will work on 16 spread out values + # Generally, the VW determines the consecutive a WI will work on, then it will skip ahead SG0*VW elements to get to the next row of VGPR inputs + "ThreadTile": validThreadTiles, + "MacroTile": validMacroTiles, # MT0 = wg0*tt0, MT1 = wg1*tt1 + + "WavefrontSize": [32, 64], + + # MatrixInstruction: (M x N x K x B) + # XDLOPS tile definition, only valid for gfx908, gfx90a + # MxNxKxB specifies matrix instruction variants + # MxNxB determines the shape of the C tile each instruction worked on + # K determines the unroll depth + # If empty, do not use these instructions + # + # Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN) + # (Note: MxN means M-by-N in the following comments) + # MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples: + # - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed + # - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed + # WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels + # - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN) + # WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads + # - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 + # Putting it all together: + # - [32, 32, 1, 2, 1, 4, 1, 2, 2] + # ^^^^^^^^^^^^ ^ ^^^^ ^^^^ + # MatrixInst BlkM WT Wave + # - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile + # Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used + "MatrixInstruction": validMatrixInstructions, + + # StoreRemap: Optimize MatrixInstruction store patterns to enhance performance. + # MI output data between each threads are along N dims. + # But global memory is along M dim continous. + # That mean global write between each threads are not continous. + # Therefore, store performance for MI instruction is poor. + # How StoreRemap works in final store stage: + # 1. Put all thread output data into LDS. + # 2. All thread read data from LDS along M dims. + # (match global Memory continous direction) + # 3. All thread write out data into global memory. + # 0: Disable StoreRemap (default) + # 1~8: Enable StoreRemap and set the global write vector width + # Suggest optimum value: fp32 = [2,4], fp16 or bf16 = [4,8] (dwordx2 and dowrdx4) + # -1: Use dwordx2 if support SRVW, or set SRVW to 0 + "StoreRemapVectorWidth": [-1,0,1,2,4,8], + + # SourceSwap: Optimizes MatrixInstruction store pattern by swapping mfma input order. + "SourceSwap": [False, True], + + # Following parameters are designed for store scheduling. + # (store stands for load from C (with beta) and store to C/D) + # + # we want to hide store behind unroll loop + # 1. if we can launch 2 WorkGroups per CU (occupancy >= 2, large M/N) + # 2. if there are remaining global memory bandwidth in unroll loop (compute bound kernel) + # + # we can hide store behind the other WG's loop by lowering priority of store + # priority of loop is the same as priority of store + # WG0: ￣￣￣￣￣￣￣￣￣￣￣￣￣￣￣\__ + # |<-- loop --->|<-- store -->|end + # + # WG1: ___________________________/￣￣￣￣￣￣￣￣￣￣￣￣\__ + # |<--------- loop ------------------->|<-- store -->|end + # + # priority of loop is higher than priority of store + # WG0: ￣￣￣￣￣￣￣\____________________ + # |<-- loop --->|<------ store ----->|end + # + # WG1: _____________/￣￣￣￣￣\__________________ + # |<------- loop -------->|<----- store ---->|end + "StorePriorityOpt": [False, True], + # + # If we issue store in short period of time, kernel will become from compute bound to memory bound + # 0 means issue instructions as many as possible if VGPR available + "NumElementsPerBatchStore": list(range(0, 256)), + # + # add sync after per batch store in order to store contiguous elements + # add sleep after per batch store in order to distribute store over whole loops + # NOTE: this parameter is highly depends on size_k + # 0 means no sync and sleep + "StoreSyncOpt": list(range(0, 256)), + # + # There are index or address calculation between global instructions. + # issue global instruction b2b has better performance + "GroupLoadStore": [False, True], + + # In order to remove the copying from Acc vgpr to Arch vgpr, only use Arch vgprs for v_mfma_xxx. + # Only support for kernel whose totalVgpr counts less than 256 and gcn that has control bit ACC_CD. + "MIArchVgpr": [False, True], + + # alternate implementation for fp16 HPA MFMA + "Fp16AltImpl": [False, True], + + # Controls desired width (#elements) for loads from global memory -> LDS. + # and eliminates the pointer unshift logic + # -1 : Set GlobalReadVectorWidth = VectorWidth + # NOTE: for input bpe=32, max GRVW is 4 (to fit dwordx4) (FP32), min GRVW is 1 (dword) + # bpe=16, max GRVW is 8 (to fit dwordx4) (FP16), min GRVW is 2 (dword) + # bpe=8, max GRVW is 16 (to fit dwordx4) (INT8), min GRVW is 4 (dword) + "GlobalReadVectorWidth": [ -1, 1, 2, 3, 4, 6, 8, 16 ], + + # Controls desired width (#elements) for loads from LDS -> VGPR. + # -1 : Set LocalReadVectorWidth = VectorWidth + # 1 cannot be used for half type. + # used in combination with TransposeLDS=True + # in TransposeLDS=1 case, use wider load to fetch elements in summation dimension from LDS + # helps optimizing instruction scheduling between MFMA and nonMFMA instructions + # NOTE: for input bpe=32, max LRVW is 4 (to fit ds_read_b128) (FP32) + # bpe=16, max LRVW is 8 (to fit ds_read_b128) (FP16) + # bpe=8, max LRVW is 16 (to fit ds_read_b128) (INT8) + + "LocalReadVectorWidth": [ -1, 1, 2, 4, 8, 16 ], + + # threads should read/write/operate on this many contiguous elements from the C matrix. + # If VW=4 then thread0 will process 4 consec C elements, then thread1 next 4, etc. + # If the ThreadTile is > VectorWidth then thread0 will next operate on the 4 elements in C at (4*NumThreads) + # Typically the load vector width and store vector width are directly related to the VW. + # The global load width is closely related to the width of local stores so + # GlobalReadVectorWidth also controls local write width. + # Local read width also matches since VectorWidth consec elements must be read + # Typically matching 16 bytes is good choice since the stores will be optimally coalesced with 16 bytes/WI. + # -1 means use the largest vector width up to 128 bits. + # Using a VW too large which results in >16bytes/thread isn't supported + # For MFMA non SourceSwap: this parameter didn't take effect + # For MFMA SourceSwap: this parameter only take effect on A buffer for now + "VectorWidth": [ -1, 1, 2, 3, 4, 6, 8 ], + + # If 0, store 1 element per instruction. + # If 1, store vector-width elements per instruction. + # if -1, store vector-wide elements per instruction unless PBD would not generate a valid kernel + "VectorStore": [-1, 0, 1], + + # Controls desired width (#elements) for stores from reg to global memory. + # When MatrixInstruciton == None, derived parameter gwvw takes precedence. + # -1 : Set StoreVectorWidth = VectorWidth + "StoreVectorWidth": [ -1, 1, 2, 3, 4, 6, 8 ], + + # when loading all the data from global into lds requires multiple load instructions, these parameters govern which + # loads will pull which rectangle of data from global into lds + # NLC=1 means one load along the coalesced dimension, which results in the most coalescing possible + # NLC=-1 looks for the largest number of reads along the coalesced dimension which results in the least ammount of coalescing; + # however in this case the stride between one load and another is a static value, therefore buffer loads only need one set of registers + # whereas the =1 case has a stride which is a multiple of a kernel argument and therefore needs one address per load in the perpendicular dimension + "NumLoadsCoalescedA": list(range(-1, 64+1)), + "NumLoadsCoalescedB": list(range(-1, 64+1)), + + # DepthU, LocalSplitU (which is the 3rd number in WorkGroup), and LoopUnroll are closely related + # LoopUnroll=4 means there are 4 subiterations within the loop, 4 actual iterations written in the code. + # LocalSplit=2 means the workgroup is split up into 2 subgroups, and each subgroup is doing different parts of the summation. + # subgroup0 does k=0-3, 8-11... and subgroup1 does k=4-7, 12-15... + # So, each iteration through the summation loop, which has 4 actual subiterations, does 8 summation iterations, because each subgroup did 4; + # and when data is read from global memory the threads read 8 elements along the summation dimension. + # DepthU = LoopUnroll * LocalSplitU = 4*2 in this case + # it made more sense for the user to directly control LocalSplitU and DepthU, then derrive afterwards LoopUnroll=DepthU/LocalSplitU + # -1 : Only allow GLVW=1 + # -2 : Only allow max(GLVWA,GLVWB) < VW ? + # -3 : Only allow min(GLVWA,GLVWB) < VW ? + "DepthU": depthUs, + + # DepthULdsDivisor (Split LDS) determines how we pipeline the data from global memory to LDS + # Instead of moving all in-flight data from the register buffer (G2L) to the LDS at once, we divide the G2L buffer into N portions and + # write each portion of the G2L to LDS, read from LDS and do the actual matrix multiply-accumulate, before moving on to the portion and so on. + # This helps cut down LDS usage by the value of the divisor. Helps increase CU occupancy or DepthU if kernel was previously LDS limited. + # + # The premise is to be able to fetch 256B (equivalent to 128 half's or 64 single's) in TN layout problems to maximize L2 utilization. This + # was previously a problem for TN since it implies DepthU is large, and that leads to oversubscription of LDS. + # + # Preconditions: + # ScheduleIterAlg=3, TransposeLDS=1, PGR=0/1 exlcuding 2, DirectToLds=0 (DirectToLds=0 because part of the data loaded *need* to reside in registers), + # nRegs per load >= DepthULdsDivisor (since we artificially require at least 1 register per LDS write) + # + # Example: DepthULdsDivisor=2 + # v0, v1, v2, v3 | v0, v1, v2, v3 | ... ----> unroll dim + # -----Thd 0----- -----Thd 1----- ... + # 1st subloop writes v0,v1 to LDS + # 2nd subloop writes v2,v3 to LDS + "DepthULdsDivisor": [1, 2, 4], + + # integer ammount of padding to put into LDS, in 2016 this didn't seem to help performance, profilers were showing that channel conflicts weren't really hurting + # performance so this has been deprecated and probably doesn't work + # -1 means use same padding as the VectorWidth if TLU=0 else 0. (Padding only helps when transpose is required) + # With MatrixInstruciton: -1 means max(GRVW,MIInput) if TLU=0 + "LdsPadA": [ -1, 0, 1, 2, 3, 4, 8, 16, 32], + "LdsPadB": [ -1, 0, 1, 2, 3, 4, 8, 16, 32], + + # Padding boundary for LDS. defines block-size for pad insertion. for every 'LdsBlockSizePerPad' bytes, LDS padding (pad value from LdsPad parameter) + # is added (readOffset aware of the pad and adjusts offset value based on this parameter value). + # Only support LdsBlockSizePerPad >= unrollDepth * BPE + # 0 means disable LdsBlockSizePerPad, + # -1 means round up to nearest power of 2 begin with 128 + "LdsBlockSizePerPad": [-1, 0, 64, 128, 256, 512, 1024], + + # Transpose LDS format. Local store in Coalsced dimension , same as optimized global fetch dimension . applicable only in TLU=0 case for miSIMD(s) + # TODO: No code for -1 ? + "TransposeLDS": [-1, 1, 0], + + # add gls or slc after global memory read/writes to change cacheing, not cacheing the writes is promising and improved performance a tiny bit + # 1: glc, 2: slc, 3: glc+slc + "NonTemporalD": list(range(0,4)), + "NonTemporalC": list(range(0,4)), + "NonTemporalA": list(range(0,4)), + "NonTemporalB": list(range(0,4)), + + # Group together unroll iterations inside the unroll loop. + # For example, InnerUnroll=2 will fetch LDS for two unroll iterations + "InnerUnroll": [1,2,4,8,16,32,64], + + # Kernels should be written in assembly or source + # if assembly, ISA will determine architecture + # if source, Runtime will determine language + # later on, we'll relax this to inner kernel languages and outer kernel languages, such as inline asm embedded in ocl or in llvm + "KernelLanguage": [ "Assembly" ], + "ISA": validISA, # arch for assembly kernels + + # Name of the custom kernel located in globalParameters["CustomKernelDirectory"]. + # a custom kernel is a user written assembly kernel with its associated configuration parameters included in a custom.config section + # inside the yaml block between the --- and ... markers. These parameters are only used for information purposes, not kernel generation. + # Ex: + # custom.config: + # ProblemType: + # OperationType: GEMM + # etc... + # ThreadTile: [8, 8] + # etc... + # + # Custom kernels can be included in a BenchmarkProblemSizeGroup by having their name (without file extension) listed under the "CustomKernels" + # category alongside InitialSolutionParameters, BenchmarkCommonParameters, etc... + "CustomKernelName": -1, + + # Will allow a kernel to be accepted even when checks determine it's not viable. + # Intended for use with custom kernels which have confirmed to be correct + "NoReject": [False, True], + + "MinVgprNumber": list(range(0,256)), + + "MaxVgprNumber": list(range(0,257)), + + # Debug use only. + "ActivationFused": [False, True], + + # True- function call + # False- inline + "ActivationFuncCall": [False, True] + } + + +# same parameter for all solution b/c depends only on compiler +defaultBenchmarkCommonParameters = [ + {"InnerUnroll": [ 1 ] }, + {"KernelLanguage": [ "Assembly" ] }, + {"LdsPadA": [ 0 ] }, + {"LdsPadB": [ 0 ] }, + {"LdsBlockSizePerPad": [ 0 ] }, + {"TransposeLDS": [ 0 ] }, + {"MaxOccupancy": [ 40 ] }, + {"VectorWidth": [ -1 ] }, + {"VectorStore": [ -1 ] }, + {"StoreVectorWidth": [ -1 ] }, + {"GlobalReadVectorWidth": [ -1 ] }, + {"LocalReadVectorWidth": [ -1 ] }, + {"WaveSeparateGlobalReadA": [ 0 ] }, + {"WaveSeparateGlobalReadB": [ 0 ] }, + {"PrefetchGlobalRead": [ 1 ] }, + {"PrefetchLocalRead": [ 1 ] }, + {"SuppressNoLoadLoop": [ False ]}, + {"ExpandPointerSwap": [ True ]}, + + {"ScheduleGlobalRead": [ 1 ] }, + {"ScheduleLocalWrite": [ 1 ] }, + {"ScheduleIterAlg": [ 1 ] }, + + {"GlobalReadPerMfma": [ 1 ] }, + {"LocalWritePerMfma": [ -1 ] }, + + {"InterleaveAlpha": [ 0 ] }, + {"OptNoLoadLoop": [ 1 ] }, + + {"BufferLoad": [ True ] }, + {"BufferStore": [ True ] }, + {"DirectToVgprA": [ False ] }, + {"DirectToVgprB": [ False ] }, + {"DirectToLds": [ False ] }, + {"UseSgprForGRO": [ -1 ] }, + {"UseInstOffsetForGRO": [ 0 ] }, + {"AssertSummationElementMultiple": [ 1 ] }, + {"AssertFree0ElementMultiple": [ 1 ] }, + {"AssertFree1ElementMultiple": [ 1 ] }, + + {"StaggerU": [ 32 ] }, # recommend [0,32] + {"StaggerUStride": [ 256 ] }, # recommend 256 for V10,V20 + {"StaggerUMapping": [ 0 ] }, # recommend [0,1] + {"MagicDivAlg": [ 2 ] }, + {"GlobalSplitU": [ 1 ] }, + {"GlobalSplitUAlgorithm": [ "SingleBuffer" ] }, + {"Use64bShadowLimit": [ 1 ] }, + {"NumLoadsCoalescedA": [ 1 ] }, + {"NumLoadsCoalescedB": [ 1 ] }, + {"WorkGroup": [ [16,16,1]] }, + {"WorkGroupMapping": [ 8 ] }, + {"ThreadTile": [ [4,4] ] }, + {"WavefrontSize": [ 64 ]}, + {"MatrixInstruction": [ [] ] }, + {"1LDSBuffer": [ 0 ] }, + {"DepthU": [ -1 ] }, + {"DepthULdsDivisor": [ 1 ] }, + {"NonTemporalD": [ 0 ] }, + {"NonTemporalC": [ 0 ] }, + {"NonTemporalA": [ 0 ] }, + {"NonTemporalB": [ 0 ] }, + {"CustomKernelName": [ "" ] }, + {"NoReject": [ False ]}, + {"MinVgprNumber": [0]}, + {"MaxVgprNumber": [256]}, + {"StoreRemapVectorWidth": [ 0 ] }, + {"SourceSwap": [ False ] }, + {"StorePriorityOpt": [ False ] }, + {"NumElementsPerBatchStore": [ 0 ] }, + {"StoreSyncOpt": [ 0 ] }, + {"GroupLoadStore": [ False ] }, + {"MIArchVgpr": [ False ] }, + {"ActivationFused": [ True ] }, + {"ActivationFuncCall": [ True ] } + ] + +# dictionary of defaults comprised of default option for each parameter +defaultSolution = {} +for paramDict in defaultBenchmarkCommonParameters: + for key, value in paramDict.items(): + defaultSolution[key] = value[0] +# other non-benchmark options for solutions + +################################################################################ +# Default Problem Type +################################################################################ +defaultProblemType = { + # =GEMM uses TransposeA,B paramters and makes the problem type more readeable for users + # =TensorContraction requires specifying + "OperationType": "GEMM", # GEMM, TensorContraction, ConvolutionForward, ConvolutionBackwardData, ConvolutionBackwardWeights + + "DataType": 0, # data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DestDataType": 0, # destination data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "ComputeDataType": 0, # compute data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "UseBeta": True, # =True use beta parameter (asm will check for B=0 and optimize the write for that), =False don't use beta parameter + "UseBias": False, # =True use bias vector + "HighPrecisionAccumulate": False, # f32 += f16*f16 + "SilentHighPrecisionAccumulate": False, # Keep kernel names the same for HPA mode. Useful for testing. + + "ComplexConjugateA": False, # complex data should be conjugated for "C" transpose case + "ComplexConjugateB": False, + + # for OperationType == GEMM + "TransposeA": False, # =True means transA="T" or "C", =False means transA = "N" + "TransposeB": True, + "Batched": False, # add batching dimension + "StridedBatched": True, # use to select general batch or strided batch + + # for OperationType == TensorContraction + # - Indices < NumIndicesC are Free or Batch indices and appear in C and D + # - Indices which appear in both A and B, and are < NumIndicesC are batch. A and B must have same number of batch indices. + # - Indices which appear in both A and B, and are >= NumIndicesC are summation. A and B must have same number of summation indices. + # - Indices which appear in A or B (but not both), are Free. A and B may have different numbers of free indices. + # - Summation loops are nested from smallest index number to largest, with the largest summation index as the 'unroll' loop. + # - Memory order of C and D matrices is always 0..NumIndicesC-1, with 0 as the fastest-moving. + # - By choosing index assignments the output can be 'transposed'. For example if IA=[1,2] IB=[0,2] then 0 is the coalesced dim for C/D. + # - Likewise batch index may be assigned between two free indices to control the output order, ie to write in CNHW format. + # - For example : IA=[0,1,3] IB=[2,1,3]. 0,2 are free indices; 1 is batch. + "IndexAssignmentsA": [0, 2], + "IndexAssignmentsB": [1, 2], + "NumIndicesC": 2, + + # use initial strides for AB. + # This has some performance impact for the increased flexibility: + # - Additional strides will be passed into the kernel and will occupy SGPR registers + # - GlobalReadWidth must be 1 (since elements are not guaranteed to be adjacent in memory) + "UseInitialStridesAB": False, + + # use initial strides for CD. + # This has some performance impact for the increased flexibility: + # - Additional strides will be passed into the kernel and will occupy SGPR registers + # - Additional multiply on the store address path + # -VectorStore must be 0. If VectorStore is -1, it will be silently set to 0 internally. + "UseInitialStridesCD": False, + + "AllowNoFreeDims": False, # allow A or B to specify no free dims + # (if false, A and B must have at least one free dim) + # (if true, A and B must have at least one free or batch dim) + + # SetConstStride* sets the specified stride in the problem. + # These no longer generate predicates - see AssertStrideEqualA/B below + # List of pairs of [index, constValue]. + # Index is a member of the global index assignments (not an offset into IndexAssignmentsA/B) + # EX: SetConstStrideA: [ [3, 1], [2, 4] ] sets + # strideA for index3 to constant '1' and stride for index2 to constant '4'. + "SetConstStrideA": [], + "SetConstStrideB": [], + + # Summation dimension indices + "MirrorDimsA": [], + "MirrorDimsB": [], + + # for LD description + "NumIndicesLD": 4, + "IndexAssignmentsLD": [3, 4, 5, 6], # order is LDD, LDC, LDA, LDB + + # Tile aware solution selection + "TileAwareSelection": False, + + # FP16 Alternate Implementation + "Fp16AltImpl": False, + + # Activation + "Activation": False, + "ActivationHPA": False + } + +defaultProblemSizes = [{"Range": [ [2880], 0, 0 ]}] +defaultBenchmarkFinalProblemSizes = [{"Range": [ + [64, 64, 64, 512], 0, 0 ]}] +defaultBatchedProblemSizes = [{"Range": [ [2880], 0, [1], 0 ]}] +defaultBatchedBenchmarkFinalProblemSizes = [{"Range": [ + [64, 64, 64, 512], 0, [1], 0 ]}] + + +defaultSolutionSummationSizes = [32,64,96,128,256,512,1024,2048,4096,8192,16192] + + +################################################################################ +# Default Analysis Parameters +################################################################################ +defaultAnalysisParameters = { + "ScheduleName": "Tensile", + "DeviceNames": "fallback", + "ArchitectureName": "gfx000", + "SolutionImportanceMin": 0.01, # = 0.01=1% total time saved by keeping this solution + } + + +################################################################################ +# Searching Nested Lists / Dictionaries +# to see if keys exist and what their values are +################################################################################ +# param name in structures? +def inListOfDictionaries(param, dictionaries): + for dictionary in dictionaries: + if param in dictionary: + return True + return False +def inListOfListOfDictionaries(param, dictionaries): + for dictionaryList in dictionaries: + if inListOfDictionaries(param, dictionaryList): + return True + return False +def inListOfLists(param, lists): + for l in lists: + if param in l: + return True + return False + +# get param values from structures. +def hasParam( name, structure ): + if isinstance(structure, list): + for l in structure: + if hasParam(name, l): + return True + return False + elif isinstance(structure, dict): + return name in structure + else: + return name == structure + #printExit("structure %s is not list or dict" % structure) + +def getParamValues( name, structure ): + if isinstance(structure, list): + for l in structure: + param = getParamValues(name, l) + if param != None: + return param + return None + elif isinstance(structure, dict): + if name in structure: + return structure[name] + else: + return None + else: + printExit("structure %s is not list or dict" % structure) + +################################################################################ +# Print Debug +################################################################################ +def print1(message): + if globalParameters["PrintLevel"] >= 1: + print(message) + sys.stdout.flush() +def print2(message): + if globalParameters["PrintLevel"] >= 2: + print(message) + sys.stdout.flush() + +def printWarning(message): + print("Tensile::WARNING: %s" % message) + sys.stdout.flush() +def printExit(message): + print("Tensile::FATAL: %s" % message) + sys.stdout.flush() + sys.exit(-1) + +################################################################################ +# Locate Executables +# rocm-smi, hip-clang, rocm_agent_enumerator, clang-offload-bundler +################################################################################ +def isExe( filePath ): + return os.path.isfile(filePath) and os.access(filePath, os.X_OK) +def locateExe( defaultPath, exeName ): # /opt/rocm/bin, hip-clang + # look in defaultPath first + exePath = os.path.join(defaultPath, exeName) + if isExe(exePath): + return exePath + # look in PATH second + for path in os.environ["PATH"].split(os.pathsep): + exePath = os.path.join(path, exeName) + if isExe(exePath): + return exePath + return None + +def gfxArch(name): + import re + match = re.search(r'gfx([0-9a-fA-F]{3,})', name) + if not match: return None + + ipart = match.group(1) + + step = int(ipart[-1], 16) + ipart = ipart[:-1] + + minor = int(ipart[-1]) + ipart = ipart[:-1] + + major = int(ipart) + + rv = (major, minor, step) + + return rv + +def detectGlobalCurrentISA(): + """ + Returns returncode if detection failure + """ + global globalParameters + + if globalParameters["CurrentISA"] == (0,0,0) and globalParameters["ROCmAgentEnumeratorPath"]: + process = subprocess.run([globalParameters["ROCmAgentEnumeratorPath"]], stdout=subprocess.PIPE) + if os.name == "nt": + line = "" + for line_in in process.stdout.decode().splitlines(): + if 'gcnArchName' in line_in: + line += line_in.split()[1] + break # detemine if hipinfo will support multiple arch + arch = gfxArch(line.strip()) + if arch is not None: + if arch in globalParameters["SupportedISA"]: + print1("# Detected local GPU with ISA: " + getGfxName(arch)) + globalParameters["CurrentISA"] = arch + else: + archList = [] + for line in process.stdout.decode().split("\n"): + arch = gfxArch(line.strip()) + if arch is not None: + if arch in globalParameters["SupportedISA"]: + print1("# Detected local GPU with ISA: " + getGfxName(arch)) + archList.append(arch) + globalParameters["CurrentISA"] = archList[globalParameters["Device"]] + if (process.returncode): + printWarning("%s exited with code %u" % (globalParameters["ROCmAgentEnumeratorPath"], process.returncode)) + return process.returncode + return 0 + +def restoreDefaultGlobalParameters(): + """ + Restores `globalParameters` back to defaults. + """ + global globalParameters + global defaultGlobalParameters + # Can't just assign globalParameters = deepcopy(defaultGlobalParameters) because that would + # result in dangling references, specifically in Tensile.Tensile(). + globalParameters.clear() + for key, value in deepcopy(defaultGlobalParameters).items(): + globalParameters[key] = value + +def printTable(rows): + rows = list([[str(cell) for cell in row] for row in rows]) + colWidths = list([max([len(cell) for cell in col]) for col in zip(*rows)]) + + for row in rows: + for (width, cell) in zip(colWidths, row): + pad = ' ' * (width - len(cell)) + print(pad, cell, sep='', end=' ') + print() + +def printCapTable(parameters): + import itertools + archs = [(0,0,0)] + parameters["SupportedISA"] + gfxNames = list(map(getGfxName, archs)) + + headerRow = ['cap'] + gfxNames + + def capRow(caps, cap): + return [cap] + [('1' if cap in caps[arch] and caps[arch][cap] else '0') for arch in archs] + + allAsmCaps = set(itertools.chain(*[caps.keys() for arch, caps in parameters["AsmCaps"].items()])) + allAsmCaps = sorted(allAsmCaps, key=lambda k: (k.split("_")[-1], k)) + asmCapRows = [capRow(parameters["AsmCaps"], cap) for cap in allAsmCaps] + + allArchCaps = set(itertools.chain(*[caps.keys() for arch, caps in parameters["ArchCaps"].items()])) + allArchCaps = sorted(allArchCaps) + archCapRows = [capRow(parameters["ArchCaps"], cap) for cap in allArchCaps] + + printTable([headerRow] + asmCapRows + archCapRows) + +def which(p): + exes = [p+x for x in ['', '.exe', '.bat']] + system_path = os.environ['PATH'].split(os.pathsep) + if p == 'hipcc' and 'CMAKE_CXX_COMPILER' in os.environ and os.path.isfile(os.environ['CMAKE_CXX_COMPILER']): + return os.environ['CMAKE_CXX_COMPILER'] + for dirname in system_path+[globalParameters["ROCmBinPath"]]: + for exe in exes: + candidate = os.path.join(os.path.expanduser(dirname), exe) + if os.path.isfile(candidate): + return candidate + return None + +################################################################################ +################################################################################ +def assignGlobalParameters( config ): + """ + Assign Global Parameters + Each global parameter has a default parameter, and the user + can override them, those overridings happen here + """ + + global globalParameters + + # Minimum Required Version + if "MinimumRequiredVersion" in config: + if not versionIsCompatible(config["MinimumRequiredVersion"]): + printExit("Config file requires version=%s is not compatible with current Tensile version=%s" \ + % (config["MinimumRequiredVersion"], __version__) ) + + # User-specified global parameters + print2("GlobalParameters:") + for key in globalParameters: + defaultValue = globalParameters[key] + if key in config: + configValue = config[key] + if configValue == defaultValue: + print2(" %24s: %8s (same)" % (key, configValue)) + else: + print2(" %24s: %8s (overriden)" % (key, configValue)) + else: + print2(" %24s: %8s (unspecified)" % (key, defaultValue)) + + globalParameters["ROCmPath"] = "/opt/rocm" + if "ROCM_PATH" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("ROCM_PATH") + if "TENSILE_ROCM_PATH" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("TENSILE_ROCM_PATH") + if os.name == "nt" and "HIP_DIR" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("HIP_DIR") # windows has no ROCM + globalParameters["CmakeCxxCompiler"] = None + if "CMAKE_CXX_COMPILER" in os.environ: + globalParameters["CmakeCxxCompiler"] = os.environ.get("CMAKE_CXX_COMPILER") + + globalParameters["ROCmBinPath"] = os.path.join(globalParameters["ROCmPath"], "bin") + + # ROCm Agent Enumerator Path + if os.name == "nt": + globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "hipinfo.exe") + else: + globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm_agent_enumerator") + + if "CxxCompiler" in config: + globalParameters["CxxCompiler"] = config["CxxCompiler"] + + if "TENSILE_ROCM_ASSEMBLER_PATH" in os.environ: + globalParameters["AssemblerPath"] = os.environ.get("TENSILE_ROCM_ASSEMBLER_PATH") + elif globalParameters["AssemblerPath"] is None and globalParameters["CxxCompiler"] == "hipcc": + if os.name == "nt": + globalParameters["AssemblerPath"] = locateExe(globalParameters["ROCmBinPath"], "clang++.exe") + else: + globalParameters["AssemblerPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "clang++") + + globalParameters["ROCmSMIPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm-smi") + + globalParameters["ExtractKernelPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "hip/bin"), "extractkernel") + + if "TENSILE_ROCM_OFFLOAD_BUNDLER_PATH" in os.environ: + globalParameters["ClangOffloadBundlerPath"] = os.environ.get("TENSILE_ROCM_OFFLOAD_BUNDLER_PATH") + else: + if os.name == "nt": + globalParameters["ClangOffloadBundlerPath"] = locateExe(globalParameters["ROCmBinPath"], "clang-offload-bundler.exe") + else: + globalParameters["ClangOffloadBundlerPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "clang-offload-bundler") + + if "ROCmAgentEnumeratorPath" in config: + globalParameters["ROCmAgentEnumeratorPath"] = config["ROCmAgentEnumeratorPath"] + + # read current gfx version + returncode = detectGlobalCurrentISA() + if globalParameters["CurrentISA"] == (0,0,0): + printWarning("Did not detect SupportedISA: %s; cannot benchmark assembly kernels." % globalParameters["SupportedISA"]) + if returncode: + if os.name == "nt": + globalParameters["CurrentISA"] = (9,0,6) + printWarning("Failed to detect ISA so forcing (gfx906) on windows") + + # TODO Remove this when rocm-smi supports gfx90a + if globalParameters["CurrentISA"] == (9,0,10): + printWarning("HardwareMonitor currently disabled for gfx90a") + globalParameters["HardwareMonitor"] = False + + globalParameters["AsmCaps"] = {} + globalParameters["ArchCaps"] = {} + globalParameters["AsmBugs"] = {} + + for v in globalParameters["SupportedISA"] + [(0,0,0)]: + ti = TensileInstructions() + ti.init(v, globalParameters["AssemblerPath"], (globalParameters["PrintLevel"] >= 2)) + globalParameters["AsmCaps"][v] = ti.getAsmCaps() + globalParameters["ArchCaps"][v] = ti.getArchCaps() + globalParameters["AsmBugs"][v] = ti.getAsmBugs() + + if globalParameters["PrintLevel"] >= 1: + printCapTable(globalParameters) + + globalParameters["SupportedISA"] = list([i for i in globalParameters["SupportedISA"] if globalParameters["AsmCaps"][i]["SupportedISA"]]) + + validParameters["ISA"] = [(0,0,0), *globalParameters["SupportedISA"]] + + if "MergeFiles" in config and "NumMergedFiles" in config: + if not config["MergeFiles"] and config["NumMergedFiles"] > 1: + config["NumMergedFiles"] = 1 + printWarning("--num-merged-files and --no-merge-files specified, ignoring --num-merged-files") + + # For ubuntu platforms, call dpkg to grep the version of hip-clang. This check is platform specific, and in the future + # additional support for yum, dnf zypper may need to be added. On these other platforms, the default version of + # '0.0.0' will persist + + # Due to platform.linux_distribution() being deprecated, just try to run dpkg regardless. + # The alternative would be to install the `distro` package. + # See https://docs.python.org/3.7/library/platform.html#platform.linux_distribution + try: + if os.name == "nt": + compileArgs = ['perl'] + [which('hipcc')] + ['--version'] + output = subprocess.run(compileArgs, check=True, stdout=subprocess.PIPE).stdout.decode() + else: + compiler = "hipcc" + output = subprocess.run([compiler, "--version"], check=True, stdout=subprocess.PIPE).stdout.decode() + + for line in output.split('\n'): + if 'HIP version' in line: + globalParameters['HipClangVersion'] = line.split()[2] + print1("# Found hipcc version " + globalParameters['HipClangVersion']) + + except (subprocess.CalledProcessError, OSError) as e: + printWarning("Error: {} running {} {} ".format('hipcc', '--version', e)) + + for key in config: + value = config[key] + if key not in globalParameters: + printWarning("Global parameter %s = %s unrecognised." % ( key, value )) + globalParameters[key] = value + +def setupRestoreClocks(): + import atexit + def restoreClocks(): + if globalParameters["PinClocks"]: + rsmi = globalParameters["ROCmSMIPath"] + subprocess.call([rsmi, "-d", "0", "--resetclocks"]) + subprocess.call([rsmi, "-d", "0", "--setfan", "50"]) + atexit.register(restoreClocks) +setupRestoreClocks() + +################################################################################ +# Assign Parameters +# populate dst with src[key] else give it the default/backup value +################################################################################ +def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, \ + defaultDictionary): + if key in sourceDictionary: + destinationDictionary[key] = deepcopy(sourceDictionary[key]) + else: + destinationDictionary[key] = deepcopy(defaultDictionary[key]) + +################################################################################ +# Push / Pop Working Path +# store a WorkingPath where to write files (like benchmark files) +################################################################################ +def pushWorkingPath( foldername ): + # Warning: this is not thread-safe, modifies the global WorkingPath! + globalParameters["WorkingPath"] = \ + os.path.join(globalParameters["WorkingPath"], foldername ) + return ensurePath( globalParameters["WorkingPath"] ) +def popWorkingPath(): + # Warning: this is not thread-safe, modifies the global WorkingPath! + if len(workingDirectoryStack) == 0: + globalParameters["WorkingPath"] = \ + os.path.split(globalParameters["WorkingPath"])[0] + else: + globalParameters["WorkingPath"] = workingDirectoryStack.pop() +def ensurePath(path): + try: + os.makedirs(path) + except FileExistsError: + pass + except OSError: + printExit("Failed to create directory \"%s\" " % (path) ) + return path +def setWorkingPath( fullPathName ): + # Warning: this is not thread-safe, modifies the global WorkingPath! + workingDirectoryStack.append(globalParameters["WorkingPath"]) + globalParameters["WorkingPath"] = ensurePath(fullPathName) + + +def roundUp(f): + return (int)(math.ceil(f)) + +################################################################################ +# Is query version compatible with current version +# a yaml file is compatible with tensile if +# tensile.major == yaml.major and tensile.minor.step > yaml.minor.step +################################################################################ +def versionIsCompatible(queryVersionString): + (qMajor, qMinor, qStep) = queryVersionString.split(".") + (tMajor, tMinor, tStep) = __version__.split(".") + + # major version must match exactly + if qMajor != tMajor: + return False + + # minor.patch version must be >= + if int(qMinor) > int(tMinor): + return False + if qMinor == tMinor: + if int(qStep) > int(tStep): + return False + return True + +def ClientExecutionLock(): + if not globalParameters["ClientExecutionLockPath"]: + return open(os.devnull) + + import filelock + return filelock.FileLock(globalParameters["ClientExecutionLockPath"]) + +# convert python list to C++ initializer style syntax +def listToInitializer(l): + return "{" + ','.join(map(str, l)) + "}" + +################################################################################ +# Progress Bar Printing +# prints "||||" up to width +################################################################################ +class ProgressBar: + def __init__(self, maxValue, width=80): + self.char = '|' + self.maxValue = maxValue + self.width = width + self.maxTicks = self.width - 7 + + + self.priorValue = 0 + self.fraction = 0 + self.numTicks = 0 + self.createTime = time.time() + + def increment(self, value=1): + self.update(self.priorValue+value) + + def update(self, value): + currentFraction = 1.0 * value / self.maxValue + currentNumTicks = int(currentFraction * self.maxTicks) + if currentNumTicks > self.numTicks: + self.numTicks = currentNumTicks + self.fraction = currentFraction + self.printStatus() + self.priorValue = value + + def printStatus(self): + sys.stdout.write("\r") + sys.stdout.write("[%-*s] %3d%%" \ + % (self.maxTicks, self.char*self.numTicks, self.fraction*100) ) + if self.numTicks == self.maxTicks: + stopTime = time.time() + sys.stdout.write(" (%-.1f secs elapsed)\n"%(stopTime-self.createTime)) + sys.stdout.flush() + + def finish(self): pass + +from copy import copy +class Backup: + """RAII class to restore backed up fields from object""" + fields = {} + object = None + def __init__(self, object, **fields): + self.object = object + for k, v in fields.items(): + self.fields[k] = copy(v) + def __del__(self): + for k, v in self.fields.items(): + setattr(self.object, k, v) + +# Append copyrights to all files generated by tensile since they belong to Tensile intellectual property +CMakeHeader = """################################################################################ +# Copyright (C) 2016-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +################################################### +# This file was generated by Tensile: # +# https://github.com/ROCmSoftwarePlatform/Tensile # +################################################### + + +""" + +CHeader = """/******************************************************************************* +* Copyright (C) 2016-2021 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCmSoftwarePlatform/Tensile * +**************************************************/ + + +""" + +HR = "################################################################################" diff --git a/tensilelite/Tensile/Component.py b/tensilelite/Tensile/Component.py new file mode 100644 index 0000000000..fffe3be879 --- /dev/null +++ b/tensilelite/Tensile/Component.py @@ -0,0 +1,260 @@ +################################################################################ +# Copyright 2020-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +""" +A component is a piece of code that is chosen among compatible components based on the current hardware capabilities and/or kernel options. + +The class hierarchy is automatically used to detect which type of component a given class belongs to. For example, all the MAC components should inherit from the MAC class. + +Most components should be able to get away with defining their requirements via the class properties (e.g.): + +```python + class FMA_NonPacked(MAC): + asmCaps = {"v_fma_f16": True, + "v_pk_fma_f16": False} + #archCaps = {} + kernel = {"ProblemType": {"DataType": DataType(DataType.half), + "HighPrecisionAccumulate": False}} +``` + +Values in the dictionaries can be lambdas for more advanced logic: + +```python + class FMA_HPA_MAD_MIX(MAC): + asmCaps = {"v_mad_mix_f32": True} + #archCaps = {} + kernel = {"ProblemType": {"DataType": DataType(DataType.half), + "HighPrecisionAccumulate": True}, + } +``` + +Any more advanced logic should be implemented by overriding the matches() method. + +Components are found by calling `Component..find(writer)` where `writer` is a `KernelWriter` object: + +```python + component = Component.MAC.find(self) + if component: + return component(self, m, innerUnroll) + + # No component was found, fall back to existing code... +``` + +With this fallback mechanism, components can be added one at a time, without disrupting existing code. + +Components can be categorized in different files in the `Tensile/Components` directory. Each file should be listed in the `__all__` member of `Tensile/Components/__init__.py`. +""" + +import abc +from collections.abc import Mapping +import inspect + +def PartialMatch(pattern, obj, debug=False, level=0): + indent = " " * level + if debug and level == 0: + print("") + if hasattr(pattern, "__call__"): + if not pattern(obj): + if debug: + print("{indent}call({obj}) == False".format(indent=indent, obj=obj)) + return False + elif isinstance(pattern, Mapping) and \ + isinstance(obj, Mapping): + for key, value in pattern.items(): + if key not in obj: + if debug: + print("{indent}{key} not in object.".format(indent=indent, key=key)) + return False + + if debug: + print("{indent} recursing into {key}".format(indent=indent, key=key)) + if not PartialMatch(value, obj[key], debug, level+1): + return False + + elif pattern != obj: + if debug: + print("{indent}{pattern} != {obj}".format(indent=indent, pattern=pattern, obj=obj)) + return False + + if debug: + print("{indent}: True".format(indent=indent)) + return True + +class ComponentMeta(abc.ABCMeta): + """ + Metaclass which auto-registers each subclass in an "implementations" + member of its parent class, to allow for hierarchical searching. + """ + def __init__(cls, name, bases, namespace, **kwargs): + if inspect.isabstract(cls): + cls.implementations = {} + + for base in bases: + base.implementations[name] = cls + setattr(base, name, cls) + +class Component(metaclass=ComponentMeta): + """ + Modular component which allows kernel components to be specified based on + capability rather than based on individual architecture IDs. + """ + + @classmethod + def matches(cls, writer, debug=False): + if hasattr(cls, "versions"): + if not writer.version in cls.versions: + return False + + attrs = ["asmCaps", "archCaps", "kernel"] + for attr in attrs: + if hasattr(cls, attr): + if not PartialMatch(getattr(cls, attr), getattr(writer, attr), debug): + return False + + return True + + @classmethod + def findAll(cls, writer, debug=False, *args, **kwargs): + found = [] + for name, impl in cls.implementations.items(): + if debug: + print(name) + if inspect.isabstract(impl): + foundHere = impl.findAll(writer, debug, *args, **kwargs) + if debug: + print(name, ": found ", foundHere) + found += foundHere + elif impl.matches(writer, debug, *args, **kwargs): + if debug: + print(name, ": found impl") + found.append(impl) + else: + if debug: + print(name, "mismatch!") + + return found + + @classmethod + def find(cls, writer, debug=False, *args, **kwargs): + found = cls.findAll(writer, debug, *args, **kwargs) + + if len(found) == 0: + return None + + if len(found) > 1: + raise RuntimeError("Found {} implementations for {}".format(len(found), cls.__name__)) + + return found[0]() + + @classmethod + def componentPath(cls, path=None, bases=None): + if path is None: + path = [] + + if bases is None: + bases = cls.__bases__ + + if not isinstance(cls, str): + className = cls.__name__ + path = [className] + path + + if cls == Component or len(bases) == 0: + return path + return bases[0].componentPath(path) + + @abc.abstractmethod + def __call__(self): + """ + Concrete subclasses must implement __call__. + """ + pass + + def commentHeader(self): + """ + Returns a comment which helps identify where a piece of code was generated. + """ + return "{}".format('.'.join(self.componentPath())) + +class MAC(Component): + """ + Multiply-accumulate block. + """ + pass + +class Signature(Component): + """ + Function signature block. + """ + pass + +class LocalRead(Component): + """ + Local read block. + """ + pass + +class ShiftVectorComponents(Component): + """ + Shift vector components block. + """ + pass + +class ComputeStoreVgprs(Component): + """ + Compute store vgprs block. + """ + pass + +class NotLocalFullTileElements(Component): + """ + Not local full tile elements block. + """ + pass + +class LraTileAssignment(Component): + """ + Lra tile assignment block. + """ + pass + +class PackData(Component): + """ + Pack data block. + """ + pass + +class SIA(Component): + """ + ScheduleIterAlg block. + """ + @abc.abstractmethod + def schedIntoIteration(self, writer, kernel, tensorParametersA, tensorParametersB, \ + localWriteEndIter, uDu, firstIter, lastLoop, lastLc, maxVmcnt, globalReadIncACode, \ + globalReadIncBCode): + pass + +class GlobalWriteComponents(Component): + pass + +# Importing here allows auto-registry of components in the Components directory. +# Each file must be listed in __all__ in Components/__init__.py +# "noqa" prevents linter from complaining here. +from .Components import * # noqa diff --git a/tensilelite/Tensile/Components/ComputeStoreVgprs.py b/tensilelite/Tensile/Components/ComputeStoreVgprs.py new file mode 100644 index 0000000000..86d3365d03 --- /dev/null +++ b/tensilelite/Tensile/Components/ComputeStoreVgprs.py @@ -0,0 +1,231 @@ +################################################################################ +# Copyright 2021-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import Module, SMulI32, VAddLShiftLeftU32, VAddU32, VMulLOU32, \ + staticMultiply, vectorStaticDivide, vectorStaticRemainder, \ + RegisterPoolResource, vgpr, sgpr, log2 +from ..Component import ComputeStoreVgprs + +class ComputeStoreVgprsMFMA(ComputeStoreVgprs): + kernel = {"EnableMatrixInstruction": True, + "SourceSwap": False} + + """ + computeStoreVgprs + Compute workitem/TT offsets in VGPRS + and coord0/coord1 + tid0Scale specifies the number of output elements in 0/coalesced dim + that should be written by each work-item in each batch element. + """ + def __call__(self, writer, kernel, divisor, tid0Scale, tid1Scale): + + # writer.coord0 + # writer.coord1 + # writer.vgprs.cinRowPtr : C buffer coulmn offset + # writer.vgprs.coutRowPtr : D buffer coulmn offset + + # alloc resources + tid0 = writer.vgprPool.checkOut(1) + tid1 = writer.vgprPool.checkOut(1) + if kernel["BufferStore"]: + writer.vgprs.cinRowPtr = writer.vgprPool.checkOut(1, "cinRowPtr") + writer.vgprs.coutRowPtr = writer.vgprPool.checkOut(1, "coutRowPtr") + + wave_id = writer.vgprPool.checkOut(1) + + tmpVgpr0 = writer.vgprPool.checkOut(1,"tmpVgpr0") + tmpVgpr1 = writer.vgprPool.checkOutAligned(2,2,"tmpVgpr1") + tmpVgpr0Res = RegisterPoolResource(tmpVgpr0, 1) + tmpVgpr1Res = RegisterPoolResource(tmpVgpr1, 2) + dummy = writer.vgprPool.checkOut(1,"dummy") + + with writer.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + + # constant + MIBShape0 = kernel["MatrixInstM"] * kernel["MatrixInstBM"] + MIBShape1 = kernel["MatrixInstN"] * kernel["MatrixInstBN"] + + # matrixInstM = kernel["MatrixInstM"] * kernel["MatrixInstBM"] if (kernel["MatrixInstM"] == 4) else kernel["MatrixInstM"] + matrixInstN = kernel["MatrixInstN"] * kernel["MatrixInstBN"] if (kernel["MatrixInstN"] == 4) else kernel["MatrixInstN"] + + module = Module("ComputeStoreVgprsMFMA") + + # coord 1 : wave part + module.add(vectorStaticDivide(wave_id, "Serial", writer.kernel["WavefrontSize"], tmpVgpr1Res)) + module.add(vectorStaticDivide(tid1, wave_id, kernel["MIWaveGroup"][0], tmpVgpr1Res)) + module.add(VMulLOU32(dst=vgpr(tid1), src0=hex(MIBShape1), src1=vgpr(tid1), comment="wave coordination offset 1")) + + # coord 1 : thread part + module.add(vectorStaticRemainder(dummy, tmpVgpr0, "Serial", matrixInstN, tmpVgpr1Res, tmpSgprInfo)) + module.add(VAddU32(dst=vgpr(tid1), src0=vgpr(tmpVgpr0), src1=vgpr(tid1), comment="coordination 1 = wave_id1 + tid1")) + + # coord 1 : offset part + packedC1 = kernel["PackedC1IndicesX"] + strideC1 = "StrideC%s" % (writer.indexChars[packedC1[0]]) + strideD1 = "StrideD%s" % (writer.indexChars[packedC1[0]]) + module.add(VMulLOU32(dst=vgpr(writer.vgprs.cinRowPtr), src0=vgpr(tid1), src1=sgpr(strideC1), comment=" offset 1")) + module.add(VMulLOU32(dst=vgpr(writer.vgprs.coutRowPtr), src0=vgpr(tid1), src1=sgpr(strideD1), comment=" offset 1")) + + # coord 0 : wave part + module.add(vectorStaticRemainder(dummy, tmpVgpr0, wave_id, kernel["MIWaveGroup"][0], tmpVgpr1Res, tmpSgprInfo)) + module.add(VMulLOU32(dst=vgpr(tmpVgpr0), src0=hex(MIBShape0), src1=vgpr(tmpVgpr0), comment="wave coordination offset 0")) + + # coord 0 : thread part + module.add(vectorStaticRemainder(dummy, tid0, "Serial", writer.kernel["WavefrontSize"], tmpVgpr1Res, tmpSgprInfo)) + module.add(vectorStaticDivide(tid0, tid0, matrixInstN, tmpVgpr1Res)) + module.add(staticMultiply(vgpr(tid0), vgpr(tid0), kernel["MIOutputVectorWidth"], tmpSgprInfo, "thread0 * continuous_output")) + module.add(VAddU32(dst=vgpr(tid0), src0=vgpr(tmpVgpr0), src1=vgpr(tid0), comment="coordination 0 = wave_id0 + tid0")) + + wg0="WorkGroup0" + wg1="WorkGroup1" + + # macro tile 0 part + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=kernel["MacroTile0"], src1=sgpr(wg0), comment="wgp0 * MT0")) + module.add(VAddU32(dst=vgpr(tid0), src0=sgpr(tmpSgpr), src1=vgpr(tid0), comment="coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0")) + + # macro tile 1 part + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=kernel["MacroTile1"], src1=sgpr(wg1), comment="wgp1 * MT1")) + module.add(VAddU32(dst=vgpr(tid1), src0=sgpr(tmpSgpr), src1=vgpr(tid1), comment="coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1")) + + # extract packed rowStart vgpr + if len(packedC1) > 1: + module.add(writer.extractPackedCoord1ToRowStart(kernel, packedC1, tid1, 'D')) + + # release resource + writer.vgprPool.checkIn(dummy) + writer.vgprPool.checkIn(tmpVgpr1) + writer.vgprPool.checkIn(tmpVgpr0) + writer.vgprPool.checkIn(wave_id) + + # StoreRemap: calculate + # 1. local read address + # 2. local write address + # 3. global write coord0 and coord1 + if kernel["StoreRemapVectorWidth"]: + module.add(writer.storeRemapComputeStoreVgprs(kernel)) + + writer.vgprs.coord0 = tid0 + writer.vgprs.coord1 = tid1 + + return module + +class ComputeStoreVgprsMFMASwap(ComputeStoreVgprs): + kernel = {"EnableMatrixInstruction": True, + "SourceSwap": True} + + """ + computeStoreVgprs + Compute workitem/TT offsets in VGPRS + and coord0/coord1 + tid0Scale specifies the number of output elements in 0/coalesced dim + that should be written by each work-item in each batch element. + """ + def __call__(self, writer, kernel, divisor, tid0Scale, tid1Scale): + + # writer.coord0 + # writer.coord1 + # writer.vgprs.cinRowPtr : C buffer coulmn offset + # writer.vgprs.coutRowPtr : D buffer coulmn offset + + # alloc resources + tid0 = writer.vgprPool.checkOut(1) + tid1 = writer.vgprPool.checkOut(1) + if kernel["BufferStore"]: + writer.vgprs.cinRowPtr = writer.vgprPool.checkOut(1, "cinRowPtr") + writer.vgprs.coutRowPtr = writer.vgprPool.checkOut(1, "coutRowPtr") + + wave_id = writer.vgprPool.checkOut(1) + + tmpVgpr0 = writer.vgprPool.checkOut(1,"tmpVgpr0") + tmpVgpr1 = writer.vgprPool.checkOutAligned(2,2,"tmpVgpr1") + tmpVgpr1Res = RegisterPoolResource(tmpVgpr1, 2) + dummy = writer.vgprPool.checkOut(1,"dummy") + + with writer.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + + # constant + MIBShape0 = kernel["MatrixInstM"] * kernel["MatrixInstBM"] + MIBShape1 = kernel["MatrixInstN"] * kernel["MatrixInstBN"] + + matrixInstM = kernel["MatrixInstM"] * kernel["MatrixInstBM"] if (kernel["MatrixInstM"] == 4) else kernel["MatrixInstM"] + # matrixInstN = kernel["MatrixInstN"] * kernel["MatrixInstBN"] if (kernel["MatrixInstN"] == 4) else kernel["MatrixInstN"] + + module = Module("ComputeStoreVgprsMFMASwap") + + module.add(vectorStaticDivide(wave_id, "Serial", writer.kernel["WavefrontSize"], tmpVgpr1Res)) + + # coord 1 : wave part + module.add(vectorStaticDivide(tmpVgpr0, wave_id, kernel["MIWaveGroup"][0], tmpVgpr1Res)) + module.add(VMulLOU32(dst=vgpr(tmpVgpr0), src0=hex(MIBShape1), src1=vgpr(tmpVgpr0), comment="wave coordination offset 1")) + + # coord 1 : thread part + module.add(vectorStaticRemainder(dummy, tid1, "Serial", writer.kernel["WavefrontSize"], tmpVgpr1Res, tmpSgprInfo)) + module.add(vectorStaticDivide(tid1, tid1, matrixInstM, tmpVgpr1Res)) + module.add(staticMultiply(vgpr(tid1), vgpr(tid1), kernel["MIOutputVectorWidth"], tmpSgprInfo, "thread0 * continuous_output")) + module.add(VAddU32(dst=vgpr(tid1), src0=vgpr(tmpVgpr0), src1=vgpr(tid1), comment="coordination 1 = wave_id1 + tid1")) + if kernel["allowLRVWforTLUandMI"] and writer.lrvwB > 1: + module.add(staticMultiply(vgpr(tid1), vgpr(tid1), writer.lrvwB, tmpSgprInfo, "coordination 1 *= lrvwB")) + + # coord 1 : offset part + packedC1 = kernel["PackedC1IndicesX"] + strideC1 = "StrideC%s" % (writer.indexChars[packedC1[0]]) + strideD1 = "StrideD%s" % (writer.indexChars[packedC1[0]]) + module.add(VMulLOU32(dst=vgpr(writer.vgprs.cinRowPtr), src0=vgpr(tid1), src1=sgpr(strideC1), comment=" offset 1")) + module.add(VMulLOU32(dst=vgpr(writer.vgprs.coutRowPtr), src0=vgpr(tid1), src1=sgpr(strideD1), comment=" offset 1")) + + # coord 0 : wave part + module.add(vectorStaticRemainder(dummy, tid0, wave_id, kernel["MIWaveGroup"][0], tmpVgpr1Res, tmpSgprInfo)) + module.add(VMulLOU32(dst=vgpr(tid0), src0=hex(MIBShape0), src1=vgpr(tid0), comment="wave coordination offset 0")) + + # coord 0 : thread part + module.add(vectorStaticRemainder(dummy, tmpVgpr0, "Serial", matrixInstM, tmpVgpr1Res, tmpSgprInfo)) + module.add(VAddLShiftLeftU32(dst=vgpr(tid0), src0=vgpr(tmpVgpr0), src1=vgpr(tid0), shiftHex=log2(kernel["VectorWidth"]), comment="coordination 0 = wave_id0 + tid0")) + + wg0="WorkGroup0" + wg1="WorkGroup1" + + # macro tile 0 part + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=kernel["MacroTile0"], src1=sgpr(wg0), comment="wgp0 * MT0")) + module.add(VAddU32(dst=vgpr(tid0), src0=sgpr(tmpSgpr), src1=vgpr(tid0), comment="coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0")) + + # macro tile 1 part + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=kernel["MacroTile1"], src1=sgpr(wg1), comment="wgp1 * MT1")) + module.add(VAddU32(dst=vgpr(tid1), src0=sgpr(tmpSgpr), src1=vgpr(tid1), comment="coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1")) + + # release resource + writer.vgprPool.checkIn(dummy) + writer.vgprPool.checkIn(tmpVgpr1) + writer.vgprPool.checkIn(tmpVgpr0) + writer.vgprPool.checkIn(wave_id) + + # StoreRemap: calculate + # 1. local read address + # 2. local write address + # 3. global write coord0 and coord1 + if kernel["StoreRemapVectorWidth"]: + module.add(writer.storeRemapComputeStoreVgprs(kernel)) + + writer.vgprs.coord0 = tid0 + writer.vgprs.coord1 = tid1 + + return module diff --git a/tensilelite/Tensile/Components/GlobalWriteBatch.py b/tensilelite/Tensile/Components/GlobalWriteBatch.py new file mode 100644 index 0000000000..ca98d26ba2 --- /dev/null +++ b/tensilelite/Tensile/Components/GlobalWriteBatch.py @@ -0,0 +1,1143 @@ +################################################################################ +# Copyright 2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..Common import globalParameters +from ..Component import GlobalWriteComponents +from ..SolutionStructs import Solution +from ..Activation import ActivationModule, ActivationType +from ..AsmStoreState import StoreState +from ..TensileInstructions import Label, Module, EXEC, SDWAModifiers, VCC, SelectBit, \ + vgpr, sgpr, replaceHolder, SaturateCastType, VCvtBF16toFP32, \ + DataType +from ..TensileInstructions.Instructions import * +from ..AsmAddressCalculation import AddrCalculation + +from math import ceil + +class GlobalWriteBatchComponent(GlobalWriteComponents): + kernel = {"ProblemType": {"OperationType": "GEMM" }} + def __call__(self, kernel: Solution, tPA, tPB, activation: ActivationModule, ss: StoreState, \ + batchIdx, applyAlpha, beta, edge, atomic, gwvw, atomicW, \ + batchElements, addrD, addrC, addrBias, \ + tmpVgpr, bf16CVTVgprStruct, activationSetPCStruct, batchElementSgprs, tmpSgpr, codeAccVgprRead, codeMulAlpha, \ + packdata, parentWriter) -> Module: + return GlobalWriteBatchWriter(kernel, tPA, tPB, activation, ss, batchIdx, applyAlpha, \ + beta, edge, atomic, gwvw, atomicW, \ + batchElements, addrD, addrC, addrBias, \ + tmpVgpr, bf16CVTVgprStruct, activationSetPCStruct, batchElementSgprs, tmpSgpr, codeAccVgprRead, codeMulAlpha, \ + packdata, parentWriter).emit() + +class GlobalWriteBatchWriter: + def __init__(self, kernel: Solution, tPA, tPB, activation: ActivationModule, ss: StoreState, \ + batchIdx, applyAlpha, beta, edge, atomic, gwvw, atomicW, \ + batchElements, addrD, addrC, addrBias, \ + tmpVgpr, bf16CVTVgprStruct, activationSetPCStruct, batchElementSgprs, tmpSgpr, codeAccVgprRead, codeMulAlpha, \ + packdata, parentWriter): + self.kernel = kernel + self.tPA = tPA + self.tPB = tPB + self.activation = activation + self.ss = ss + self.batchIdx = batchIdx + self.applyAlpha = applyAlpha + self.beta = beta + self.edge = edge + self.atomic = atomic + self.gwvw = gwvw + self.atomicW = atomicW + self.batchElements = batchElements + self.addrD = addrD + self.addrC = addrC + self.addrBias = addrBias + self.activationSetPCStruct = activationSetPCStruct + self.tmpVgpr = tmpVgpr + self.bf16CVTVgprStruct = bf16CVTVgprStruct + self.batchElementSgprs = batchElementSgprs + self.tmpSgpr = tmpSgpr + self.codeAccVgprRead = codeAccVgprRead + self.codeMulAlpha = codeMulAlpha + self.packdata = packdata + self.parentWriter = parentWriter + self.loadsIssued = 0 + self.storesIssued = 0 + + @property + def useAtomicAdd(self) -> bool: + return self.parentWriter.asmCaps["HasAtomicAdd"] and self.kernel["ProblemType"]["ComputeDataType"].isSingle() and \ + self.kernel["_GlobalAccumulation"] == "SingleBuffer" + + @property + def wavelen(self) -> int: + return self.kernel["WavefrontSize"] + + @property + def laneSGPRC(self) -> int: + return self.parentWriter.laneSGPRCount + + @property + def tmpS01(self): + return self.tmpSgpr + + @property + def tmpS23(self): + return self.tmpS01 + self.laneSGPRC + + @property + def debugConfig(self): + return self.parentWriter.db + + @property + def archCaps(self): + return self.parentWriter.archCaps + + @property + def computeDataType(self) -> DataType: + return self.kernel["ProblemType"]["ComputeDataType"] + + @property + def destDataType(self) -> DataType: + return self.kernel["ProblemType"]["DestDataType"] + + @property + def moduleName(self): + return "globalWriteBatch (Atomic)" if self.atomic else "globalWriteBatch (Non atomic)" + + def getEdgeMovInstType(self): + return SMovB32 if self.wavelen == 32 else SMovB64 + + def getEdgeOrInstType(self): + return SOrB32 if self.wavelen == 32 else SOrB64 + + def getEdgeAndInstType(self): + return SAndB32 if self.wavelen == 32 else SAndB64 + + def getSOrSaveExecType(self): + return SOrSaveExecB32 if self.wavelen == 32 else SOrSaveExecB64 + + def emit(self) -> Module: + assert self._checkAtomicPreconditions() + module = Module(self.moduleName) + self._prolog(module) + self._emitAdd(module) + self._epilog(module) + return module + + ############################################################################## + # choose the ADD instruction for combining external C with internal C + # used in atomic=1 case to compute expected external data + ############################################################################## + def _chooseAddForAtomic(self, kernel, dst, src0, src1, comment): + module = Module("chooseAddForAtomic") + if kernel["ProblemType"]["DataType"].isBFloat16(): + if kernel["_GlobalAccumulation"]: + module.add(VAddF32(dst, src0, src1, comment=comment)) + elif kernel["ProblemType"]["DataType"].isHalf(): + if kernel["_GlobalAccumulation"]: + module.add(VAddF32(dst, src0, src1, comment=comment)) + elif kernel["ProblemType"]["HighPrecisionAccumulate"]: + if self.parentWriter.asmCaps["v_fma_mix_f32"]: + module.add(VFmaMixF32(dst, src0, 1, src1, comment=comment)) + elif self.parentWriter.asmCaps["v_mad_mix_f32"]: + module.add(VMadMixF32(dst, src0, 1, src1, comment=comment)) + else: + assert False, "No valid v_mad_mix_f32 equivalent" + else: + module.add(VAddPKF16(dst, src0, src1, comment)) + elif kernel["ProblemType"]["DataType"].isInt8x4() or kernel["ProblemType"]["DataType"].isInt8(): + # assume v_add_i32 can be used in place of v_add_f32 + # need to add saturation directive to v_add_i32 instruction to clamp integer arithmetic + module.add(VAddI32(dst, src0, src1, comment=comment)) + elif kernel["ProblemType"]["DataType"].isSingle(): + module.add(VAddF32(dst, src0, src1, comment=comment)) + else: + #support for double + module.add(VAddF64(dst, src0, src1, comment=comment)) + + return module + + def _prolog(self, module: Module): + module.addComment0("optSingleColVgpr=%u optSharedColVgpr=%u optSGPRUsage=%s optSrdIncForRow=%u" % \ + (self.ss.optSingleColVgpr, self.ss.optSharedColVgpr, self.ss.optSGPRUsage, self.ss.optSrdIncForRow)) + + if self.kernel["StoreSyncOpt"]: + self._storeSyncOpt(module) + + # comment tt1, tt0, vc1, vc0 + # tt = thread tile, vc=vector component + commentStr = "Global Write%s%s Batch #%u (d1,d0,vc1,vc0) =\n " \ + % (" Beta" if self.beta else "", " Edge" if self.edge else "", self.batchIdx) + + commentStr = ''.join([commentStr] \ + + ["(%u,%u,%u,%u:vw%u%s)%s" % \ + (element[0], element[1], element[2], element[3], self.gwvw, + ":vaw:%u"%self.atomicW if self.atomic else "", + "" if idx == len(self.batchElements) -1 else "; ") + for idx, element in enumerate(self.batchElements)]) + module.addComment2(commentStr) + + self.ss.setupStoreElementsForBatch(self.kernel, self.gwvw, self.batchElements, self.batchElementSgprs, isOptNLL=False, \ + allowLRVWforTLUandMI=self.kernel["allowLRVWforTLUandMI"], lrvwB=self.parentWriter.lrvwB) + + self.loadsIssued = 0 + self.storesIssued = 0 + + ######################################## + # calculate addr and masks + module.addComment1("calc coords, apply mask, and issue loads (if necessary)") + # On input, coord0 and coord1 are VGPRs computed in the pre-batch code, based + # on the thread and tid number. These are ELEMENT offsets from start of tensor C + # for the top-left corner this thread will write. These are not changed + # across all the store loop iters. + if self.debugConfig["ConservativeWaitCnt"] & 0x10: + module.add(SBarrier("debug")) + module.add(SWaitCnt(vmcnt=0, comment="ConservativeWaitCnt")) + if self.archCaps["SeparateVscnt"]: + module.add(SWaitCnt(vscnt=0, comment="writes")) + module.add(SBarrier("debug")) + if not self.edge and self.debugConfig["ForceEdgeStores"] >= 2: + module.add(self.parentWriter.getBomb()) # should not get here + if self.edge and self.debugConfig["AssertNoEdge"]: + module.add(self.parentWriter.getBomb()) # should not get here + + ######################################## + # rC *= alpha + if not self.kernel["InterleaveAlpha"] and self.applyAlpha and self.parentWriter.alphaBeforeLoadC: + module.addComment1("rC *= alpha batchElements=%s"%self.batchElements) + if self.codeMulAlpha is None: + for elementIdx in range(len(self.batchElements)): + module.add(self._applyAlpha(self.kernel, self.gwvw, self.ss.elementSumIdx, elementIdx, self.tmpS01)) + else: + regsPerScalar = self.parentWriter.bpeCinternal // self.parentWriter.bpr # register per scalar + for elementIdx in range(len(self.batchElements)): + for vi in range(self.gwvw): + module.add(replaceHolder(self.codeMulAlpha.items().pop(0), self.ss.elementSumIdx[elementIdx]*regsPerScalar + regsPerScalar*vi)) + + loadInputCode = Module("loadInputCode") + + self.biasLoadIssued = [] + loadedDataBias = {} + for elementIdx, element in enumerate(self.batchElements): + addrCalc: AddrCalculation = self.ss.elementAddr[elementIdx] + addrCVgpr = addrCalc.addrCVgpr + addrDVgpr = addrCalc.addrDVgpr + addrBiasVgpr = addrCalc.addrBiasVgpr + data = self.ss.elementData[elementIdx] + dataBias = self.ss.elementDataBias[elementIdx] + mask = self.ss.elementMask[elementIdx] + vc0 = element[3] + + module.add(addrCalc.emitAddressSetupCode(self.kernel, self.tPB, self.ss, self.tmpVgpr, self.tmpS01, self.edge, self.beta, self.atomic, elementIdx, addrDVgpr)) + + if self.edge: + module.add(addrCalc.edgeProtectCode(self.kernel, self.edge, self.beta, self.atomic, mask, self.tmpSgpr)) + + # create code Module to push mov vgpr,acc instructions + if self.beta: + module.add(addrCalc.emitLdChange(self.kernel, self.ss, 'C', self.edge, self.beta, mask, (elementIdx == 0), self.tmpVgpr, addrCVgpr, self.addrC)) + if self.kernel["GroupLoadStore"]: + loadInputCode.add(self.parentWriter.readCInput(self.kernel, self.ss, addrCalc, vc0, data, self.gwvw, addrCVgpr, self.tmpS01)) + else: + module.add(self.parentWriter.readCInput(self.kernel, self.ss, addrCalc, vc0, data, self.gwvw, addrCVgpr, self.tmpS01)) + self.loadsIssued += 1 + if self.kernel["ProblemType"]["UseBias"]: + module.add(addrCalc.emitLdChange(self.kernel, self.ss, 'Bias', self.edge, self.beta, mask, (elementIdx == 0), self.tmpVgpr, addrBiasVgpr, self.addrBias)) + if dataBias not in loadedDataBias: + # Shift right several vgprs for cvt ops if needed + numVgprs = int(ceil(self.kernel["ProblemType"]["DataType"].numRegisters() * self.ss.cfg.gwvw)) + reg = self.kernel["ProblemType"]["DataType"].numRegisters() if self.kernel["ProblemType"]["DataType"].numRegisters() >= 1 else 1 + gprShiftBias = dataBias + (self.ss.cfg.gwvw * reg - numVgprs) + if self.kernel["GroupLoadStore"]: + # Group bias load with C input to + loadInputCode.add(self.parentWriter.addBiasLoad(self.kernel, self.ss, addrCalc, gprShiftBias)) + else: + module.add(self.parentWriter.addBiasLoad(self.kernel, self.ss, addrCalc, gprShiftBias)) + loadedDataBias[dataBias] = 1 + self.loadsIssued += 1 + self.biasLoadIssued.append(len(loadedDataBias)) + + module.add(addrCalc.emitLdChange(self.kernel, self.ss, 'D', self.edge, self.beta, mask, (elementIdx == len(self.batchElements) - 1), self.tmpVgpr, addrDVgpr, self.addrD)) + + if self.atomic and (not self.useAtomicAdd): + # load c into data+1 because of CAS structure + # TODO - Fix for double here, would need bigger load + # FIXME + # gwvw is the number of elements in the batch + # iterate over number of atomic operations to perform, each of width atomicW + for avi in range(self.gwvw // self.atomicW): + dataV = self.ss.elementData[elementIdx] + int(avi*self.ss.cfg.numVgprsPerDataPerVI) + bpm = self.parentWriter.bpeCexternal * self.atomicW + useBuffer = self.kernel["BufferStore"] + if self.kernel["BufferStore"]: # yes, BufferStore here - use same addressing regs for this load + addr0 = vgpr(addrDVgpr) + addr1 = sgpr("SrdD", 4) + else: + addr0 = vgpr(addrDVgpr, 2) + addr1 = "" + # Calculate vgpr Index for 32-bit/64-bit instruction + # DGEMM use SRCS[2] register + vgprIdx = bpm // 4 + module.add(self.parentWriter.chooseGlobalRead(useBuffer, bpm, dataV + vgprIdx, \ + addr0, addr1, soffset=0, offset=addrCalc.globalOffset, + comment="load D (atomic) bpm=%u vaw=%u"%(bpm,self.atomicW))) + + if self.kernel["InterleaveAlpha"] and self.applyAlpha: + module.add(self._applyAlpha(self.kernel, self.gwvw, self.ss.elementSumIdx, elementIdx, self.tmpS01)) + + if not self.kernel["BufferStore"]: + offsetSrc = (self.tmpVgpr + 2) if self.beta else addrDVgpr + + module.add(VAddCOU32(vgpr(addrDVgpr+0), VCC(), vgpr(self.addrD+0), \ + vgpr(offsetSrc+0), "addrDVgpr = D + index*bytes (lo)")) + module.add(VAddCCOU32(vgpr(addrDVgpr+1), VCC(), vgpr(self.addrD+1), \ + vgpr(offsetSrc+1), VCC(), "addrDVgpr = D + index*bytes (hi)")) + + if self.kernel["ProblemType"]["UseBias"] and (self.kernel["GlobalSplitU"] == 1): + module.add(VAddCOU32(dst=vgpr(addrBiasVgpr+0), dst1=VCC(), src0=vgpr(self.addrBias+0), \ + src1=vgpr(addrBiasVgpr+0), comment="addrBiasVgpr = Bias + index*bytes (lo)" )) + module.add(VAddCCOU32(dst=vgpr(addrBiasVgpr+1), dst1=VCC(), src0=vgpr(self.addrBias+1), \ + src1=VCC(), src2=vgpr(addrBiasVgpr+1), comment="addrBiasVgpr = Bias + index*bytes (hi)")) + + # restore full exec mask for calculating addr of next element + if self.edge and (self.beta or self.atomic): + module.add(self.getEdgeMovInstType()(EXEC(), -1, "full mask -1 -> exec")) + + module.add(loadInputCode) + + if self.beta and self.kernel["StoreSyncOpt"]: + self._storeSyncOpt(module) + + ######################################## + # AccVgpr read + if self.kernel.enabledSetPrioSplitLDS: + module.add(SSetPrior(0)) + if self.codeAccVgprRead is not None: + regsPerScalar = self.parentWriter.bpeCinternal // self.parentWriter.bpr # register per scalar + # loop over store instructions within one batch + for elementIdx in range(len(self.batchElements)): + # loop over scalars within one store instruction + for vi in range(self.gwvw): + # loop over registers within one scalar + for rIdx in range(0, regsPerScalar): + module.add(replaceHolder(self.codeAccVgprRead.items().pop(0), self.ss.elementSumIdx[elementIdx]*regsPerScalar + regsPerScalar*vi + rIdx - self.parentWriter.startVgprValuC)) + + if not self.kernel["MIArchVgpr"]: + module.add(SNop(1, "2 wait states required before reading vgpr")) + + ######################################## + # rC *= alpha + if not self.kernel["InterleaveAlpha"] and self.applyAlpha and not self.parentWriter.alphaBeforeLoadC: + module.addComment1("rC *= alpha batchElements=%s"%self.batchElements) + if self.codeMulAlpha is None: + for elementIdx in range(len(self.batchElements)): + module.add(self._applyAlpha(self.kernel, self.gwvw, self.ss.elementSumIdx, elementIdx, self.tmpS01)) + else: + regsPerScalar = self.parentWriter.bpeCinternal // self.parentWriter.bpr # register per scalar + for elementIdx in range(len(self.batchElements)): + for vi in range(self.gwvw): + module.add(replaceHolder(self.codeMulAlpha.items().pop(0), self.ss.elementSumIdx[elementIdx]*regsPerScalar + regsPerScalar*vi - self.parentWriter.startVgprValuC )) + + def _epilog(self, module: Module): + # return registers to pool: + lastData = -1 + checkedDataBias = {} + for elementIdx in range(len(self.batchElements)): + if not self.ss.sharedColDVgprs: + addrCalc: AddrCalculation = self.ss.elementAddr[elementIdx] + addrDVgpr = addrCalc.addrDVgpr + addrCVgpr = addrCalc.addrCVgpr + addrBiasVgpr = addrCalc.addrBiasVgpr + self.parentWriter.vgprPool.checkIn(addrDVgpr) + if addrCVgpr != addrDVgpr: + self.parentWriter.vgprPool.checkIn(addrCVgpr) + if addrBiasVgpr != None: + self.parentWriter.vgprPool.checkIn(addrBiasVgpr) + + data = self.ss.elementData[elementIdx] + if data != 0: + if data != lastData: + self.parentWriter.vgprPool.checkIn(data) + lastData = data + + dataBias = self.ss.elementDataBias[elementIdx] + if dataBias != 0: + if dataBias not in checkedDataBias: + self.parentWriter.vgprPool.checkIn(dataBias) + checkedDataBias[dataBias] = 1 + + self.ss.firstBatch = False + self.ss.checkInTempVgprC() + if self.kernel["StoreRemapVectorWidth"]: + if self.parentWriter.StoreRemapLastBatch == 1: + module.addComment1("Handle local read and global write") + # this seems buggy? it's possible to issue more than one stores for SR + # module.add(self.storeRemapAddStore(kernel, tmpVgpr, tmpS01, edge)) + # storesIssued += 1 + storeModule, numNewStores = self.parentWriter.storeRemapAddStore(self.kernel, self.tmpVgpr, self.tmpS01, self.edge) + module.add(storeModule) + self.storesIssued += numNewStores + + if self.parentWriter.serializedStore: + module.add(SNop(0, "1 wait state required when next inst writes vgprs held by previous dwordx4 store inst")) + + def _emitAdd(self, module: Module): + if self.atomic: + del self.tmpVgpr # catch bugs + if self.useAtomicAdd: + self._emitAtomicAdd(module) + else: + self._emitCasAdd(module) + else: + self._emitNonatomicAdd(module) + + def _emitNonatomicAdd(self, module: Module): + ######################################## + # Not Atomic + ######################################## + # edge has v_cndmask so loads or stores may not issue, hard to track vmcnt: + interleaveStoreVmcnt = self.parentWriter.interleaveStoreVmcnt and not self.edge + + for elementIdx in range(len(self.batchElements)): + for vi in range(self.gwvw): + sumIdxV = self.ss.elementSumIdx[elementIdx] + vi + newSumIdxV = sumIdxV - self.parentWriter.startVgprValuC + # covers sgemm, gemm_ex(HHS/HSS/BBS/BSS (HPA=T)), int8 (int8x4?) + if self.kernel["ProblemType"]["ComputeDataType"].isInt32() or \ + self.kernel["ProblemType"]["ComputeDataType"].isSingle(): # covers sgemm/gemm_ex(HHS/HSS/BBS/BSS) + if self.debugConfig["ForceExpectedValue"]: + module.add(VMovB32(vgpr("ValuC+%u"%newSumIdxV), self.debugConfig["ValueCExpectedValue"], "force expected value" )) + if self.parentWriter.db["ForceVSerial"]: + module.add(VMovB32(vgpr("ValuC+%u"%newSumIdxV), vgpr("Serial"), "force expected value to serial" )) + if self.parentWriter.db["CheckValueC"]: + module.add(SMovB32(sgpr(self.tmpS01), self.debugConfig["ValueCExpectedValue"], "Move expected value")) + module.add(self.parentWriter.getCmpAssert(self.parentWriter.asmAssert.eq, vgpr("ValuC+%u"%newSumIdxV), sgpr(self.tmpS01))) + + ######################################## + # wait for batched load + if self.beta and not interleaveStoreVmcnt: + module.add(SWaitCnt(vmcnt=0, vscnt=0, comment="writes & wait C")) + + module.addComment1("apply mask, calc new C and issue writes") + # module.add(self.getBomb()) # can see store addresses just before the store inst + + # Create a suffix and check if the string exists + activationLabelSuffix = self.parentWriter.labels.getNameInc( \ + "%s%s%u"%("Beta_" if self.beta else "", "Edge_" if self.edge else "", self.batchIdx)) + activationCDataType = self.kernel["ProblemType"]["ActivationComputeDataType"] + activationLabelEndModule = Label("Activation_End_%s"%activationLabelSuffix, "") + activationLabelModules = [] + activationEnumStrList = [] + if self.kernel["ActivationFuncCall"]: + activationLabelModules.append("") + activationEnumStrList.append("none") + elif ((self.kernel["GlobalSplitU"] == 1) and self.kernel["ActivationFused"]) and \ + (self.kernel["ProblemType"]["ActivationType"] != 'none'): + if self.kernel["ProblemType"]["ActivationType"] == 'all': + activationEnumStrList = ActivationType.getEnumStrList(activationCDataType) + for index, enumStr in enumerate(activationEnumStrList): + activationLabelModule = Label("Activation_%s_%s"% (enumStr.capitalize(), activationLabelSuffix), "") + activationLabelModules.append(activationLabelModule) + for index, activationLabelModule in enumerate(activationLabelModules): + if index != 0: + enumIndex = ActivationType.getEnumIndex(activationEnumStrList[index]) + module.add(SCmpKEQU32(sgpr("ActivationType"), enumIndex, "activationType == %u"%enumIndex)) + module.add(SCBranchSCC1(activationLabelModule.getLabelName(), "Branch if true")) + else: + activationEnumStrList.append(str(self.kernel["ProblemType"]["ActivationType"]).lower()) + else: + activationLabelModules.append("") + activationEnumStrList.append("none") + loadsIssuedRestore = self.loadsIssued + storesIssuedRestore = self.storesIssued + for index, activationLabelModule in enumerate(activationLabelModules): + self.loadsIssued = loadsIssuedRestore + self.storesIssued = storesIssuedRestore + activationTypeStr = activationEnumStrList[index] + if activationLabelModule: + module.add(activationLabelModule) + + if self.kernel["ProblemType"]["DestDataType"].isBFloat16() and self.kernel["ProblemType"]["HighPrecisionAccumulate"]: + module.add(VMovB32(vgpr(self.bf16CVTVgprStruct.vgprBf16Mask), "0xffff0000", "mask for pack two bfloat16 element to 32bit" )) + module.add(VMovB32(vgpr(self.bf16CVTVgprStruct.vgprFp32Nan), "0x7fff0000", "fp32 Nan" )) + module.add(VMovB32(vgpr(self.bf16CVTVgprStruct.vgprBf16Inc), "0x7fff", "rounding bias for bfloat16" )) + elif self.kernel["ProblemType"]["DataType"].isBFloat16() and self.kernel["ProblemType"]["UseBias"]: + module.add(VMovB32(dst=vgpr(self.bf16CVTVgprStruct.vgprBf16Mask), src="0xffff0000", comment="mask for pack two bfloat16 element to 32bit")) + + storeCode = Module("GroupLoadStore") + cvtDataBias = {} + for elementIdx in range(0, len(self.batchElements)): + element = self.batchElements[elementIdx] + addrCalc: AddrCalculation = self.ss.elementAddr[elementIdx] + addr = addrCalc.addrDVgpr + dataBias = self.ss.elementDataBias[elementIdx] + mask = self.ss.elementMask[elementIdx] + vc0 = element[3] + sumIdx = self.ss.elementSumIdx[elementIdx] + + # print(str(element)+" rowInc="+str(addrCalc.rowInc)) + # Already write wave column block into LDS + # Now read lds data back to registers and write to global memroy + if self.ss.optSrdIncForRow and addrCalc.rowInc and self.kernel["StoreRemapVectorWidth"] > 0: + module.addComment1("StoreRemap: shift coord1 address") + module.add(addrCalc.incrementToNextRow(self.kernel, "D", self.ss, self.tmpS01)) + module.add(VMovB32(vgpr(self.tmpVgpr), addrCalc.rowInc, "set shift rows")) + module.add(VAddU32(vgpr(self.parentWriter.vgprs.storeRemapCoord1), vgpr(self.parentWriter.vgprs.storeRemapCoord1), vgpr(self.tmpVgpr), "shift storeRemap coord1")) + + # apply in-bounds exec mask + if self.edge and not self.kernel["BufferStore"]: + module.add(self.getEdgeMovInstType()(EXEC(), sgpr(mask, self.laneSGPRC), "sgprs -> exec")) + + # if GWVW=1 the half path still assumes we have + # at least two stores so does some combining across VI - + # for example assuming we can have two elements and can use pk_mul + # here: + if (self.beta or self.kernel["ProblemType"]["UseBias"]) and interleaveStoreVmcnt: + waitLoadCnt = 0 + if self.beta: waitLoadCnt = elementIdx + if self.kernel["ProblemType"]["UseBias"]: waitLoadCnt += self.biasLoadIssued[elementIdx] + vmcnt = self.loadsIssued - waitLoadCnt - 1 + if self.parentWriter.archCaps["SeparateVscnt"]: + waitStoreCnt = 0 + vmComment = "{} = {} - {} - 1".format(vmcnt, self.loadsIssued, waitLoadCnt) + else: + waitStoreCnt = self.storesIssued if not self.kernel["GroupLoadStore"] else 0 + vmComment = "{} = {} - {} + {} - 1".format(vmcnt + waitStoreCnt, self.loadsIssued, waitLoadCnt, waitStoreCnt) + + maxVmcnt = globalParameters["AsmCaps"][self.parentWriter.version]["MaxVmcnt"] + vmcnt = min(vmcnt, maxVmcnt) + #print "wmvcnt=", vmcnt + module.addSpaceLine() + module.add(SWaitCnt(lgkmcnt=-1, vmcnt=vmcnt, vscnt=waitStoreCnt, comment="wait C (interleaved) " + vmComment)) + + if self.beta: + module.add(self._addSumAlphaWithCBeta(self.kernel, self.ss, self.gwvw, elementIdx, vc0, self.tmpVgpr, self.bf16CVTVgprStruct)) + + # Add bias + mergeActFuncCall = False + if self.kernel["ProblemType"]["UseBias"] and (self.kernel["GlobalSplitU"] == 1): + mergeActFuncCall = True + for vi in range(0, self.gwvw): + numVgprs = int(ceil(self.kernel["ProblemType"]["DataType"].numRegisters() * self.ss.cfg.gwvw)) + reg = self.kernel["ProblemType"]["DataType"].numRegisters() if self.kernel["ProblemType"]["DataType"].numRegisters() >= 1 else 1 + loadVgpr = dataBias + (self.ss.cfg.gwvw * reg - numVgprs) + int(vi * self.kernel["ProblemType"]["DataType"].numRegisters()) + inputVgpr = dataBias + vi + sumIdxV = self.ss.elementSumIdx[elementIdx] + vi + if self.kernel["ProblemType"]["ComputeDataType"].isSingle(): + vgprIdx = sumIdxV - self.parentWriter.startVgprValuC + vgprDst = (self.activationSetPCStruct.vgprActCopy + vi) if self.kernel["ActivationFuncCall"] else "ValuC+%d"%vgprIdx + # Generate single f32 code if edge is detected. + if ((vi + 1) == self.gwvw) and ((self.gwvw % 2) == 1): + if inputVgpr not in cvtDataBias: + if self.kernel["ProblemType"]["DataType"].isHalf(): + if vi % 2 == 0: + sdwa = SDWAModifiers(src0_sel=SelectBit.WORD_0) + else: + sdwa = SDWAModifiers(src0_sel=SelectBit.WORD_1) + module.add(VCvtF16toF32(dst=vgpr(inputVgpr), src=vgpr(loadVgpr), sdwa=sdwa, comment="convert f16 to fp32")) + elif self.kernel["ProblemType"]["DataType"].isBFloat16(): + module.add(VCvtBF16toFP32(dst=inputVgpr, src=loadVgpr, vgprMask=self.bf16CVTVgprStruct.vgprBf16Mask, vi=vi)) + cvtDataBias[inputVgpr] = 1 + module.add(VAddF32(dst=vgpr(vgprDst), src0=vgpr(inputVgpr), src1=vgpr("ValuC+%d"%vgprIdx), \ + comment="C += bias")) + # Original packed route + elif vi%2 == 1: + assert (self.gwvw % 2 == 0) + else: + if inputVgpr not in cvtDataBias: + if self.kernel["ProblemType"]["DataType"].isHalf(): + module.add(VCvtF16toF32(dst=vgpr(inputVgpr), src=vgpr(loadVgpr), \ + sdwa=SDWAModifiers(src0_sel=SelectBit.WORD_0), comment="convert f16 to fp32")) + module.add(VCvtF16toF32(dst=vgpr(inputVgpr+1), src=vgpr(loadVgpr), \ + sdwa=SDWAModifiers(src0_sel=SelectBit.WORD_1), comment="convert f16 to fp32")) + elif self.kernel["ProblemType"]["DataType"].isBFloat16(): + module.add(VCvtBF16toFP32(dst=inputVgpr, src=loadVgpr, vgprMask=self.bf16CVTVgprStruct.vgprBf16Mask, vi=vi)) + module.add(VCvtBF16toFP32(dst=inputVgpr+1, src=loadVgpr, vgprMask=self.bf16CVTVgprStruct.vgprBf16Mask, vi=vi+1)) + cvtDataBias[inputVgpr] = 1 + module.add(VAddPKF32(dst=vgpr(vgprDst, 2), src0=vgpr(inputVgpr, 2), \ + src1=vgpr("ValuC+%d"%vgprIdx, 2), comment="C += bias")) + else: + raise RuntimeError("Unsupported bias compute data type %s."%str(self.kernel["ProblemType"]["ComputeDataType"])) + + SaturateTypeInt8 = SaturateCastType.NORMAL + # Activation + activationModule = None + isActivationInsertAfter = False + if self.kernel["ActivationFuncCall"]: + activationModule = Module("ActivationFuncCall") + if not mergeActFuncCall: + for vi in range(0, self.gwvw): + sumIdxV = self.ss.elementSumIdx[elementIdx] + vi + activationModule.add(VMovB32(dst=vgpr(self.activationSetPCStruct.vgprActCopy + vi), src=vgpr(sumIdxV))) + activationModule.add(SSwapPCB64(dst=sgpr(self.activationSetPCStruct.sgprOffsetBack, 2), \ + src=sgpr(self.activationSetPCStruct.sgprOffsetActivation, 2))) + for vi in range(0, self.gwvw): + sumIdxV = self.ss.elementSumIdx[elementIdx] + vi + activationModule.add(VMovB32(dst=vgpr(sumIdxV), src=vgpr(self.activationSetPCStruct.vgprActCopy + vi))) + elif self.parentWriter.insertActivationAfterPacked(self.kernel, activationTypeStr): + isActivationInsertAfter = True + activationModule = self.parentWriter.getActivationDestDataType(self.kernel, self.activation, \ + activationTypeStr, self.gwvw, self.ss.elementSumIdx[elementIdx], self.tmpVgpr, self.tmpSgpr) + else: + satInt8 = False + if self.kernel["ProblemType"]["DestDataType"].isInt8(): + if (activationTypeStr == 'abs') or (activationTypeStr == 'relu'): + SaturateTypeInt8 = SaturateCastType.DO_NOTHING + satInt8 = True + activationModule = self.parentWriter.getActivationActivationComputeType(self.kernel, self.activation, \ + activationTypeStr, self.gwvw, self.ss.elementSumIdx[elementIdx], self.tmpVgpr, self.tmpSgpr, satInt8) + + # pack stores, beta and non-beta reach here: + packModule = Module("Empty pack module") + if self.kernel["ProblemType"]["HighPrecisionAccumulate"] and (self.kernel["_GlobalAccumulation"] != 'MultipleBuffer'): + if self.kernel["ProblemType"]["DestDataType"].isHalf(): + packModule = self.packdata(self.gwvw, self.ss.elementSumIdx[elementIdx], inputPrefix="ValuC+", prefixOffset=self.parentWriter.startVgprValuC) + elif self.kernel["ProblemType"]["DestDataType"].isBFloat16(): + packModule = self.packdata(self.gwvw, self.ss.elementSumIdx[elementIdx], bf16CVTVgprStruct=self.bf16CVTVgprStruct, + tmpS01=self.tmpS01, laneSGPRC=self.laneSGPRC, inputPrefix="ValuC+", prefixOffset=self.parentWriter.startVgprValuC) + elif self.kernel["ProblemType"]["DestDataType"].isInt8(): + packModule = self.packdata(self.gwvw, self.ss.elementSumIdx[elementIdx], self.tmpVgpr, self.tmpS01, + SaturateTypeInt8=SaturateTypeInt8, inputPrefix="ValuC+", prefixOffset=self.parentWriter.startVgprValuC) + + if isActivationInsertAfter: + module.add(packModule) + module.add(activationModule) + else: + module.add(activationModule) + module.add(packModule) + + if not self.kernel["StoreRemapVectorWidth"]: + tmpStoreCode = self.parentWriter.addStore(self.kernel, self.ss, addrCalc, sumIdx, self.tmpS01, self.edge) + if self.kernel["GroupLoadStore"]: + storeCode.add(tmpStoreCode) + else: + module.add(tmpStoreCode) + self.storesIssued += 1 + + else: + rpe = self.parentWriter.bpeCinternal // self.parentWriter.bpr + module.add(self.parentWriter.storeRemapAddLocalWrite(self.ss, addrCalc, sumIdx*rpe)) + # Column Block Shape has been written to LDS + # Now read back and write out to global memory + + module.add(storeCode) + + if self.parentWriter.db["CheckStoreC"]>=0: + useBuffer = self.kernel["BufferStore"] + # Note - CheckStoreC won't work for EDGE store cases since they load 0 for OOB, would need more sophisticated check + # Note - TODO- CheckStoreC also won't work for StoreRemap + module.add(SWaitCnt(vmcnt=0, vscnt=0, comment="CheckStoreC, wait for stores to complete")) + for elementIdx in range(0, len(self.batchElements)): + addr = self.ss.elementAddr[elementIdx].addrDVgpr + sumIdx = self.ss.elementSumIdx[elementIdx] + + bps = self.kernel["ProblemType"]["DestDataType"].numBytes() * self.gwvw + if self.kernel["BufferStore"]: + addr0 = vgpr(addr) + addr1 = sgpr("SrdC", 4) + else: + addr0 = vgpr(addr,2) + addr1 = "" + + if self.kernel["ProblemType"]["DestDataType"].isHalf() or self.kernel["ProblemType"]["DestDataType"].isBFloat16(): + if not self.kernel["ProblemType"]["HighPrecisionAccumulate"]: + module.add(self.parentWriter.chooseGlobalRead(useBuffer, bps, sumIdx//2, \ + addr0, addr1, soffset=0, offset=0, hi16=sumIdx%2)) + else: + module.add(self.parentWriter.chooseGlobalRead(useBuffer, bps, sumIdx, \ + addr0, addr1, soffset=0, offset=0, hi16=0)) + elif self.kernel["ProblemType"]["DestDataType"].isInt32() or self.kernel["ProblemType"]["DestDataType"].isSingle(): + module.add(self.parentWriter.chooseGlobalRead(useBuffer, bps, sumIdx, \ + addr0, addr1, soffset=0, offset=0)) + elif self.kernel["ProblemType"]["DestDataType"].isDouble() or self.kernel["ProblemType"]["DestDataType"].isSingleComplex() : + module.add(self.parentWriter.chooseGlobalRead(useBuffer, bps, sumIdx*2, \ + addr0, addr1, soffset=0, offset=0)) + elif self.kernel["ProblemType"]["DestDataType"].isDoubleComplex(): + module.add(self.parentWriter.chooseGlobalRead(useBuffer, bps, sumIdx*4, \ + addr0, addr1, soffset=0, offset=0)) + module.add(SWaitCnt(vmcnt=0, vscnt=0, comment="CheckStoreC, wait for stores to complete")) + # Add checks for expected values: + module.add(SMovB32(sgpr(self.tmpS01), self.parentWriter.db["CheckStoreC"], "expected value")) + for elementIdx in range(0, len(self.batchElements)): + sumIdx = self.ss.elementSumIdx[elementIdx] + # Need to fix for other types: + assert (self.kernel["ProblemType"]["DestDataType"].isSingle() or self.kernel["ProblemType"]["DestDataType"].isInt32()) + module.add(self.parentWriter.getCmpAssert(self.parentWriter.asmAssert.eq, vgpr(sumIdx), sgpr(self.tmpS01))) + + + if self.edge and (self.atomic or not self.kernel["BufferStore"]): + # subsequent batch must start with full exec mask + # BufferStore doesn't need exec since it used buffer range checking when + # possible + module.add(self.getEdgeMovInstType()(EXEC(), -1, "full mask -> exec")) + + if self.parentWriter.db["ConservativeWaitCnt"] & 0x40: + module.add(SBarrier("debug")) + module.add(SWaitCnt(vmcnt=0, vscnt=0, comment="ConservativeWaitCnt")) + module.add(SBarrier("debug")) + + if (index < (len(activationLabelModules) - 1)): + module.add(SBranch(labelName=activationLabelEndModule.getLabelName())) + if len(activationLabelModules) > 1: + module.add(activationLabelEndModule) + + def _emitAtomicAdd(self, module: Module): + ######################################## + # first attempt write + module.addComment1("issue first atomic writes") + for elementIdx in range(len(self.batchElements)): + addrCalc = self.ss.elementAddr[elementIdx] + mask = self.ss.elementMask[elementIdx] + + # apply in-bounds exec mask + if self.edge: + module.add(self.getEdgeMovInstType()(EXEC(), sgpr(mask, self.laneSGPRC), "sgprs -> exec (before atomic)")) + + for avi in range(0, self.gwvw // self.atomicW): + sumIdxV = self.ss.elementSumIdx[elementIdx] + avi + newSumIdxV = sumIdxV - self.parentWriter.startVgprValuC + if self.parentWriter.do["GlobalWrite"]: + if self.kernel["BufferStore"]: + module.add(BufferAtomicAddF32(vgpr("ValuC+%u"%newSumIdxV), \ + vgpr(addrCalc.addrDVgpr,1), \ + sgpr("SrdD", 4), \ + 0, + MUBUFModifiers(offen=True, offset12=addrCalc.globalOffset), + "attempt write avi=%u" % (avi))) + else: + pass # TODO: + + if self.edge: + module.add(self.getEdgeMovInstType()(EXEC(), -1, "full mask -> exec")) + + def _emitCasAdd(self, module: Module): + # TODO for atomic GWVW: + # - Use vi to compute addresses, sumIdx. + # - Need a solution for the mask. Can move to all buffer or can fix? + element = self.batchElements[0] + d1 = element[0] + d0 = element[1] + vc1 = element[2] + vc0 = element[3] + labels = self.parentWriter.labels + labelString = "Global_Write%s%s_%u_%u_%u_%u" % ("_Beta" if self.beta else "", "_Edge" if self.edge else "", vc0, vc1, d0, d1 ) + labelComment = "Global_Write (Beta) (Edge) vc0 vc1 d0 d1" + label = Label(labels.getName(labelString), labelComment) + labelString += "_EarlyExit" + labelAfterAtomicLoop = Label(labels.getName(labelString), labelComment) + + ######################################## + # wait for batched load + # TODO - we are always atomic here? + module.add(SWaitCnt(vmcnt=0, vscnt=0, comment="wait C (atomic)")) + ######################################## + # first attempt write + module.addComment1("issue first atomic writes") + for elementIdx, element in enumerate(self.batchElements): + addrCalc = self.ss.elementAddr[elementIdx] + mask = self.ss.elementMask[elementIdx] + + # apply in-bounds exec mask + if self.edge: + module.add(self.getEdgeMovInstType()(EXEC(), sgpr(mask, self.parentWriter.laneSGPRCount), "sgprs -> exec (before atomic)")) + + for avi in range(0, self.gwvw//self.atomicW): + dataV = self.ss.elementData[elementIdx] + int(avi*self.ss.cfg.numVgprsPerDataPerVI) + sumIdxV = self.ss.elementSumIdx[elementIdx] + avi + ## number of src[s]/dst[s] register for DGEMM / SGEMM HGEMM + vgprCnt = 2 if self.kernel["ProblemType"]["DestDataType"].isDouble() else 1 + if self.kernel["ProblemType"]["DestDataType"].numRegisters() < 1 and not self.kernel["_GlobalAccumulation"]: + sumIdxV //= 2 + if self.kernel["ProblemType"]["DestDataType"].isDouble(): sumIdxV = sumIdxV * 2 + newSumIdxV = sumIdxV - self.parentWriter.startVgprValuC + bpm = self.parentWriter.bpeCexternal * self.atomicW + # Calculate vgpr Index for 32-bit/64-bit instruction + # DGEMM use SRCS[2] register + vgprIdx = 1*(bpm//4) + # for atomic, data[1] = original c, data[0] = new c + module.add(self._chooseAddForAtomic(self.kernel, \ + vgpr(dataV+0,vgprCnt), vgpr(dataV+1*vgprIdx,vgprCnt), vgpr("ValuC+%u"%newSumIdxV,vgprCnt), \ + "desired value avi=%u"%avi)) + + # attempt write + atomicDestVgpr = dataV if self.kernel["BufferStore"] else dataV+2 + if self.parentWriter.do["GlobalWrite"]: + if self.kernel["BufferStore"]: + # use cmpswap_x2 for DGEMM in CAS loop + if self.kernel["ProblemType"]["DestDataType"].isDouble(): + module.add(BufferAtomicCmpswapB64(vgpr(dataV,4), \ + vgpr(addrCalc.addrDVgpr,1), \ + sgpr("SrdD", 4), \ + 0, + MUBUFModifiers(offen=True, offset12=addrCalc.globalOffset, glc=True), + "attempt write avi=%u"%(avi))) + else: + # use cmpswap for SGEMM in CAS loop + module.add(BufferAtomicCmpswapB32(vgpr(dataV,2), \ + vgpr(addrCalc.addrDVgpr,1), \ + sgpr("SrdD", 4), \ + 0, \ + MUBUFModifiers(offen=True, offset12=addrCalc.globalOffset, glc=True), \ + "attempt write avi=%u"%(avi))) + else: + module.add(FlatAtomicCmpswapB32(vgpr(atomicDestVgpr), \ + vgpr(addrCalc.addrDVgpr,2), \ + vgpr(dataV,2), + FLATModifiers(glc=True), + "attempt write")) + else: + # Fake successful CAS swap + module.add(VMovB32(vgpr(atomicDestVgpr), vgpr(dataV+1), "Fake successful CAS" )) + + ######################################## + # wait for first attempt write + module.add(SWaitCnt(vmcnt=0, vscnt=0, comment="wait for atomic writes")) + ######################################## + # check first attempt + module.addComment1("check success of writes, update masks") + for elementIdx, element in enumerate(self.batchElements): + mask = self.ss.elementMask[elementIdx] + + # calculate new masks + if self.edge: + module.add(self.getEdgeMovInstType()(EXEC(), sgpr(mask, self.laneSGPRC), "sgprs -> exec")) + for avi in range(0, self.gwvw // self.atomicW): + dataV = self.ss.elementData[elementIdx] + int(avi * self.ss.cfg.numVgprsPerDataPerVI) + atomicDestVgpr = dataV if self.kernel["BufferStore"] else dataV+2 + # need to apply element mask before comparison + # so that all valid lanes are doing the cmp + if avi == 0: + # use u64 for DGEMM + if self.kernel["ProblemType"]["DestDataType"].isDouble(): + module.add(VCmpNeU64(sgpr(self.tmpS01, self.laneSGPRC), vgpr(atomicDestVgpr,2), \ + vgpr(dataV+2,2), comment="c read during atomic == c read during prior load (avi=%u, first)"%avi)) + else: + module.add(VCmpNeU32(sgpr(self.tmpS01, self.laneSGPRC), vgpr(atomicDestVgpr), \ + vgpr(dataV+1), comment="c read during atomic == c read during prior load (avi=%u, first)"%avi)) + else: + if self.kernel["ProblemType"]["DestDataType"].isDouble(): + module.add(VCmpNeU64(sgpr(self.tmpS23, self.laneSGPRC), vgpr(atomicDestVgpr,2), \ + vgpr(dataV+2,2), comment="c read during atomic != c read during prior load")) + else: + module.add(VCmpNeU32(sgpr(self.tmpS23, self.laneSGPRC), vgpr(atomicDestVgpr), \ + vgpr(dataV+1), comment="c read during atomic == c read during prior load (avi=%u)"%avi)) + module.add(self.getEdgeOrInstType()(sgpr(self.tmpS01, self.laneSGPRC), \ + sgpr(self.tmpS01, self.laneSGPRC), sgpr(self.tmpS23, self.laneSGPRC), "combine with tmp mask")) + + module.add(self.getEdgeAndInstType()(sgpr(mask, self.laneSGPRC), sgpr(self.tmpS01, self.laneSGPRC), sgpr(mask,self.laneSGPRC), "inBounds & must try again")) + + else: + for avi in range(0, self.gwvw//self.atomicW): + dataV = self.ss.elementData[elementIdx] + int(avi*self.ss.cfg.numVgprsPerDataPerVI) + atomicDestVgpr = dataV if self.kernel["BufferStore"] else dataV+2 + if self.kernel["ProblemType"]["DestDataType"].isDouble(): + module.add(VCmpNeU64(sgpr(mask, self.laneSGPRC), vgpr(atomicDestVgpr,2), \ + vgpr(dataV+2,2), comment="c read during atomic != c read during prior load")) + else: + module.add(VCmpNeU32(sgpr(mask, self.laneSGPRC), vgpr(atomicDestVgpr), \ + vgpr(dataV+1), comment="c read during atomic != c read during prior load")) + + # or masks together to check early exit + module.addComment1("or masks to check for exit") + module.add(self.getEdgeMovInstType()(sgpr(self.tmpS01, self.laneSGPRC), hex(0), "empty mask")) + for elementIdx in range(0, len(self.batchElements)): + mask = self.ss.elementMask[elementIdx] + module.add(self.getEdgeOrInstType()(sgpr(self.tmpS01, self.laneSGPRC), sgpr(mask, self.laneSGPRC), sgpr(self.tmpS01, self.laneSGPRC), "or to add threads")) + module.add(self.getSOrSaveExecType()(sgpr(self.tmpS23,self.laneSGPRC), sgpr(self.tmpS01,self.laneSGPRC), "apply combined mask")) + module.add(SCBranchExecZ(labelAfterAtomicLoop.getLabelName(), "if exec is zero skip loop")) + + # begin atomic loop + module.addComment1("atomic CAS loop") + module.add(label) + + module.addComment1("apply updated masks and issue writes again") + for elementIdx in range(0, len(self.batchElements)): + addrCalc = self.ss.elementAddr[elementIdx] + addr = addrCalc.addrDVgpr + mask = self.ss.elementMask[elementIdx] + vgprCnt = 2 if self.kernel["ProblemType"]["DestDataType"].isDouble() else 1 # number of registers for f32/f64 + bpm = self.parentWriter.bpeCexternal * self.atomicW + vgprIdx = 1*(bpm//4) # index register + + for avi in range(0, self.gwvw//self.atomicW): + dataV = self.ss.elementData[elementIdx] + int(avi*self.ss.cfg.numVgprsPerDataPerVI) + atomicDestVgpr = dataV if self.kernel["BufferStore"] else dataV+2 + sumIdxV = self.ss.elementSumIdx[elementIdx] + avi + if self.kernel["ProblemType"]["DestDataType"].numRegisters() < 1 and not self.kernel["_GlobalAccumulation"]: + sumIdxV //= 2 + if self.kernel["ProblemType"]["DestDataType"].isDouble(): + sumIdxV = sumIdxV * 2 + newSumIdxV = sumIdxV - self.parentWriter.startVgprValuC + + # apply mask for element + module.add(self.getEdgeMovInstType()(EXEC(), sgpr(mask,self.laneSGPRC), "must try again")) + if self.kernel["ProblemType"]["DestDataType"].isDouble(): + #64-bit C val move by 2 32-bit instructions + module.add(VMovB32(vgpr(dataV+2), vgpr(atomicDestVgpr), "dataV+2 = tmp (new original C)" )) + module.add(VMovB32(vgpr(dataV+3), vgpr(atomicDestVgpr+1), "dataV+3 = tmp (new original C)" )) + else: + module.add(VMovB32(vgpr(dataV+1), vgpr(atomicDestVgpr), "dataV+1 = tmp (new original C)" )) + module.add(self._chooseAddForAtomic(self.kernel, \ + vgpr(dataV+0,vgprCnt), vgpr(dataV+1*vgprIdx,vgprCnt), vgpr("ValuC+%u"%newSumIdxV,vgprCnt), \ + "newC = rC + originalC")) + if self.parentWriter.do["GlobalWrite"]: + if self.kernel["BufferStore"]: + # Using no-ret version here? + # cmpswap_x2 for DGEMM + if self.kernel["ProblemType"]["DestDataType"].isDouble(): + module.add(BufferAtomicCmpswapB64(vgpr(dataV,4), \ + vgpr(addr,1), \ + sgpr("SrdD", 4), \ + 0, + MUBUFModifiers(offen=True, offset12=addrCalc.globalOffset, glc=True), + "try again")) + else: + module.add(BufferAtomicCmpswapB32( + vgpr(dataV,2), \ + vgpr(addr,1), \ + sgpr("SrdD", 4), \ + 0, + MUBUFModifiers(offen=True, offset12=addrCalc.globalOffset, glc=True), + "try again")) + else: + module.add(FlatAtomicCmpswapB32(vgpr(atomicDestVgpr), \ + vgpr(addr,2), \ + vgpr(dataV,2), \ + FLATModifiers(glc=True), \ + "try again")) + + # wait for batched write + module.add(SWaitCnt(vmcnt=0, vscnt=0, comment="wait for atomic writes")) + # check batched write success + module.addComment1("apply masks and check for success") + for elementIdx in range(0, len(self.batchElements)): + data = self.ss.elementData[elementIdx] + mask = self.ss.elementMask[elementIdx] + for avi in range(0, self.gwvw//self.atomicW): + dataV = self.ss.elementData[elementIdx] + int(avi*self.ss.cfg.numVgprsPerDataPerVI) + atomicDestVgpr = dataV if self.kernel["BufferStore"] else dataV+2 + + # apply mask for element + module.add(self.getEdgeMovInstType()(EXEC(), sgpr(mask,self.laneSGPRC), "must try again")) + + # compare success + if self.kernel["ProblemType"]["DestDataType"].isDouble(): + module.add(VCmpNeU64(sgpr(self.tmpS01,self.laneSGPRC), vgpr(data+2,2), vgpr(atomicDestVgpr,2), \ + comment="c read during atomic != c read during prior load")) + else: + module.add(VCmpNeU32(sgpr(self.tmpS01,self.laneSGPRC), vgpr(data+1), vgpr(atomicDestVgpr), \ + comment="c read during atomic == c read during prior load")) + # update element mask + module.add(self.getEdgeAndInstType()(sgpr(mask,self.laneSGPRC), sgpr(self.tmpS01,self.laneSGPRC), sgpr(mask,self.laneSGPRC), "inBounds & must try again")) + + # or masks together + module.addComment1("or masks to check for exit") + module.add(self.getEdgeMovInstType()(sgpr(self.tmpS01,self.laneSGPRC), hex(0), "empty mask")) + for elementIdx in range(0, len(self.batchElements)): + mask = self.ss.elementMask[elementIdx] + module.add(self.getEdgeOrInstType()(sgpr(self.tmpS01,self.laneSGPRC), sgpr(mask,self.laneSGPRC), sgpr(self.tmpS01,self.laneSGPRC), "or to add threads")) + + # apply combined masks and exit + module.add(self.getSOrSaveExecType()(sgpr(self.tmpS23, self.laneSGPRC), sgpr(self.tmpS01,self.laneSGPRC), "apply combined mask")) + module.add(SCBranchExecNZ(label.getLabelName(), "try again if not complete")) + module.add(labelAfterAtomicLoop) + module.add(self.getEdgeMovInstType()(EXEC(), -1, "full mask -> exec")) + + def _checkAtomicPreconditions(self) -> bool: + if self.atomic: + # all kinds of code relies on this assumption: + if self.atomicW > self.gwvw: + return False + + if (self.kernel["ProblemType"]["DataType"].isHalf() or self.kernel["ProblemType"]["DataType"].isBFloat16()) \ + and not self.kernel["_GlobalAccumulation"]: + return self.atomicW >= 2 + return True + + def _storeSyncOpt(self, module: Module): + module.add(SSleep(self.kernel["StoreSyncOpt"] - 1, "optimization: sync and wait")) + module.add(SBarrier()) + + def _applyAlpha(self, kernel, gwvw, elementSumIdx, elementIdx, tmpS01): + module = Module("applyAlpha") + + if kernel["_GlobalAccumulation"] == 'MultipleBuffer': + return module + + if self.parentWriter.do["ApplyAlpha"]: + for vi in range(0, gwvw): + sumIdxV = elementSumIdx[elementIdx] + vi + + if kernel["ProblemType"]["ComputeDataType"].isHalf() and not kernel["ProblemType"]["HighPrecisionAccumulate"]: + # (h,h,h,h,h,h), internal alpha is f16 (2-16bits) + if sumIdxV%2: + newSumIdx = sumIdxV // 2 - self.parentWriter.startVgprValuC + module.add(VMulPKF16(dst=vgpr("ValuC+%u"%(newSumIdx)), src0=sgpr("Alpha"), src1=vgpr("ValuC+%u"%(newSumIdx)), comment="*= alpha sumIdx=%u vi=%u"%(elementSumIdx[elementIdx], vi))) + + # Int8 (TODO- Int8x4 not checked, but should be OK) + elif kernel["ProblemType"]["ComputeDataType"].isInt32(): + newSumIdx = sumIdxV - self.parentWriter.startVgprValuC + # below assume we use v_mul_lo_u32. Could also use v_mul_i32_i24. + # module.add(VMulI32I24(dst=vgpr("ValuC+%u"%newSumIdx), src0=sgpr("Alpha"), src1=vgpr("ValuC+%u"%newSumIdx), comment="*= alpha" )_ + module.add(VMulLOU32(dst=vgpr("ValuC+%u"%newSumIdx), src0=sgpr("Alpha"), src1=vgpr("ValuC+%u"%newSumIdx), comment="*= alpha" )) + if self.parentWriter.db["ForceExpectedValue"]: + module.add(VMovB32(dst=vgpr("ValuC+%u"%newSumIdx), src=self.parentWriter.db["ValueCExpectedValue"], comment="force expected value" )) + if self.parentWriter.db["CheckValueC"]: + module.add(SMovB32(dst=sgpr(tmpS01), src=self.parentWriter.db["ValueCExpectedValue"], comment="Move expected value")) + module.add(self.parentWriter.getCmpAssert(self.parentWriter.asmAssert.eq, vgpr("ValuC+%u"%newSumIdx), sgpr(tmpS01))) + + # sgemm, HPA-bfgemm(b,b,b,b,s,s), and HPA-hgemm(h,h,h,h,s,s) + # (h,h,h,h,h,h) + HPA (will be converted to (h,h,h,h,s,s)), internal alpha is single + elif kernel["ProblemType"]["ComputeDataType"].isSingle() or (kernel["ProblemType"]["ComputeDataType"].isHalf() and kernel["ProblemType"]["HighPrecisionAccumulate"]): + newSumIdx = sumIdxV - self.parentWriter.startVgprValuC + module.add(VMulF32(dst=vgpr("ValuC+%u"%newSumIdx), src0=sgpr("Alpha"), src1=vgpr("ValuC+%u"%newSumIdx), comment="*= alpha" )) + if self.parentWriter.db["ForceExpectedValue"]: + module.add(VMovB32(dst=vgpr("ValuC+%u"%newSumIdx), src=self.parentWriter.db["ValueCExpectedValue"], comment="force expected value" )) + if self.parentWriter.db["ForceVSerial"]: + module.add(VMovB32(dst=vgpr("ValuC+%u"%newSumIdx), src=vgpr("Serial"), comment="force expected value to serial" )) + if self.parentWriter.db["CheckValueC"]: + module.add(SMovB32(dst=sgpr(tmpS01), src=self.parentWriter.db["ValueCExpectedValue"], comment="Move expected value")) + module.add(self.parentWriter.getCmpAssert(self.parentWriter.asmAssert.eq, vgpr("ValuC+%u"%newSumIdx), sgpr(tmpS01))) + + # dgemm + elif kernel["ProblemType"]["ComputeDataType"].isDouble(): + newSumIdx = sumIdxV * 2 - self.parentWriter.startVgprValuC + module.add(VMulF64(dst=vgpr("ValuC+%u"%(newSumIdx),2), src0=sgpr("Alpha",2), src1=vgpr("ValuC+%u"%(newSumIdx),2), comment="*= alpha")) + + # single precision complex + elif kernel["ProblemType"]["ComputeDataType"].isSingleComplex(): + newSumIdx = sumIdxV * 2 - self.parentWriter.startVgprValuC + tmpVgpr = self.parentWriter.vgprPool.checkOut(1) + module.add(VMovB32(dst=vgpr(tmpVgpr), src=vgpr("ValuC+%u"%(newSumIdx)), comment="store Cr")) + module.add(VMulF32(dst=vgpr("ValuC+%u"%(newSumIdx)), src0=sgpr("Alpha"), src1=vgpr("ValuC+%u"%(newSumIdx)), comment="*= alpha ( Cr = Ar * Cr)")) + module.add(VMacF32(dst=vgpr("ValuC+%u"%(newSumIdx)), src0=(sgpr("Alpha+1").getMinus()), src1=vgpr("ValuC+%u"%(newSumIdx+1)), comment="*= alpha ( Cr += -Ai * Ci )")) + module.add(VMulF32(dst=vgpr("ValuC+%u"%(newSumIdx+1)), src0=sgpr("Alpha"), src1=vgpr("ValuC+%u"%(newSumIdx+1)), comment="*= alpha ( Ci = Ar * Ci)")) + module.add(VMacF32(dst=vgpr("ValuC+%u"%(newSumIdx+1)), src0=sgpr("Alpha+1"), src1=vgpr(tmpVgpr), comment="*= alpha ( Ci += Ai * Cr_backup )")) + self.parentWriter.vgprPool.checkIn(tmpVgpr) + + # double precision complex + elif kernel["ProblemType"]["ComputeDataType"].isDoubleComplex(): + newSumIdx = sumIdxV * 4 - self.parentWriter.startVgprValuC + vtmp1 = self.parentWriter.vgprPool.checkOutAligned(2, 2) + vtmp2 = self.parentWriter.vgprPool.checkOutAligned(2, 2) + # tmp1 = a.real * b.real + module.add(VMulF64(dst=vgpr(vtmp1,2), src0=sgpr("Alpha+0",2), src1=vgpr("ValuC+%u"%(newSumIdx+0),2))) + # tmp2 = a.imag * b.real + module.add(VMulF64(dst=vgpr(vtmp2,2), src0=sgpr("Alpha+2",2), src1=vgpr("ValuC+%u"%(newSumIdx+0),2))) + # c.real = a.real * b.real - a.imag * b.imag = tmp1 - a.imag * b.imag + module.add(VFmaF64(dst=vgpr("ValuC+%u"%(newSumIdx+0),2), src0=sgpr("Alpha+2",2), src1=vgpr("ValuC+%u"%(newSumIdx+2),2), src2=vgpr(vtmp1,2))) + # c.imag = a.real * b.imag + a.imag * b.real = a.real * b.imag + tmp2 + module.add(VFmaF64(dst=vgpr("ValuC+%u"%(newSumIdx+2),2), src0=sgpr("Alpha+0",2), src1=vgpr("ValuC+%u"%(newSumIdx+2),2), src2=vgpr(vtmp2,2))) + self.parentWriter.vgprPool.checkIn(vtmp1) + self.parentWriter.vgprPool.checkIn(vtmp2) + return module + + def _addSumAlphaWithCBeta(self, kernel, ss, gwvw, elementIdx, vc0, tmpVgpr, bf16CVTVgprStruct): + module = Module("addSumAlphaWithCBeta #elementIdx%u, vc0 %u"%(elementIdx, vc0)) + for vi in range(0, gwvw): + dataV = ss.elementData[elementIdx] + int(vi*ss.cfg.numVgprsPerDataPerVI) + sumIdxV = ss.elementSumIdx[elementIdx] + vi + if kernel["ProblemType"]["DestDataType"].isHalf(): + if not kernel["ProblemType"]["HighPrecisionAccumulate"]: + if sumIdxV%2==0 or (not ss.cfg.halfDataRegPerVI and gwvw==1): + newSumIdxV = sumIdxV // 2 - self.parentWriter.startVgprValuC + # dataV+0 = new c = old c*beta + module.add(VMulPKF16(dst=vgpr(dataV), src0=sgpr("Beta"), src1=vgpr(dataV+0), \ + comment="%s = C*beta ei=%u vi=%u"%(vgpr(dataV),elementIdx, vi))) + # dataV+0 = new c = old c*beta + rC + module.add(VAddPKF16(dst=vgpr("ValuC+%u"%(newSumIdxV)), src0=vgpr(dataV), src1=vgpr("ValuC+%u"%(newSumIdxV)), \ + comment="sum*alpha + C*beta")) + else: + pass # add will have been done previously + else: # HPA + newSumIdxV = sumIdxV - self.parentWriter.startVgprValuC + # dataV+0 = new c = old c*beta + rC + # src0 = beta = f32 = opsel 00 + # src1 = dataV = f16.lo = opsel 10 or 11 depending on even/odd + # src2 = sumIdxV = f32 = opsel 00 + dataCExternal = ss.elementData[elementIdx] + vi//2 + hi16 = (vi + gwvw*vc0) % 2 + module.add(self.parentWriter.mixinst(dst=vgpr("ValuC+%u"%newSumIdxV), src0=sgpr("Beta"), \ + src1=vgpr(dataCExternal), src2=vgpr("ValuC+%u"%newSumIdxV), \ + vop3=VOP3PModifiers(op_sel=[0,hi16,0], op_sel_hi=[0,1,0]), + comment="//C*=beta")) + + elif kernel["ProblemType"]["DestDataType"].isBFloat16(): + if kernel["ProblemType"]["HighPrecisionAccumulate"]: + # dataV+0 = new c = old c*beta + rC + # src0 = beta = f32 = opsel 00 + # src1 = dataV = f16.lo = opsel 10 or 11 depending on even/odd + # src2 = sumIdxV = f32 = opsel 00 + dataCExternal = ss.elementData[elementIdx] + vi//2 + if (vi%2) == 1: + module.add(VAndB32(dst=vgpr(tmpVgpr), src0=vgpr(dataCExternal), src1=vgpr(bf16CVTVgprStruct.vgprBf16Mask), comment="convert bf16 to fp32")) + else: + module.add(VLShiftLeftB32(dst=vgpr(tmpVgpr), shiftHex=16, src=vgpr(dataCExternal), comment="convert bf16 to fp32" )) + newSumIdxV = sumIdxV - self.parentWriter.startVgprValuC + module.add(VMacF32(dst=vgpr("ValuC+%u"%newSumIdxV), src0=vgpr(tmpVgpr), src1=sgpr("Beta"), \ + comment="finalSum = sum*alpha + C*beta")) + elif kernel["ProblemType"]["DestDataType"].isSingle(): + newSumIdxV = sumIdxV - self.parentWriter.startVgprValuC + module.add(VMacF32(dst=vgpr("ValuC+%u"%newSumIdxV), src0=vgpr(dataV+0), src1=sgpr("Beta"), \ + comment="finalSum = sum*alpha + C*beta")) + + elif kernel["ProblemType"]["DestDataType"].isInt8(): + if kernel["ProblemType"]["HighPrecisionAccumulate"]: + assert (gwvw % 4 == 0) + if (vi%4) != 3: + tmpC = tmpVgpr + module.add(VBfeI32(dst=vgpr(tmpC), src0=vgpr(dataV+0), src1=(vi * 8), src2=8, comment="int8 to int32")) + else: + tmpC = dataV+0 + module.add(VAShiftRightI32(dst=vgpr(dataV+0), shiftHex=24, src=vgpr(dataV+0), comment="int8 to int32")) + newSumIdxV = sumIdxV - self.parentWriter.startVgprValuC + module.add(VMulLOU32(dst=vgpr(tmpC), src0=sgpr("Beta"), src1=vgpr(tmpC), comment="C = C*beta")) + module.add(VAddU32(dst=vgpr("ValuC+%u"%newSumIdxV), src0=vgpr(tmpC), src1=vgpr("ValuC+%u"%newSumIdxV), comment="finalSum = sum*alpha + C*beta")) + + elif kernel["ProblemType"]["DestDataType"].isInt32(): + # assume we will need to replace v_mac_f32 with v_add_u32 and s_mul_lo_i32 + # v_mad_i32_i24 + # module.add(VMadI32I24(dst=vgpr("ValuC+%u"%sumIdxV), src0=vgpr(dataV+0), src1=sgpr("Beta"), src2=vgpr("ValuC+%u"%sumIdxV), \ + # comment="finalSum = sum*alpha + C*beta")) + newSumIdxV = sumIdxV - self.parentWriter.startVgprValuC + module.add(VMulLOU32(dst=vgpr(dataV+0), src0=sgpr("Beta"), src1=vgpr(dataV+0), comment="C = C*beta")) + module.add(VAddU32(dst=vgpr("ValuC+%u"%newSumIdxV), src0=vgpr(dataV+0), src1=vgpr("ValuC+%u"%newSumIdxV), comment="finalSum = sum*alpha + C*beta")) + + elif kernel["ProblemType"]["DestDataType"].isDouble(): + newSumIdxV = sumIdxV * 2 - self.parentWriter.startVgprValuC + # dataV+0 = new c = old c*beta + module.add(VFmaF64(dst=vgpr("ValuC+%u"%(newSumIdxV),2), src0=vgpr(dataV+0,2), src1=sgpr("Beta",2), src2=vgpr("ValuC+%u"%(newSumIdxV),2), \ + comment="finalSum = sum*alpha + C*beta")) + + # single precision complex + elif kernel["ProblemType"]["DestDataType"].isSingleComplex(): + newSumIdxV = sumIdxV * 2 - self.parentWriter.startVgprValuC + module.add(VMacF32(dst=vgpr("ValuC+%u"%(newSumIdxV)), src0=vgpr(dataV+0), src1=sgpr("Beta"), comment="finalSum Cr += old Cr * Br")) + module.add(VMacF32(dst=vgpr("ValuC+%u"%(newSumIdxV)), src0=vgpr(dataV+1), src1=sgpr("Beta+1").getMinus(), comment="finalSum Cr += old Ci * -Bi")) + module.add(VMacF32(dst=vgpr("ValuC+%u"%(newSumIdxV+1)), src0=vgpr(dataV+1), src1=sgpr("Beta"), comment="finalSum Ci += old Ci * Br")) + module.add(VMacF32(dst=vgpr("ValuC+%u"%(newSumIdxV+1)), src0=vgpr(dataV+0), src1=sgpr("Beta+1"), comment="finalSum Ci += old Cr * Bi")) + + # double precision complex + elif kernel["ProblemType"]["DestDataType"].isDoubleComplex(): + newSumIdxV = sumIdxV * 4 - self.parentWriter.startVgprValuC + module.add(VFmaF64(dst=vgpr("ValuC+%u"%(newSumIdxV+0),2), src0=vgpr(dataV+0,2), src1=sgpr("Beta+0",2), src2=vgpr("ValuC+%u"%(newSumIdxV+0),2), comment="c.real += a.real * b.real")) + module.add(VFmaF64(dst=vgpr("ValuC+%u"%(newSumIdxV+0),2), src0=vgpr(dataV+2,2), src1=sgpr("Beta+2",2), src2=vgpr("ValuC+%u"%(newSumIdxV+0),2), comment="c.real -= a.imag * b.imag")) + module.add(VFmaF64(dst=vgpr("ValuC+%u"%(newSumIdxV+2),2), src0=vgpr(dataV+0,2), src1=sgpr("Beta+2",2), src2=vgpr("ValuC+%u"%(newSumIdxV+2),2), comment="c.imag += a.real * b.imag")) + module.add(VFmaF64(dst=vgpr("ValuC+%u"%(newSumIdxV+2),2), src0=vgpr(dataV+2,2), src1=sgpr("Beta+0",2), src2=vgpr("ValuC+%u"%(newSumIdxV+2),2), comment="c.imag += a.imag * b.real")) + return module diff --git a/tensilelite/Tensile/Components/LocalRead.py b/tensilelite/Tensile/Components/LocalRead.py new file mode 100644 index 0000000000..dda59755fd --- /dev/null +++ b/tensilelite/Tensile/Components/LocalRead.py @@ -0,0 +1,226 @@ +################################################################################ +# Copyright 2021-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..Component import LocalRead +from ..TensileInstructions import Module, DSModifiers, vgpr, sgpr, \ + SMovB32, SWaitCnt, VOrB32, VLShiftLeftOrB32 +from math import ceil + +class LocalReadMFMA(LocalRead): + kernel = {"EnableMatrixInstruction": True} + + """ + Local Read: Do It A/B + iui = Inner Unroll Idx + epsi = expand pointer swap index. Only used for PAP + """ + def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): + imod = Module("LocalReadDo%s_I%s" % (tP["tensorChar"],iui)) + + tc = tP["tensorChar"] + if tc == "A": + lrvw = writer.lrvwA + writer.localReadDoCntA += 1 + else: + lrvw = writer.lrvwB + writer.localReadDoCntB += 1 + tile01 = tP["tile01Idx"] + instruction = tP["localReadInstruction"] + + numOffsets = instruction.numOffsets + blockWidth = instruction.blockWidth + vectorWidth = kernel["VectorWidth"] if kernel["SourceSwap"] else 1 # TODO: nonSwap VectorWidth + vwB = writer.lrvwB if kernel["allowLRVWforTLUandMI"] else 1 + MIWaveGroupShape = [ kernel["MatrixInstM"] * kernel["MatrixInstBM"] * kernel["MIWaveGroup"][0] * vectorWidth, \ + kernel["MatrixInstN"] * kernel["MatrixInstBN"] * kernel["MIWaveGroup"][1] * vwB] + + LdsPad = kernel["LdsPad%s"%tc] if kernel["LdsBlockSizePerPad%s"%tc] == 0 else 0 + tileStride = 1 + UnrollStride = kernel["MacroTile%s" % tP["tensorChar"]] + LdsPad + if kernel["UnrollMajorLDS%s" % tP["tensorChar"]]: + tileStride = kernel["_DepthULds"] + LdsPad + UnrollStride = 1 + + numReadPerTileVector = vectorWidth if (tile01 == 0) else 1 + if (tile01 == 0) and kernel["allowLRVWforTLUandMI"] and numReadPerTileVector >= lrvw: + numReadPerTileVector //= lrvw + numVectorsPerTile = kernel["MIWaveTile"][tile01] // numReadPerTileVector + if kernel["allowLRVWforTLUandMI"] and numVectorsPerTile >= lrvw: + numVectorsPerTile //= lrvw + # overloading numReadsPerUnroll for DirectToLds x2/x4 case when blockWidth of instruction < LocalReadVectorWidth + # fp64 TLU=1 reading 0.5element/lane/read.. + # for TLU=0 case, blockWidth and LRVW should match + if tc == "A": + numReadsPerUnroll = tP["bpe"] * writer.lrvwA // int(blockWidth * 4) # bytes/register + else: + numReadsPerUnroll = tP["bpe"] * writer.lrvwB // int(blockWidth * 4) # bytes/register + numVgpr = int(ceil(blockWidth)) + + # pack register + needPack = blockWidth < 1 + pack = Module("pack%s_I%s"%(tc,iui)) + if needPack: + packTimesPerVgpr = int(1/blockWidth) - 1 # 0.5->pack once (16->32) / 0.25->pack three times (8->16, 8->16, 16->32) + tmpVgprIdx = writer.vgprPool.checkOut(writer.numVgprValuAPerBlock*writer.numReadsIterCoalescedA*packTimesPerVgpr if tc == 'A' \ + else writer.numVgprValuBPerBlock*writer.numReadsIterCoalescedB*packTimesPerVgpr) + pack.addTempVgpr(tmpVgprIdx) # important, add to pack Module for later CheckIn + + valufIdx = 0 + for vIdx in range(0, numVectorsPerTile): + for eIdx in range(0, numReadPerTileVector): + valuiIdx = int(valufIdx) + localReadCode = imod.add(Module("LocalRead%s Valu%u"%(tc,valuiIdx))) + if needPack: + packCode = pack.add(Module("packCode")) + for rIdx in range(0, numReadsPerUnroll): + valuiIdx = int(valufIdx) + baseLRVgpr = vgpr("Valu%s_X%u_I%u+%u"%(tc, bufferIdx, iui, valuiIdx), numVgpr) + destVgpr = baseLRVgpr + + # pack for blockWidth 0.5 type + highBitsForHalf = (blockWidth == 0.5) and ((rIdx % 2) == 1) # rIdx = 1 + if needPack and highBitsForHalf: + # highVgpr = vgpr(tmpVgprIdx + valuiIdx) + highVgpr = vgpr(tmpVgprIdx) + tmpVgprIdx += 1 + packCode.add(VOrB32(dst=destVgpr, src0=destVgpr, src1=highVgpr, comment="pack two half Vgpr to one Vgpr")) + destVgpr = highVgpr + + isHigh8Bits = (blockWidth == 0.25) and ( ((rIdx % 4) % 2) == 1) # 1,3 + isHigh16Bits = (blockWidth == 0.25) and ( ((rIdx % 4) //2) == 1) # 2,3 + if needPack: + if isHigh8Bits or isHigh16Bits: + highVgpr = vgpr(tmpVgprIdx) + destVgpr = highVgpr + if isHigh8Bits: + lowVgpr = vgpr(tmpVgprIdx-1) if isHigh16Bits else baseLRVgpr + packCode.add(VLShiftLeftOrB32(dst=lowVgpr, src0=highVgpr, shiftHex=hex(0x8), src1=lowVgpr, comment="pack two int8 Vgpr to one half Vgpr")) + if isHigh16Bits: + packCode.add(VOrB32(dst=baseLRVgpr, src0=baseLRVgpr, src1=lowVgpr, comment="pack two half Vgpr to one Vgpr")) + if isHigh8Bits or isHigh16Bits: + tmpVgprIdx += 1 + + valufIdx += blockWidth + + # load read instrution + paramList = [] + + for oIdx in range(0, numOffsets): + if (kernel["DirectToLds%s" % tP["tensorChar"]] and \ + kernel["GlobalLoadVectorWidth%c"%tc] * tP["bpe"] > 4): + # directToLds special case + divVal = 4 if kernel["ProblemType"]["DataType"].isDoubleComplex() else 2 + rIdxMod = rIdx % divVal + rIdxDiv = rIdx // divVal + offset_val = (eIdx + (vIdx * numOffsets+oIdx) * MIWaveGroupShape[tile01]) * tileStride + offset_val = (rIdxDiv * UnrollStride + offset_val + tP["localReadOffset"]) * tP["bpe"] + rIdxMod * writer.bpr + else: + # normal case + offset_val = (eIdx + (vIdx * numOffsets+oIdx) * MIWaveGroupShape[tile01]) * tileStride + offset_val = (rIdx * UnrollStride + offset_val + tP["localReadOffset"]) * tP["bpe"] + if (kernel["LdsBlockSizePerPad%s"%tc] != 0) and (kernel["LdsPad%s"%tc] != 0): + offset_val = offset_val + (offset_val // kernel["LdsBlockSizePerPad%s"%tc]) * kernel["LdsPad%s"%tc] * tP["bpe"] + offset_val = offset_val + tP["localReadSwapByteOffset"] + if (kernel["DirectToLds%s" % tc] and \ + kernel["GlobalLoadVectorWidth%c"%tc] * tP["bpe"] > 4): + + # another address conversion for DirectToLds + NumLoadsCoalesced > 1 + dummy, offset_val = writer.lraOffsetConversionForDTLandNLC(kernel, tP, offset_val) + + # offset conversion for DirectToLds + # TLU=0 case, modify bit3-6 of offset_val as follows + # (bit2<<3) | (bit3 <<1) | (bit4>>2) | (bit5>>2) + bit2 = offset_val & 4 + bit3 = offset_val & 8 + bit4 = offset_val & 16 + bit5 = offset_val & 32 + if (kernel["GlobalLoadVectorWidth%s"%tc] * tP["bpe"] == 8): + # dword_x2 case + # (bit2<<3) | (bit3 >>1) | (bit4>>1) | (bit5>>1) + newVal = (bit2<<3) | (bit3 >>1) | (bit4>>1) | (bit5>>1) + else: #if (kernel["GlobalLoadVectorWidth%s"%tc] * tP["bpe"] == 16): # most preferred case + # dword_x4 case + # (bit2<<3) | (bit3 <<1) | (bit4>>2) | (bit5>>2) + newVal = (bit2<<3) | (bit3 <<1) | (bit4>>2) | (bit5>>2) + offset_val = offset_val & (~0x3c) + offset_val = offset_val | newVal + + paramList.append(int(offset_val)) + + comment = "L -> Reg lro=%d swapByteOffset=%u ti=%u vIdx=%u rIdx=%u oIdx=%u buffer=%u iui=%u" \ + % (tP["localReadOffset"], tP["localReadSwapByteOffset"], MIWaveGroupShape[tile01], vIdx, rIdx, oIdx, bufferIdx, iui) + + highBits = highBitsForHalf or isHigh16Bits + readToTempVgpr = highBitsForHalf or isHigh8Bits or isHigh16Bits + + if numOffsets == 1: + ds = DSModifiers(na=1, offset=paramList[0]) + else: + ds = DSModifiers(na=2, offset0=paramList[0], offset1=paramList[1]) + LocalReadX = instruction.getInst(highBits) + localReadCode.add(LocalReadX(dst=destVgpr, src=vgpr("LocalReadAddr%s"%tc), readToTempVgpr=readToTempVgpr, ds=ds, comment=comment)) + + # TODO - handle vector-load + with writer.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + if writer.db["CheckValue1%s"%tc] and not writer.inTailLoop: + + dbgVgpr = destVgpr + dbgVgprList = destVgpr.split("v[") + if len(dbgVgprList) == 1: # vIdx, no [] + dbgVgpr = dbgVgprList[0] + else: + # We only check the first one now + # TODO: Handle vector, but need to take care the last one + dbgVgprList = (dbgVgprList[1].split("]")[0]).split(':') + dbgVgpr = "v[%s]"%dbgVgprList[0] + + localReadCode.add(SWaitCnt(lgkmcnt=0, vscnt=0, comment="CheckValue1 wait for LDS read")) + + if kernel["ProblemType"]["DataType"].isHalf(): + hexValue = hex(0x3c003c00) # packed 1s + if needPack: + hexValue = hex(0x3c000000) if highBitsForHalf else hex(0x00003c00) + localReadCode.add(SMovB32(dst=sgpr(tmpSgpr), src=hexValue, comment="CheckValue1: FP16")) + localReadCode.add(writer.assert_eq( dbgVgpr, sgpr(tmpSgpr))) + + elif kernel["ProblemType"]["DataType"].isBFloat16(): + hexValue = hex(0x3f803f80) # packed 1s + if needPack: + hexValue = hex(0x3f800000) if highBitsForHalf else hex(0x00003f80) + localReadCode.add(SMovB32(dst=sgpr(tmpSgpr), src=hexValue, comment="CheckValue1: BF16")) + localReadCode.add(writer.assert_eq( dbgVgpr, sgpr(tmpSgpr))) + + if kernel["ProblemType"]["DataType"].isInt8(): + if needPack: + hexValue = hex(0x00010000) if isHigh16Bits else hex(0x00000001) + localReadCode.add(SMovB32(dst=sgpr(tmpSgpr), src=hexValue, comment="CheckValue1: INT8")) + localReadCode.add(writer.assert_eq( dbgVgpr, sgpr(tmpSgpr))) + + # TODO - Check if this works. But need this? MFMA would use INT8 + elif kernel["ProblemType"]["DataType"].isInt8x4(): + localReadCode.add(SMovB32(dst=sgpr(tmpSgpr), src=hex(0x01010101), comment="CheckValue1: INT8x4")) + localReadCode.add(writer.assert_eq( dbgVgpr, sgpr(tmpSgpr))) + + elif kernel["ProblemType"]["DataType"].isSingle(): + localReadCode.add(writer.assert_eq( dbgVgpr, 1.0) ) + + return imod, pack diff --git a/tensilelite/Tensile/Components/LraTileAssignment.py b/tensilelite/Tensile/Components/LraTileAssignment.py new file mode 100644 index 0000000000..ec2d7a823b --- /dev/null +++ b/tensilelite/Tensile/Components/LraTileAssignment.py @@ -0,0 +1,132 @@ +################################################################################ +# Copyright 2021-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import Module, VAddU32, staticMultiply, vectorStaticDivide, \ + vectorStaticRemainder, RegisterPoolResource, vgpr +from ..Component import LraTileAssignment + +class LraTileAssignmentMFMA(LraTileAssignment): + kernel = {"EnableMatrixInstruction": True} + + """ + Local Read Addresses: Tile Assignment A/B + """ + def __call__(self, writer, kernel, tP): + module = Module("LraTileAssignmentMFMA") + module.addComment0("lr%s" % tP["tileChar"]) + + # alloc vgpr + wReg = writer.vgprPool.checkOut(1,"wReg") # quotient + tReg = writer.vgprPool.checkOut(1,"tReg") # remainder + kReg = writer.vgprPool.checkOut(1,"kReg") # remainder + tmpVgpr = writer.vgprPool.checkOutAligned(2,2,"tmpVgpr") + tmpVgprRes = RegisterPoolResource(tmpVgpr, 2) + ldsVgpr = writer.vgprPool.checkOut(1,"ldsVgpr") + ldsVgpr1 = writer.vgprPool.checkOut(1,"ldsVgpr1") + dummy = writer.vgprPool.checkOut(1,"dummy") + + # get constant parameter + tc = tP["tensorChar"] + tile01 = tP["tile01Idx"] + waveWidth = writer.kernel["WavefrontSize"] + inputPerThread = max(writer.lrvwA,writer.lrvwB) + if kernel["DirectToVgprA"]: + # DirectToVgprA case, ignore lrvwA + inputPerThread = writer.lrvwB + elif kernel["DirectToVgprB"]: + # DirectToVgprB case, ignore lrvwB + inputPerThread = writer.lrvwA + LdsPad = kernel["LdsPad%s" % tc] if kernel["LdsBlockSizePerPad%s" % tc] == 0 else 0 + + # parameter for get each type index + dividendForKId = kernel["MatrixInstM"] * kernel["MatrixInstB"] + num1DBlocks = kernel["MatrixInstBM"] if (tile01 == 0) else kernel["MatrixInstBN"] + num1DWaves = kernel["MIWaveGroup"][0] if (tile01 == 0) else kernel["MIWaveGroup"][1] + if kernel["SourceSwap"]: + dividedForBlkId = kernel["MatrixInstM"] if (tile01 == 0) else (kernel["MatrixInstM"] * kernel["MatrixInstBM"]) + else: + dividedForBlkId = (kernel["MatrixInstN"] * kernel["MatrixInstBN"]) if (tile01 == 0) else kernel["MatrixInstN"] + dividedForWaveId = waveWidth if (tile01 == 0) else (waveWidth * kernel["MIWaveGroup"][0]) + vectorWidth = kernel["VectorWidth"] if ((tile01 == 0) and kernel["SourceSwap"]) else 1 # TODO: nonSwap VectorWidth + if kernel["allowLRVWforTLUandMI"]: + lrvw = writer.lrvwA if tP["isA"] else writer.lrvwB + if lrvw > 1: + vectorWidth = lrvw + inputPerThread = 1 + + # strider for each type of index + umlds = kernel["UnrollMajorLDS%s" % tc] + mt = kernel["MacroTile%u" % tile01] + strideTile = kernel["_DepthULds"] + LdsPad if umlds else 1 + strideK = inputPerThread if umlds else (mt + LdsPad) * inputPerThread + strideBlock = kernel["MatrixInstM"] * strideTile + strideWave = kernel["MatrixInstM"] * num1DBlocks * strideTile * vectorWidth + + with writer.allocTmpSgpr(1) as tmpSgprInfo: + # tile offset + module.add(vectorStaticRemainder(dummy, kReg, "Serial", waveWidth, tmpVgprRes, tmpSgprInfo, \ + "0. thread id in wave: wtid = tid %% wavelength(%u)" % waveWidth)) + module.add(vectorStaticRemainder(dummy, tReg, kReg, kernel["MatrixInstN"], tmpVgprRes, tmpSgprInfo, \ + "1. N offset: nIdx = wtid %% MI_N(%u)" % kernel["MatrixInstN"])) + module.add(staticMultiply(vgpr(tReg), vgpr(tReg), strideTile, tmpSgprInfo, \ + "1. N offset: nOffset = nIdx * nStride(%u)" % strideTile)) + # block offset + module.add(vectorStaticDivide(wReg, kReg, dividedForBlkId, tmpVgprRes, \ + "2. block offset: bnIdx = wtid / dividedForBlkId(%u)" % dividedForBlkId)) + module.add(vectorStaticRemainder(dummy, wReg, wReg, num1DBlocks, tmpVgprRes, tmpSgprInfo, \ + "2. block offset: bnIdx = bnIdx %% num1DBlocks(%u)" % num1DBlocks)) + module.add(staticMultiply(vgpr(wReg), vgpr(wReg), strideBlock, tmpSgprInfo, \ + "2. block offset: bnOffset = bnIdx * strideBlock(%u)" % strideBlock)) + module.add(VAddU32(dst=vgpr(tReg), src0=vgpr(wReg), src1=vgpr(tReg), \ + comment="3. add N and block offset: bnOffset = block and N offset")) + module.add(staticMultiply(vgpr(tReg), vgpr(tReg), vectorWidth, tmpSgprInfo, \ + "3. apply VectorWidth: bnOffset = bnOffset * vw(%u)" % vectorWidth)) + + # unroll offset + module.add(vectorStaticDivide(kReg, kReg, dividendForKId, tmpVgprRes, \ + "4. K offset: kIdx = wtid / (MIN(%u) * MIBB(%u))" % (kernel["MatrixInstN"], kernel["MatrixInstB"]))) + module.add(staticMultiply(vgpr(kReg), vgpr(kReg), strideK, tmpSgprInfo, \ + "4. K offset: lrKOffset = kIdx * mStride(%u)" % strideK)) + + module.add(VAddU32(dst=vgpr(tReg), src0=vgpr(kReg), src1=vgpr(tReg), \ + comment="5. offset in wave: lrOffset = bnOffset + lrKOffset")) + + # wave offset + if num1DWaves > 1: + module.add(vectorStaticDivide(wReg, "Serial", dividedForWaveId, tmpVgprRes, \ + "6. wave offset in N dimen: wtid = tid / dividedForWaveId(%u)" % dividedForWaveId)) + module.add(vectorStaticRemainder(dummy, wReg, wReg, num1DWaves, tmpVgprRes, tmpSgprInfo, \ + "6. wave offset in M dimen: wtid0 = wtid / num1DWaves(%u)" % num1DWaves)) + module.add(staticMultiply(vgpr(wReg), vgpr(wReg), strideWave, tmpSgprInfo, \ + "6. wave offset in M dimen: wOffset = wtid0 * W0Stride(%u)" % strideWave)) + module.add(VAddU32(dst=vgpr(tReg), src0=vgpr(wReg), src1=vgpr(tReg), \ + comment="7. final local read offset: flrOffset = lrOffset + WOffset")) + + # release register + tP["gpr"]["lro"] = tReg + writer.vgprPool.checkIn(wReg) + writer.vgprPool.checkIn(kReg) + writer.vgprPool.checkIn(tmpVgpr) + writer.vgprPool.checkIn(ldsVgpr) + writer.vgprPool.checkIn(ldsVgpr1) + writer.vgprPool.checkIn(dummy) + + return module diff --git a/tensilelite/Tensile/Components/MAC_BF16_HPA.py b/tensilelite/Tensile/Components/MAC_BF16_HPA.py new file mode 100644 index 0000000000..a88b7a2f7c --- /dev/null +++ b/tensilelite/Tensile/Components/MAC_BF16_HPA.py @@ -0,0 +1,106 @@ +################################################################################ +# Copyright 2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import DataType, Module +from ..Component import Component, MAC + +class FMA_BF16_HPA(MAC): + asmCaps = {"v_fma_f32": True} + kernel = {"ProblemType": {"DataType": DataType(DataType.bfloat16), + "HighPrecisionAccumulate": True}} + + def __call__(self, writer, m, innerUnroll): + kernel = writer.kernel + module = Module("FMA_BF16_HPA") + module.addComment(self.commentHeader()) + priority = Component.Priority.find(writer) + + vars = {} + vars["m"] = m + vars["ThreadTile0"] = kernel["ThreadTile0"] + vars["ThreadTile0Half"] = kernel["ThreadTile0"] // 2 + + for iui in range(0, innerUnroll): + vars["iui"] = iui + for blockA in range(kernel["ThreadTileA"]//2-1, -1, -1): + vars["blockA"] = blockA + dst = "v[vgprValuA_X{m}_I{iui}+{blockA}*2+1]".format_map(vars) + src = "v[vgprValuA_X{m}_I{iui}+{blockA}]".format_map(vars) + module.addInst("v_and_b32", dst, "0xffff0000", src, "") + dst = "v[vgprValuA_X{m}_I{iui}+{blockA}*2]".format_map(vars) + module.addInst("v_lshlrev_b32", dst, 16, src, "") + for blockB in range(kernel["ThreadTileB"]//2-1, -1, -1): + vars["blockB"] = blockB + dst = "v[vgprValuB_X{m}_I{iui}+{blockB}*2+1]".format_map(vars) + src = "v[vgprValuB_X{m}_I{iui}+{blockB}]".format_map(vars) + module.addInst("v_and_b32", dst, "0xffff0000", src, "") + dst = "v[vgprValuB_X{m}_I{iui}+{blockB}*2]".format_map(vars) + module.addInst("v_and_b32", dst, 16, src, "") + + for block1 in range(0, kernel["ThreadTile1"]//2): + vars["block1"] = block1 + for block0 in range(0, kernel["ThreadTile0"]//2): + vars["block0"] = block0 + if kernel["ProblemType"]["HighPrecisionAccumulate"]: + # we treat HighPrecisionAccumulate as expanded packed math + for iui in range(0, innerUnroll): + vars["iui"] = iui + + vars["blockA"] = block0 if writer.tPB["tile01Idx"] else block1 + vars["blockB"] = block1 if writer.tPB["tile01Idx"] else block0 + + aStr0 = "v[vgprValuA_X{m}_I{iui}+{blockA}*2+0]".format_map(vars) + aStr1 = "v[vgprValuA_X{m}_I{iui}+{blockA}*2+1]".format_map(vars) + bStr0 = "v[vgprValuB_X{m}_I{iui}+{blockB}*2+0]".format_map(vars) + bStr1 = "v[vgprValuB_X{m}_I{iui}+{blockB}*2+1]".format_map(vars) + + cidx = block0*2 + block1*kernel["ThreadTile0"]*2 + 0 + cStr = "v[vgprValuC+{block0}*2+{block1}*{ThreadTile0}*2+0*2+0]".format_map(vars) + module.addInst("v_fma_f32", cStr, aStr0, bStr0, cStr, "ValuC[%u]" % cidx) + + module.add(priority(writer, 1, "Raise priority while processing macs")) + + aStr = aStr1 if writer.tPB["tile01Idx"] else aStr0 + bStr = bStr0 if writer.tPB["tile01Idx"] else bStr1 + cidx = block0*2 + block1*kernel["ThreadTile0"]*2 + 1 + cStr = "v[vgprValuC+{block0}*2+{block1}*{ThreadTile0}*2+0*2+1]".format_map(vars) # *2 b/c of fp32 + module.addInst("v_fma_f32", cStr, aStr, bStr, cStr, "ValuC[%u]" % cidx) + + aStr = aStr0 if writer.tPB["tile01Idx"] else aStr1 + bStr = bStr1 if writer.tPB["tile01Idx"] else bStr0 + cidx = block0*2 + block1*kernel["ThreadTile0"]*2 + kernel["ThreadTile0"] + 0 + cStr = "v[vgprValuC+{block0}*2+{block1}*{ThreadTile0}*2+{ThreadTile0Half}*2+0]".format_map(vars) + module.addInst("v_fma_f32", cStr, aStr, bStr, cStr, "ValuC[%u]" % cidx) + + cidx = block0*2 + block1*kernel["ThreadTile0"]*2 + kernel["ThreadTile0"] + 1 + cStr = "v[vgprValuC+{block0}*2+{block1}*{ThreadTile0}*2+{ThreadTile0Half}*2+1]".format_map(vars) + module.addInst("v_fma_f32", cStr, aStr1, bStr1, cStr, "ValuC[%u]" % cidx) + """ + ignore this, not quite correct for mixed precision + D.f[31:16] = S0.f[31:16] * S1.f[31:16] + S2.f[31:16] + D.f[15:00] = S0.f[15:00] * S1.f[15:00] + S2.f[15:00] + C[0] = A[0]*B[0]+D[0] + C[1] = A[1]*B[1]+D[1] + """ + #module.add(self.getBomb(-13)) + + module.add(priority(writer, 0, "Reset priority after macs")) + return module diff --git a/tensilelite/Tensile/Components/MAC_F16.py b/tensilelite/Tensile/Components/MAC_F16.py new file mode 100644 index 0000000000..e381aef4e0 --- /dev/null +++ b/tensilelite/Tensile/Components/MAC_F16.py @@ -0,0 +1,177 @@ +################################################################################ +# Copyright 2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import DataType, Module +from ..Component import Component, MAC + +class MAC_F16_Plain(MAC): + """ + Plain MAC instruction implementation + """ + asmCaps = {"v_mac_f16": True, + "v_pk_fma_f16": False, + "v_fma_f16": False} + #archCaps = {} + kernel = {"ProblemType": {"DataType": DataType(DataType.half), + "HighPrecisionAccumulate": False}} + + def __call__(self, writer, m, innerUnroll): + kernel = writer.kernel + + module = Module("MAC_F16_Plain") + module.addComment(self.commentHeader()) + + priority = Component.Priority.find(writer) + + vars = {} + + vars["m"] = m + vars["kernel"] = kernel + + vars["ThreadTile0"] = kernel["ThreadTile0"] + vars["ThreadTile1"] = kernel["ThreadTile1"] + + vars["Half_ThreadTile0"] = kernel["ThreadTile0"] // 2 + vars["Half_ThreadTile1"] = kernel["ThreadTile1"] // 2 + + for blockB in range(0, kernel["ThreadTile1"]//2): + for blockA in range(0, kernel["ThreadTile0"]//2): + for b in range(blockB*2, (blockB+1)*2): + for a in range(blockA*2, (blockA+1)*2): + for iui in range(0, innerUnroll): + vars["blockA"] = blockA + vars["blockB"] = blockB + vars["iui"] = iui + + cStr = "v[vgprValuC+{blockA}+{blockB}*{ThreadTile0}+0]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui}+{blockA}]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui}+{blockB}]".format_map(vars) + module.addInst("v_mac_f16", cStr, aStr, bStr, "") + module.add(priority(writer, 1, "Raise priority while processing macs")) + + module.add(priority(writer, 0, "Reset priority after macs")) + return module + + +class FMA_F16_NonPacked(MAC): + asmCaps = {"v_fma_f16": True, + "v_pk_fma_f16": False} + #archCaps = {} + kernel = {"ProblemType": {"DataType": DataType(DataType.half), + "HighPrecisionAccumulate": False}} + + def __call__(self, writer, m, innerUnroll): + kernel = writer.kernel + + module = Module("FMA_F16_NonPacked") + module.addComment(self.commentHeader()) + + priority = Component.Priority.find(writer) + + vars = {} + + vars["m"] = m + vars["kernel"] = kernel + + vars["ThreadTile0"] = kernel["ThreadTile0"] + vars["ThreadTile1"] = kernel["ThreadTile1"] + + vars["Half_ThreadTile0"] = kernel["ThreadTile0"] // 2 + vars["Half_ThreadTile1"] = kernel["ThreadTile1"] // 2 + + for blockB in range(0, kernel["ThreadTile1"]//2): + for blockA in range(0, kernel["ThreadTile0"]//2): + for iui in range(0, innerUnroll): + vars["blockA"] = blockA + vars["blockB"] = blockB + vars["iui"] = iui + + cIdxExpr0 = "{blockA} + {blockB}*{ThreadTile0} + 0".format_map(vars) + cStr0 = "v[vgprValuC + {}]".format(eval(cIdxExpr0)) + + # /2 b/c of 2 f16's per 32-bit vgpr + cIdxExpr1 = "{blockA} + {blockB}*{ThreadTile0} + {Half_ThreadTile0}".format_map(vars) + cStr1 = "v[vgprValuC + {}]".format(eval(cIdxExpr1)) + + aStr = "v[vgprValuA_X{m}_I{iui} + {blockA}]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui} + {blockB}]".format_map(vars) + + module.addInst("v_fma_f16", cStr0, aStr, bStr, cStr0, "op_sel:[0,0,0,0]", cIdxExpr0) + module.add(priority(writer, 1, "Raise priority while processing macs")) + module.addInst("v_fma_f16", cStr1, aStr, bStr, cStr1, "op_sel:[0,1,0,0]", cIdxExpr1) + module.addInst("v_fma_f16", cStr0, aStr, bStr, cStr0, "op_sel:[1,0,1,1]", cIdxExpr0) + module.addInst("v_fma_f16", cStr1, aStr, bStr, cStr1, "op_sel:[1,1,1,1]", cIdxExpr1) + + module.add(priority(writer, 0, "Reset priority after macs")) + return module + +class FMA_F16_Packed(MAC): + asmCaps = {"v_pk_fma_f16": True} + #archCaps = {} + kernel = {"ProblemType": {"DataType": DataType(DataType.half), + "HighPrecisionAccumulate": False}} + + def __call__(self, writer, m, innerUnroll): + kernel = writer.kernel + + module = Module("FMA_F16_Packed") + module.addComment(self.commentHeader()) + + priority = Component.Priority.find(writer) + + vars = {} + + vars["m"] = m + vars["kernel"] = kernel + + vars["ThreadTile0"] = kernel["ThreadTile0"] + vars["ThreadTile1"] = kernel["ThreadTile1"] + + vars["Half_ThreadTile0"] = kernel["ThreadTile0"] // 2 + vars["Half_ThreadTile1"] = kernel["ThreadTile1"] // 2 + + for blockB in range(0, kernel["ThreadTile1"]//2): + for blockA in range(0, kernel["ThreadTile0"]//2): + for iui in range(0, innerUnroll): + vars["blockA"] = blockA + vars["blockB"] = blockB + vars["iui"] = iui + + cIdxExpr = "{blockA} + {blockB}*{ThreadTile0} + 0".format_map(vars) + + # /2 b/c of 2 f16's per 32-bit vgpr + cStr = "v[vgprValuC + {}]".format(eval(cIdxExpr)) + + aStr = "v[vgprValuA_X{m}_I{iui} + {blockA}]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui} + {blockB}]".format_map(vars) + + module.addInst("v_pk_fma_f16", cStr, aStr, bStr, cStr, "op_sel:[0,0,0]", "op_sel_hi:[1,0,1]", cIdxExpr) + module.add(priority(writer, 1, "Raise priority while processing macs")) + + cIdxExpr = "{blockA} + {blockB}*{ThreadTile0} + {Half_ThreadTile0}".format_map(vars) + cIdxVal = eval(vars["cIdxExpr"]) + + cStr = "v[vgprValuC + {cIdxExpr}]".format_map(vars) + + module.addInst("v_pk_fma_f16", cStr, aStr, bStr, cStr, "op_sel:[0,1,0]", "op_sel_hi:[1,1,1]", cIdxExpr) + + module.add(priority(writer, 0, "Reset priority after macs")) + return module diff --git a/tensilelite/Tensile/Components/MAC_F16_HPA.py b/tensilelite/Tensile/Components/MAC_F16_HPA.py new file mode 100644 index 0000000000..9a55c9d111 --- /dev/null +++ b/tensilelite/Tensile/Components/MAC_F16_HPA.py @@ -0,0 +1,95 @@ +################################################################################ +# Copyright 2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import DataType, Module +from ..Component import Component, MAC + +class FMA_F16_HPA_MAD_MIX(MAC): + asmCaps = lambda caps: caps['v_mad_mix_f32'] or caps['v_fma_mix_f32'] + #archCaps = {} + kernel = {"ProblemType": {"DataType": DataType(DataType.half), + "HighPrecisionAccumulate": True}, + } + + def __call__(self, writer, m, innerUnroll): + kernel = writer.kernel + + module = Module("FMA_F16_HPA_MAD_MIX") + module.addComment(self.commentHeader()) + priority = Component.Priority.find(writer) + + vars = {} + + if writer.asmCaps["v_fma_mix_f32"]: + instruction = "v_fma_mix_f32" + else: + instruction = "v_mad_mix_f32" + + vars["m"] = m + vars["kernel"] = kernel + + vars["ThreadTile0"] = kernel["ThreadTile0"] + vars["ThreadTile1"] = kernel["ThreadTile1"] + + vars["Half_ThreadTile0"] = kernel["ThreadTile0"] // 2 + vars["Half_ThreadTile1"] = kernel["ThreadTile1"] // 2 + + for block1 in range(0, kernel["ThreadTile1"]//2): + for block0 in range(0, kernel["ThreadTile0"]//2): + for iui in range(0, innerUnroll): + vars["block0"] = block0 + vars["block1"] = block1 + vars["blockA"] = block0 if writer.tPA["tileIdx"] == 0 else block1 + vars["blockB"] = block1 if writer.tPB["tileIdx"] != 0 else block0 + vars["iui"] = iui + + vars["aBase"] = "vgprValuA_X{m}_I{iui}".format_map(vars) + vars["bBase"] = "vgprValuB_X{m}_I{iui}".format_map(vars) + + vars["cIdxExpr"] = "{block0}*2 + {block1}*{ThreadTile0}*2 + 0*2 + 0".format_map(vars) + cidx = eval(vars["cIdxExpr"]) + cStr = "v[vgprValuC + {cIdxExpr}]".format_map(vars) # *2 b/c of fp32 + aStr = "v[{aBase}+{blockA}]".format_map(vars) + bStr = "v[{bBase}+{blockB}]".format_map(vars) + module.addInst(instruction, cStr, aStr, bStr, cStr, "op_sel:[0,0,0]", "op_sel_hi:[1,1,0]", "ValuC[%u] iui=%u" % (cidx, vars["iui"])) + + module.add(priority(writer, 1, "Raise priority while processing macs")) + + vars["cIdxExpr"] = "{block0}*2 + {block1}*{ThreadTile0}*2 + 0*2 + 1".format_map(vars) + cidx = eval(vars["cIdxExpr"]) + cStr = "v[vgprValuC + {cIdxExpr}]".format_map(vars) # *2 b/c of fp32 + opSel = "op_sel:[1,0,0]" if writer.tPA["tileIdx"] == 0 else "op_sel:[0,1,0]" + module.addInst(instruction, cStr, aStr, bStr, cStr, opSel, "op_sel_hi:[1,1,0]", "ValuC[%u]" % cidx) + + vars["cIdxExpr"] = "{block0}*2 + {block1}*{ThreadTile0}*2 + {Half_ThreadTile0}*2 + 0".format_map(vars) + cidx = eval(vars["cIdxExpr"]) + cStr = "v[vgprValuC+{cIdxExpr}]".format_map(vars) + opSel = "op_sel:[0,1,0]" if writer.tPA["tileIdx"] == 0 else "op_sel:[1,0,0]" + module.addInst(instruction, cStr, aStr, bStr, cStr, opSel, "op_sel_hi:[1,1,0]", "ValuC[%u]" % cidx) + + vars["cIdxExpr"] = "{block0}*2+{block1}*{ThreadTile0}*2+{Half_ThreadTile0}*2+1".format_map(vars) + cidx = eval(vars["cIdxExpr"]) + cStr = "v[vgprValuC+{cIdxExpr}]".format_map(vars) + module.addInst(instruction, cStr, aStr, bStr, cStr, "op_sel:[1,1,0]", "op_sel_hi:[1,1,0]", "ValuC[%u]" % cidx) + + module.add(priority(writer, 0, "Reset priority after macs")) + + return module diff --git a/tensilelite/Tensile/Components/MAC_F32.py b/tensilelite/Tensile/Components/MAC_F32.py new file mode 100644 index 0000000000..b81363692b --- /dev/null +++ b/tensilelite/Tensile/Components/MAC_F32.py @@ -0,0 +1,87 @@ +################################################################################ +# Copyright 2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import DataType, Module +from ..Component import Component, MAC + +class MAC_F32_Plain(MAC): + """ + Plain MAC instruction implementation + """ + @staticmethod + def asmCaps(caps): + return caps["v_mac_f32"] or caps["v_fma_f32"] + #archCaps = {} + kernel = {"ProblemType": {"DataType": DataType(DataType.single)}} + + def __call__(self, writer, m, innerUnroll): + kernel = writer.kernel + + if writer.asmCaps["v_fmac_f32"]: + instruction = "v_fmac_f32" + elif writer.asmCaps["v_fma_f32"]: + instruction = "v_fma_f32" + elif writer.asmCaps["v_mac_f32"]: + instruction = "v_mac_f32" + else: + raise RuntimeError("FMA and MAC instructions are not supported on {}".format(kernel["ISA"])) + + if not writer.asmCaps[instruction]: + raise RuntimeError("{} instruction specified but not supported on {}".format(instruction, kernel["ISA"])) + + module = Module("MAC_F32_Plain") + module.addComment(self.commentHeader()) + + vars = {} + + vars["m"] = m + vars["kernel"] = kernel + + vars["ThreadTile0"] = kernel["ThreadTile0"] + vars["ThreadTile1"] = kernel["ThreadTile1"] + + vars["instruction"] = instruction + + priority = Component.Priority.find(writer) + + for idx1 in range(0, kernel["ThreadTile1"]): + for idx0 in range(0, kernel["ThreadTile0"]): + for iui in range(0, innerUnroll): + vars["idx0"] = idx0 + vars["idx1"] = idx1 + vars["a"] = idx0 if writer.tPB["tile01Idx"] else idx1 + vars["b"] = idx1 if writer.tPB["tile01Idx"] else idx0 + vars["iui"] = iui + + cStr = "v[vgprValuC + {idx0} + {idx1}*{ThreadTile0}]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui} + {a}]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui} + {b}]".format_map(vars) + + if instruction == "v_fma_f32": + module.addInst("v_fma_f32", cStr, aStr, bStr, cStr, "") + else: + module.addInst(instruction, cStr, aStr, bStr, "") + + module.add(priority(writer, 1, "Raise priority while processing macs")) + + module.add(priority(writer, 0, "Reset priority after macs")) + + return module diff --git a/tensilelite/Tensile/Components/MAC_F32C.py b/tensilelite/Tensile/Components/MAC_F32C.py new file mode 100644 index 0000000000..3faa56bba7 --- /dev/null +++ b/tensilelite/Tensile/Components/MAC_F32C.py @@ -0,0 +1,71 @@ +################################################################################ +# Copyright 2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import DataType, Module +from ..Component import Component, MAC + +class MAC_F32C_Plain(MAC): + kernel = {"ProblemType": {"DataType": DataType(DataType.complexSingle)}} + + def __call__(self, writer, m, innerUnroll): + kernel = writer.kernel + module = Module("MAC_F32C_Plain") + module.addComment(self.commentHeader()) + priority = Component.Priority.find(writer) + + vars = {} + vars["m"] = m + vars["ThreadTile0"] = kernel["ThreadTile0"] + + for b in range(0, kernel["ThreadTile1"]): + for a in range(0, kernel["ThreadTile0"]): + for iui in range(0, innerUnroll): + vars["a"] = a + vars["b"] = b + vars["iui"] = iui + + cStr = "v[vgprValuC+({a}+{b}*{ThreadTile0})*2]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui}+{a}*2]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui}+{b}*2]".format_map(vars) + module.addInst("_v_mac_f32", cStr, aStr, bStr, "") + + cStr = "v[vgprValuC+({a}+{b}*{ThreadTile0})*2]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui}+{a}*2+1]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui}+{b}*2+1]".format_map(vars) + sign = "-" if (not kernel["ProblemType"]["ComplexConjugateA"] and not kernel["ProblemType"]["ComplexConjugateB"]) or \ + (kernel["ProblemType"]["ComplexConjugateA"] and kernel["ProblemType"]["ComplexConjugateB"]) else "" + module.addInst("_v_mac_f32", cStr, sign + aStr, bStr, "") + + cStr = "v[vgprValuC+({a}+{b}*{ThreadTile0})*2+1]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui}+{a}*2]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui}+{b}*2+1]".format_map(vars) + sign = "-" if kernel["ProblemType"]["ComplexConjugateB"] else "" + module.addInst("_v_mac_f32", cStr, aStr, sign + bStr, "") + + cStr = "v[vgprValuC+({a}+{b}*{ThreadTile0})*2+1]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui}+{a}*2+1]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui}+{b}*2]".format_map(vars) + sign = "-" if kernel["ProblemType"]["ComplexConjugateA"] else "" + module.addInst("_v_mac_f32", cStr, sign + aStr, bStr, "") + + module.add(priority(writer, 1, "Raise priority while processing macs")) + module.add(priority(writer, 0, "Reset priority after macs")) + return module diff --git a/tensilelite/Tensile/Components/MAC_F64.py b/tensilelite/Tensile/Components/MAC_F64.py new file mode 100644 index 0000000000..9be852b3e3 --- /dev/null +++ b/tensilelite/Tensile/Components/MAC_F64.py @@ -0,0 +1,52 @@ +################################################################################ +# Copyright 2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import DataType, Module +from ..Component import Component, MAC + +class FMA_F64_Plain(MAC): + asmCaps = {"v_fma_f64": True} + kernel = {"ProblemType": {"DataType": DataType(DataType.double)}} + + def __call__(self, writer, m, innerUnroll): + kernel = writer.kernel + module = Module("FMA_F64_Plain") + module.addComment(self.commentHeader()) + priority = Component.Priority.find(writer) + + vars = {} + vars["m"] = m + vars["ThreadTile0"] = kernel["ThreadTile0"] + + for b in range(0, kernel["ThreadTile1"]): + vars["b"] = b + for a in range(0, kernel["ThreadTile0"]): + vars["a"] = a + for iui in range(0, innerUnroll): + vars["iui"] = iui + cStr = "v[vgprValuC+({a}+{b}*{ThreadTile0})*2:(vgprValuC+{a}+{b}*{ThreadTile0})*2+1]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui}+{a}*2:vgprValuA_X{m}_I{iui}+{a}*2+1]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui}+{b}*2:vgprValuB_X{m}_I{iui}+{b}*2+1]".format_map(vars) + module.addInst("v_fma_f64", cStr, aStr, bStr, cStr, "") + module.add(priority(writer, 1, "Raise priority while processing macs")) + + module.add(priority(writer, 0, "Reset priority after macs")) + return module diff --git a/tensilelite/Tensile/Components/MAC_F64C.py b/tensilelite/Tensile/Components/MAC_F64C.py new file mode 100644 index 0000000000..c2130de783 --- /dev/null +++ b/tensilelite/Tensile/Components/MAC_F64C.py @@ -0,0 +1,73 @@ +################################################################################ +# Copyright 2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import DataType, Module +from ..Component import Component, MAC + +class FMA_F64C_Plain(MAC): + asmCaps = {"v_fma_f64": True} + kernel = {"ProblemType": {"DataType": DataType(DataType.complexDouble)}} + + def __call__(self, writer, m, innerUnroll): + kernel = writer.kernel + module = Module("FMA_F64C_Plain") + module.addComment(self.commentHeader()) + priority = Component.Priority.find(writer) + + vars = {} + vars["m"] = m + vars["ThreadTile0"] = kernel["ThreadTile0"] + + for b in range(0, kernel["ThreadTile1"]): + vars["b"] = b + for a in range(0, kernel["ThreadTile0"]): + vars["a"] = a + for iui in range(0, innerUnroll): + vars["iui"] = iui + # c.real += a.real * b.real + cStr = "v[vgprValuC+({a}+{b}*{ThreadTile0})*4+0:(vgprValuC+{a}+{b}*{ThreadTile0})*4+1]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui}+{a}*4+0:vgprValuA_X{m}_I{iui}+{a}*4+1]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui}+{b}*4+0:vgprValuB_X{m}_I{iui}+{b}*4+1]".format_map(vars) + module.addInst("v_fma_f64", cStr, aStr, bStr, cStr, "") + # c.real -= a.imag * b.imag + cStr = "v[vgprValuC+({a}+{b}*{ThreadTile0})*4+0:(vgprValuC+{a}+{b}*{ThreadTile0})*4+1]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui}+{a}*4+2:vgprValuA_X{m}_I{iui}+{a}*4+3]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui}+{b}*4+2:vgprValuB_X{m}_I{iui}+{b}*4+3]".format_map(vars) + sign = "-" if (not kernel["ProblemType"]["ComplexConjugateA"] and not kernel["ProblemType"]["ComplexConjugateB"]) or \ + (kernel["ProblemType"]["ComplexConjugateA"] and kernel["ProblemType"]["ComplexConjugateB"]) else "" + module.addInst("v_fma_f64", cStr, aStr, sign + bStr, cStr, "") + # c.imag += a.real * b.imag + cStr = "v[vgprValuC+({a}+{b}*{ThreadTile0})*4+2:(vgprValuC+{a}+{b}*{ThreadTile0})*4+3]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui}+{a}*4+0:vgprValuA_X{m}_I{iui}+{a}*4+1]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui}+{b}*4+2:vgprValuB_X{m}_I{iui}+{b}*4+3]".format_map(vars) + sign = "-" if kernel["ProblemType"]["ComplexConjugateB"] else "" + module.addInst("v_fma_f64", cStr, aStr, sign + bStr, cStr, "") + # c.imag += a.imag * b.real + cStr = "v[vgprValuC+({a}+{b}*{ThreadTile0})*4+2:(vgprValuC+{a}+{b}*{ThreadTile0})*4+3]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui}+{a}*4+2:vgprValuA_X{m}_I{iui}+{a}*4+3]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui}+{b}*4+0:vgprValuB_X{m}_I{iui}+{b}*4+1]".format_map(vars) + sign = "-" if kernel["ProblemType"]["ComplexConjugateA"] else "" + module.addInst("v_fma_f64", cStr, sign + aStr, bStr, cStr, "") + + module.add(priority(writer, 1, "Raise priority while processing macs")) + + module.add(priority(writer, 0, "Reset priority after macs")) + return module diff --git a/tensilelite/Tensile/Components/MAC_I8X4.py b/tensilelite/Tensile/Components/MAC_I8X4.py new file mode 100644 index 0000000000..492c76d9bd --- /dev/null +++ b/tensilelite/Tensile/Components/MAC_I8X4.py @@ -0,0 +1,53 @@ +################################################################################ +# Copyright 2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import DataType, Module +from ..Component import Component, MAC + +class MAC_I8X4_Plain(MAC): + asmCaps = {"VOP3v_dot4_i32_i8": True} + kernel = {"ProblemType": {"DataType": DataType(DataType.int8x4)}} + + def __call__(self, writer, m, innerUnroll): + kernel = writer.kernel + module = Module("MAC_I8X4_Plain") + module.addComment(self.commentHeader()) + priority = Component.Priority.find(writer) + + vars = {} + vars["m"] = m + vars["ThreadTile0"] = kernel["ThreadTile0"] + + for b in range(0, kernel["ThreadTile1"]): + vars["b"] = b + for a in range(0, kernel["ThreadTile0"]): + vars["a"] = a + for iui in range(0, innerUnroll): + vars["iui"] = iui + cidx = a + b*kernel["ThreadTile0"] + 0 + cStr = "v[vgprValuC+{a}+{b}*{ThreadTile0}]".format_map(vars) + aStr = "v[vgprValuA_X{m}_I{iui}+{a}]".format_map(vars) + bStr = "v[vgprValuB_X{m}_I{iui}+{b}]".format_map(vars) + module.addInst("v_dot4_i32_i8", cStr, aStr, bStr, cStr, "op_sel:[0,0]", "op_sel_hi:[1,1]", "valuC[%u]" % cidx) + module.add(priority(writer, 1, "Raise priority while processing macs")) + + module.add(priority(writer, 0, "Reset priority after macs")) + return module diff --git a/tensilelite/Tensile/Components/MAC_I8_HPA.py b/tensilelite/Tensile/Components/MAC_I8_HPA.py new file mode 100644 index 0000000000..d2c242f6c6 --- /dev/null +++ b/tensilelite/Tensile/Components/MAC_I8_HPA.py @@ -0,0 +1,75 @@ +################################################################################ +# Copyright 2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import DataType, Module +from ..Component import Component, MAC + +class FMA_I8_HPA(MAC): + @staticmethod + def asmCaps(caps): + return True + + kernel = { + "ProblemType": {"DataType": DataType(DataType.int8), "HighPrecisionAccumulate": True}, + } + + def __call__(self, writer, m, innerUnroll): + kernel = writer.kernel + priority = Component.Priority.find(writer) + spacePerReg = writer.bpr // writer.bpeAB + elemPerReg = min(kernel['VectorWidth'], spacePerReg) + + module = Module("FMA_I8_HPA") + module.addComment(self.commentHeader()) + + for a in range(kernel["ThreadTile0"]-1, -1, -1): + for iui in range(0, innerUnroll): + src = a // elemPerReg + idx = a % elemPerReg + sStr = f'v[vgprValuA_X{m}_I{iui}+{src}]' + tStr = f'v[vgprValuA_X{m}_I{iui}+{a}]' + module.addInst("v_lshlrev_b32", tStr, {(spacePerReg-idx-1)*8}, sStr, "") + module.add(priority(writer, 1, "Raise priority while processing macs")) + module.addInst("v_ashrrev_i32", tStr, {(spacePerReg -1)*8}, tStr, "") + + for b in range(kernel["ThreadTile1"]-1, -1, -1): + for iui in range(0, innerUnroll): + src = b // elemPerReg + idx = b % elemPerReg + sStr = f'v[vgprValuB_X{m}_I{iui}+{src}]' + tStr = f'v[vgprValuB_X{m}_I{iui}+{b}]' + module.addInst("v_lshlrev_b32", tStr, {(spacePerReg-idx-1)*8}, sStr, "") + module.add(priority(writer, 1, "Raise priority while processing macs")) + module.addInst("v_ashrrev_i32", tStr, {(spacePerReg -1)*8}, tStr, "") + + ThreadTile0 = kernel["ThreadTile0"] + for b in range(0, kernel["ThreadTile1"]): + for a in range(0, kernel["ThreadTile0"]): + for iui in range(0, innerUnroll): + cStr = f'v[vgprValuC + {a} + {b}*{ThreadTile0}]' + aStr = f'v[vgprValuA_X{m}_I{iui} + {a}]' + bStr = f'v[vgprValuB_X{m}_I{iui} + {b}]' + module.addInst("v_mad_i32_i24", cStr, aStr, bStr, cStr, "") + module.add(priority(writer, 1, "Raise priority while processing macs")) + + module.add(priority(writer, 0, "Reset priority after macs")) + + return module diff --git a/tensilelite/Tensile/Components/NotLocalFullTileElements.py b/tensilelite/Tensile/Components/NotLocalFullTileElements.py new file mode 100644 index 0000000000..b8b3a9cc39 --- /dev/null +++ b/tensilelite/Tensile/Components/NotLocalFullTileElements.py @@ -0,0 +1,68 @@ +################################################################################ +# Copyright 2021-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..Component import NotLocalFullTileElements + +class NotLocalFullTileElementsMFMA(NotLocalFullTileElements): + kernel = {"EnableMatrixInstruction": True} + + """ + Partition thread-tile into writeElements for store code + This function creates the writeElement mapping for full tiles + (ie non-edge cases) + """ + def __call__(self, writer, kernel, edge): + elements = [] + storeVectorWidth = 0 + + if edge: + storeVectorWidth = kernel["StoreVectorWidth"] if kernel["_VectorStore"] else 1 + storeVectorWidth = min(storeVectorWidth, writer.maxGwvw(kernel), kernel["AssertFree0ElementMultiple"]) + else: + storeVectorWidth = kernel["StoreVectorWidth"] if kernel["_VectorStore"] else 1 + storeVectorWidth = min(storeVectorWidth, writer.maxGwvw(kernel)) + + # handle mfma 4x4 instruction + matrixInstM = kernel["MatrixInstM"] * kernel["MatrixInstBM"] if (kernel["MatrixInstM"] == 4) else kernel["MatrixInstM"] + matrixInstN = kernel["MatrixInstN"] * kernel["MatrixInstBN"] if (kernel["MatrixInstN"] == 4) else kernel["MatrixInstN"] + matrixInstBM = 1 if (kernel["MatrixInstM"] == 4) else kernel["MatrixInstBM"] + matrixInstBN = 1 if (kernel["MatrixInstN"] == 4) else kernel["MatrixInstBN"] + + outputsPerThread = matrixInstM * matrixInstN // kernel["WavefrontSize"] + + # handle SourceSwap + totalTT0 = matrixInstBM * kernel["MIWaveTile"][0] + totalTT1 = matrixInstBN * kernel["MIWaveTile"][1] + + totalTT0 = totalTT0 if kernel["SourceSwap"] else (totalTT0 * outputsPerThread) + totalTT1 = (totalTT1 * outputsPerThread) if kernel["SourceSwap"] else totalTT1 + vectorWidth0 = kernel["VectorWidth"] if kernel["SourceSwap"] else kernel["MIOutputVectorWidth"] + MIOutputVectorWidthAdj = writer.lrvwB if kernel["allowLRVWforTLUandMI"] else kernel["MIOutputVectorWidth"] + vectorWidth1 = MIOutputVectorWidthAdj if kernel["SourceSwap"] else 1 + + for tt1 in range(0, totalTT1//vectorWidth1): + for vc1 in range(0, vectorWidth1): + for tt0 in range(0, totalTT0//vectorWidth0): + for vc0 in range(0, vectorWidth0, storeVectorWidth): # note step by storeVectorWidth + element = (tt1, tt0, vc1, vc0) + elements.append(element) + + return (storeVectorWidth, elements) diff --git a/tensilelite/Tensile/Components/PackData.py b/tensilelite/Tensile/Components/PackData.py new file mode 100644 index 0000000000..7895ef2447 --- /dev/null +++ b/tensilelite/Tensile/Components/PackData.py @@ -0,0 +1,115 @@ +################################################################################ +# Copyright 2021-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import Module, SDWAModifiers, SelectBit, UnusedBit, \ + SaturateCastType, VSaturateCastInt, \ + VAdd3U32, VCvtF32toF16, VLShiftRightB32, \ + VCmpUF32, VCndMaskB32, \ + VOrB32, VPackF16toB32, \ + VAndOrB32, VBfeU32, VLShiftLeftB16, \ + vgpr, sgpr, DataType +from ..Component import PackData + +def formatting(idx, inputPrefix, prefixOffset): + if inputPrefix: + return (inputPrefix + "%u"%(idx-prefixOffset)) + else: + return idx + +class PackData_F16(PackData): + kernel = {"ProblemType": {"DestDataType": DataType(DataType.half)}} + def __call__(self, gwvw, elementSumIdx, inputPrefix="", prefixOffset=0): + module = Module("PackData F16") + if gwvw == 1: + formatVgpr = formatting(elementSumIdx, inputPrefix, prefixOffset) + module.add(VCvtF32toF16(dst=vgpr(formatVgpr), src=vgpr(formatVgpr), comment="convert C to fp16")) + return module + + assert (gwvw % 2 == 0) + for vi in range(0, gwvw): + sumIdxV = elementSumIdx + vi + formatVgpr = formatting(sumIdxV, inputPrefix, prefixOffset) + module.add(VCvtF32toF16(dst=vgpr(formatVgpr), src=vgpr(formatVgpr), comment="convert C to fp16")) + if vi%2 == 1: + d = elementSumIdx + vi//2 + module.add(VPackF16toB32(dst=vgpr(d), src0=vgpr(formatting(sumIdxV-1, inputPrefix, prefixOffset)), src1=vgpr(formatVgpr), \ + comment="Pack with neighbor")) + return module + +class PackData_BF16(PackData): + kernel = {"ProblemType": {"DestDataType": DataType(DataType.bfloat16)}} + def __call__(self, gwvw, elementSumIdx, bf16CVTVgprStruct, tmpS01, laneSGPRC, inputPrefix="", prefixOffset=0): + vgprBf16Temp = bf16CVTVgprStruct.vgprBf16Temp + vgprBf16Inc = bf16CVTVgprStruct.vgprBf16Inc + vgprFp32Nan = bf16CVTVgprStruct.vgprFp32Nan + vgprBf16Mask = bf16CVTVgprStruct.vgprBf16Mask + + module = Module("PackData BF16") + if gwvw == 1: + formatVgpr = formatting(elementSumIdx, inputPrefix, prefixOffset) + module.add(VCmpUF32(dst=sgpr(tmpS01,laneSGPRC), src0=vgpr(formatVgpr), src1=vgpr(formatVgpr), comment="check Nan")) + module.add(VBfeU32(dst=vgpr(vgprBf16Temp), src0=vgpr(formatVgpr), src1=16, src2=1, comment="Non-Nan case: store lsb of bf16" )) + module.add(VAdd3U32(dst=vgpr(vgprBf16Temp), src0=vgpr(formatVgpr), src1=vgpr(vgprBf16Temp), src2=vgpr(vgprBf16Inc), comment="Non-Nan case: add lsb and the increment for rounding" )) + module.add(VCndMaskB32(dst=vgpr(formatVgpr), src0=vgpr(vgprBf16Temp), src1=vgpr(vgprFp32Nan), src2=sgpr(tmpS01,laneSGPRC))) + module.add(VLShiftRightB32(dst=vgpr(formatVgpr), shiftHex=16, src=vgpr(formatVgpr), comment="convert C to bf16")) + return module + + assert (gwvw % 2 == 0) + for vi in range(0, gwvw): + sumIdxV = elementSumIdx + vi + formatVgpr = formatting(sumIdxV, inputPrefix, prefixOffset) + module.add(VCmpUF32(dst=sgpr(tmpS01,laneSGPRC), src0=vgpr(formatVgpr), src1=vgpr(formatVgpr), comment="check Nan")) + module.add(VBfeU32(dst=vgpr(vgprBf16Temp), src0=vgpr(formatVgpr), src1=16, src2=1, comment="Non-Nan case: store lsb of bf16" )) + module.add(VAdd3U32(dst=vgpr(vgprBf16Temp), src0=vgpr(formatVgpr), src1=vgpr(vgprBf16Temp), src2=vgpr(vgprBf16Inc), comment="Non-Nan case: add lsb and the increment for rounding" )) + module.add(VCndMaskB32(dst=vgpr(formatVgpr), src0=vgpr(vgprBf16Temp), src1=vgpr(vgprFp32Nan), src2=sgpr(tmpS01,laneSGPRC))) + if vi%2 == 0: + module.add(VLShiftRightB32(dst=vgpr(formatVgpr), shiftHex=16, src=vgpr(formatVgpr), comment="convert C to bf16")) + elif vi%2 == 1: + d = elementSumIdx + vi//2 + module.add(VAndOrB32(dst=vgpr(d), src0=vgpr(formatVgpr), src1=vgpr(vgprBf16Mask), src2=vgpr(formatting(sumIdxV-1, inputPrefix, prefixOffset)), comment="pack two bf16 to dword")) + return module + +class PackData_INT8(PackData): + kernel = {"ProblemType": {"DestDataType": DataType(DataType.int8)}} + def __call__(self, gwvw, elementSumIdx, tmpVgpr, tmpS01, SaturateTypeInt8 = SaturateCastType.NORMAL, inputPrefix="", prefixOffset=0): + assert (gwvw % 4 == 0) + module = Module("PackData int8") + for vi in range(0, gwvw): + sumIdxV = elementSumIdx + vi + formatVgpr = formatting(sumIdxV, inputPrefix, prefixOffset) + if vi%4 == 0: + d = elementSumIdx + vi//4 + for i in range(0, 4): + module.add(VSaturateCastInt(sumIdxV+i, tmpVgpr, tmpS01, -128, 127, type=SaturateTypeInt8, initGpr=(i%4 == 0))) + module.add(VLShiftLeftB16(dst=vgpr(formatting(sumIdxV+1, inputPrefix, prefixOffset)), shiftHex=8, src=vgpr(formatting(sumIdxV+1, inputPrefix, prefixOffset)))) + module.add(VLShiftLeftB16(dst=vgpr(formatting(sumIdxV+3, inputPrefix, prefixOffset)), shiftHex=8, src=vgpr(formatting(sumIdxV+3, inputPrefix, prefixOffset)))) + module.add(VOrB32(dst=vgpr(formatVgpr), src0=vgpr(formatVgpr), src1=vgpr(formatting(sumIdxV+1, inputPrefix, prefixOffset)), \ + sdwa=SDWAModifiers(dst_sel=SelectBit.DWORD, dst_unused=UnusedBit.UNUSED_PAD, \ + src0_sel=SelectBit.BYTE_0, src1_sel=SelectBit.DWORD))) + module.add(VOrB32(dst=vgpr(formatting(sumIdxV+1, inputPrefix, prefixOffset)), \ + src0=vgpr(formatting(sumIdxV+2, inputPrefix, prefixOffset)), \ + src1=vgpr(formatting(sumIdxV+3, inputPrefix, prefixOffset)), \ + sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, dst_unused=UnusedBit.UNUSED_PAD, \ + src0_sel=SelectBit.BYTE_0, src1_sel=SelectBit.DWORD))) + module.add(VOrB32(dst=vgpr(d), src0=vgpr(formatVgpr), src1=vgpr(formatting(sumIdxV+1, inputPrefix, prefixOffset)), \ + sdwa=SDWAModifiers(dst_sel=SelectBit.DWORD, dst_unused=UnusedBit.UNUSED_PAD, \ + src0_sel=SelectBit.WORD_0, src1_sel=SelectBit.DWORD))) + return module diff --git a/tensilelite/Tensile/Components/Priority.py b/tensilelite/Tensile/Components/Priority.py new file mode 100644 index 0000000000..56c0356aa8 --- /dev/null +++ b/tensilelite/Tensile/Components/Priority.py @@ -0,0 +1,48 @@ +################################################################################ +# Copyright 2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..Component import Component +from ..TensileInstructions import SSetPrior + +class Priority(Component): + """ + Raise/lower workgroup priority. + """ + pass + +class AggressivePriority(Priority): + """ + Priority implementation which does set the priority. + + Keeps track of the previous value in the instance and only sets priority + if the new priority is different. + """ + + def __init__(self, currentPrio=None): + self.currentPrio = currentPrio + + def __call__(self, writer, prio, message=""): + if prio == self.currentPrio: + return "" + + self.currentPrio = prio + + return SSetPrior(prior=prio, comment=message) diff --git a/tensilelite/Tensile/Components/SIA.py b/tensilelite/Tensile/Components/SIA.py new file mode 100644 index 0000000000..75b1ecdffe --- /dev/null +++ b/tensilelite/Tensile/Components/SIA.py @@ -0,0 +1,843 @@ +################################################################################ +# Copyright 2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import Item, Module, HolderContainer, Instruction, \ + GlobalReadInstruction, LocalReadInstruction, \ + LocalWriteInstruction, SSetPrior, SWaitCnt, \ + replaceHolder +from ..Common import roundUp +from ..Component import SIA + +import copy +from math import ceil + +PRECISION = 100 +class SIA3(SIA): + kernel = {"ScheduleIterAlg": 3} + def __call__(self): + assert(0) + + def schedIntoIteration(self, writer, kernel, tensorParametersA, tensorParametersB, localWriteEndIter, uDu, firstIter, lastLoop, lastLc, maxVmcnt, globalReadIncACode, globalReadIncBCode): + # Get schedule information + if kernel["EnableMatrixInstruction"]: + numMfmaBetweenLWandBarrier = getLocalWriteMFMAEnd(writer, kernel, tensorParametersA, tensorParametersB) + ######### + # Internally assign an optimized LWPM value for PGR2 + ######### + # strategy is to distribute LW/GR as wide as possible to avoid hitting vmem FIFO + # LWPM = (LW_End - LW_Start) / numLW + if kernel["LocalWritePerMfma"] == -1: + lwStartMfmaIndex = getLocalWriteMFMAStart(writer, kernel, tensorParametersA, tensorParametersB) + numLocalWriteModPerMfma = getNumLocalWritePerMfma(writer, kernel, lwStartMfmaIndex) + else: + numLocalWriteModPerMfma = roundUp(kernel["LocalWritePerMfma"]*PRECISION) + + AssignGRPMandLWPM(writer, kernel, numLocalWriteModPerMfma) + localWriteEndIter = fixLocalWriteEndMfmaIndex(writer, kernel, tensorParametersA, tensorParametersB, \ + globalReadIncACode, globalReadIncBCode, numMfmaBetweenLWandBarrier, uDu, lastLoop) + numGlobalReadInsPerIter, numLocalWriteModPerIter, numEmptyGlobalReadIncCode = getScheduleParamMfma(writer) + else: + numGlobalReadInsPerIter, numLocalWriteModPerIter, numEmptyGlobalReadIncCode = getScheduleParams(kernel) + numLocalWritesPerSched = writer.numLocalWriteModPerMfma + # Schedule global read + if not writer.scheduleGlobalRead: + itemsGRToSchedLater, lastLoadIter = noSchedGlobalRead(writer, kernel, globalReadIncACode, globalReadIncBCode) + if kernel["EnableMatrixInstruction"]: + writer.grEndMfmaIndex = 0 + else: + itemsGRToSched, itemsGRToSchedLater = prepareGRInstToSched(writer, kernel) + if kernel["EnableMatrixInstruction"]: + itemsGRIncToSched = appendInstToSchedSIA3(writer, kernel, numEmptyGlobalReadIncCode, globalReadIncACode, globalReadIncBCode) + schedNumForIter0, endIter = getSchedNumForIter0SIA3(writer, kernel, itemsGRToSched, itemsGRIncToSched, numGlobalReadInsPerIter) + else: + itemsGRIncToSched = appendInstToSchedDefault(numEmptyGlobalReadIncCode, globalReadIncACode, globalReadIncBCode) + schedNumForIter0, endIter = getSchedNumForIter0Default(itemsGRToSched, itemsGRIncToSched, numGlobalReadInsPerIter, localWriteEndIter) + lastLoadIter = schedGlobalRead(writer, itemsGRToSched, itemsGRIncToSched, numGlobalReadInsPerIter, schedNumForIter0, endIter) + # Schedule local write + if not writer.scheduleLocalWrite: + noSchedLocalWrite(writer, kernel, tensorParametersA, tensorParametersB, localWriteEndIter) + if kernel["EnableMatrixInstruction"]: + writer.lwStartMfmaIndex = writer.lwEndMfmaIndex + else: + itemsLWToSched, numWritesToSched = prepareLWInstToSched(writer, kernel, numLocalWritesPerSched) + if kernel["EnableMatrixInstruction"]: + startIter = assignLWSchedIndexSIA3(writer, kernel, numLocalWritesPerSched, localWriteEndIter, numWritesToSched) + else: + startIter = assignLWSchedIndexDefault(writer, kernel, numLocalWritesPerSched, localWriteEndIter, lastLoadIter, numWritesToSched) + readsToWait, readsToWaitDTV, readsToWaitNGLL = getReadsToWait(writer, kernel) + # make sure numLocalWriteModPerIter is enough to schedule localwrite + startIterItem = numLocalWriteModPerIter - (writer.lwStartMfmaIndex % writer.numMfmaPerIter) * numLocalWritesPerSched + schedLocalWrite(writer, kernel, numLocalWriteModPerIter, numLocalWritesPerSched, localWriteEndIter, \ + itemsGRToSchedLater, itemsLWToSched, startIter, readsToWait, readsToWaitDTV, readsToWaitNGLL, \ + uDu, firstIter, lastLc, maxVmcnt, startIterItem) + +class SIA2(SIA): + kernel = {"ScheduleIterAlg": 2} + def __call__(self): + assert(0) + + def schedIntoIteration(self, writer, kernel, tensorParametersA, tensorParametersB, localWriteEndIter, uDu, firstIter, lastLoop, lastLc, maxVmcnt, globalReadIncACode, globalReadIncBCode): + # Get schedule information + numGlobalReadInsPerIter, numLocalWriteModPerIter, numEmptyGlobalReadIncCode = getScheduleParams(kernel) + numLocalWritesPerSched = numLocalWriteModPerIter + # Schedule global read + if not writer.scheduleGlobalRead: + itemsGRToSchedLater, lastLoadIter = noSchedGlobalRead(writer, kernel, globalReadIncACode, globalReadIncBCode) + else: + itemsGRToSched, itemsGRToSchedLater = prepareGRInstToSched(writer, kernel) + itemsGRIncToSched = appendInstToSchedDefault(numEmptyGlobalReadIncCode, globalReadIncACode, globalReadIncBCode) + schedNumForIter0, endIter = getSchedNumForIter0Default(itemsGRToSched, itemsGRIncToSched, numGlobalReadInsPerIter, localWriteEndIter) + lastLoadIter = schedGlobalRead(writer, itemsGRToSched, itemsGRIncToSched, numGlobalReadInsPerIter, schedNumForIter0, endIter) + # Schedule local write + if not writer.scheduleLocalWrite: + noSchedLocalWrite(writer, kernel, tensorParametersA, tensorParametersB, localWriteEndIter) + else: + itemsLWToSched, numWritesToSched = prepareLWInstToSched(writer, kernel, numLocalWritesPerSched) + startIter = assignLWSchedIndexDefault(writer, kernel, numLocalWritesPerSched, localWriteEndIter, lastLoadIter, numWritesToSched) + readsToWait, readsToWaitDTV, readsToWaitNGLL = getReadsToWait(writer, kernel) + schedLocalWrite(writer, kernel, numLocalWriteModPerIter, numLocalWritesPerSched, localWriteEndIter, \ + itemsGRToSchedLater, itemsLWToSched, startIter, readsToWait, readsToWaitDTV, readsToWaitNGLL, \ + uDu, firstIter, lastLc, maxVmcnt) + +class SIA1(SIA): + kernel = {"ScheduleIterAlg": 1} + def __call__(self): + assert(0) + + def schedIntoIteration(self, writer, kernel, tensorParametersA, tensorParametersB, localWriteEndIter, uDu, firstIter, lastLoop, lastLc, maxVmcnt, globalReadIncACode, globalReadIncBCode): + # Get schedule information + numGlobalReadInsPerIter, numLocalWriteModPerIter, numEmptyGlobalReadIncCode = getScheduleParams(kernel) + numLocalWritesPerSched = numLocalWriteModPerIter + # Schedule global read + if not writer.scheduleGlobalRead: + itemsGRToSchedLater, lastLoadIter = noSchedGlobalRead(writer, kernel, globalReadIncACode, globalReadIncBCode) + else: + itemsGRToSched, itemsGRToSchedLater = prepareGRInstToSched(writer, kernel) + itemsGRIncToSched = appendInstToSchedDefault(numEmptyGlobalReadIncCode, globalReadIncACode, globalReadIncBCode) + schedNumForIter0, endIter = getSchedNumForIter0Default(itemsGRToSched, itemsGRIncToSched, numGlobalReadInsPerIter, localWriteEndIter) + lastLoadIter = schedGlobalRead(writer, itemsGRToSched, itemsGRIncToSched, numGlobalReadInsPerIter, schedNumForIter0, endIter) + # Schedule local write + if not writer.scheduleLocalWrite: + noSchedLocalWrite(writer, kernel, tensorParametersA, tensorParametersB, localWriteEndIter) + else: + itemsLWToSched, numWritesToSched = prepareLWInstToSched(writer, kernel, numLocalWritesPerSched) + startIter = assignLWSchedIndexDefault(writer, kernel, numLocalWritesPerSched, localWriteEndIter, lastLoadIter, numWritesToSched) + readsToWait, readsToWaitDTV, readsToWaitNGLL = getReadsToWait(writer, kernel) + schedLocalWrite(writer, kernel, numLocalWriteModPerIter, numLocalWritesPerSched, localWriteEndIter, \ + itemsGRToSchedLater, itemsLWToSched, startIter, readsToWait, readsToWaitDTV, readsToWaitNGLL, \ + uDu, firstIter, lastLc, maxVmcnt) + +################################################################################ +################################################################################ +### +### Schedule Parameters +### +################################################################################ +################################################################################ + +def getScheduleParams(kernel): + numGlobalReadInsPerIter = roundUp(kernel["GlobalReadPerMfma"] * PRECISION) if kernel["GlobalReadPerMfma"] > 0 else PRECISION + numLocalWriteModPerIter = roundUp(kernel["LocalWritePerMfma"] * PRECISION) if kernel["LocalWritePerMfma"] > 0 else PRECISION + numEmptyGlobalReadIncCode = numGlobalReadInsPerIter - 1 + return numGlobalReadInsPerIter, numLocalWriteModPerIter, numEmptyGlobalReadIncCode + +def getLocalWriteMFMAEnd(writer, kernel, tensorParametersA, tensorParametersB): + numMfmaPerIter = writer.numMfmaPerIter + ######### + # Get localWriteEnd + ######### + # assign parameter + # 1. we calculate number of mfma to prefetch localReads for next loop + # 2. we put barrier 1 mfma ahead that + # 3. we put last localWrite 1~2 mfma ahead barrier + # localReads followed following sequence to be scheduled + # ds_read[A][0], ds_read[B][0], ds_read[A][1:], ds_read[B][1:] + # NOTE: we need this sequence for new feature "breaking waitcnt" + # TODO: breaking waitcnt + writer.numMfmaForLR = 1 + latencyLeft = writer.miLatencyLeft + miLatencyLeft = writer.miLatencyLeft + # ds_read[A][0] + for i in range(writer.numReadPerVectorA): + latencyLeft -= tensorParametersA["localReadInstruction"].issueLatency*2 + if latencyLeft < 0: + writer.numMfmaForLR += 1 + latencyLeft = max(miLatencyLeft - tensorParametersA["localReadInstruction"].issueLatency*2,0) + # ds_read[B][0] + for i in range(writer.numReadPerVectorB): + latencyLeft -= tensorParametersB["localReadInstruction"].issueLatency*2 + if latencyLeft < 0: + writer.numMfmaForLR += 1 + latencyLeft = max(miLatencyLeft - tensorParametersB["localReadInstruction"].issueLatency*2,0) + # ds_read[A][1:] + for i in range(writer.numReadsPerIterA-writer.numReadPerVectorA): + latencyLeft -= tensorParametersA["localReadInstruction"].issueLatency*2 + if latencyLeft < 0: + writer.numMfmaForLR += 1 + latencyLeft = max(miLatencyLeft - tensorParametersA["localReadInstruction"].issueLatency*2,0) + # ds_read[B][1:] + for i in range(writer.numReadsPerIterB-writer.numReadPerVectorB): + latencyLeft -= tensorParametersB["localReadInstruction"].issueLatency*2 + if latencyLeft < 0: + writer.numMfmaForLR += 1 + latencyLeft = max(miLatencyLeft - tensorParametersB["localReadInstruction"].issueLatency*2,0) + # to calculate number of mfma we need to wait before data arrive from lds to vgpr. + # latency: 40 quad-cycle for 4 word, 20 quad-cycle for 2 word, 10 quad-cycle for 1 word / half word + writer.numMfmaForNextLoopLR = writer.numMfmaForLR + latencyForLR = roundUp(tensorParametersB["localReadInstruction"].blockWidth)*10 + latencyForLR -= max(latencyLeft,0) # remaining latency in mfma + latencyForLR -= writer.miLatency # last LR will have 1 mfma latency + while latencyForLR > 0: + latencyForLR -= writer.miLatency + writer.numMfmaForNextLoopLR += 1 + # final index definition + writer.numMfmaForNextLoopLR = min(writer.numMfmaForNextLoopLR,numMfmaPerIter-1) + writer.barrierMfmaIndex = numMfmaPerIter*(kernel["LoopIters"]-writer.numItersPLR+1) - writer.numMfmaForNextLoopLR - 1 if writer.numItersPLR else 0 + numMfmaBetweenLWandBarrier = 2 if kernel["MatrixInstM"] == 32 else 3 + writer.lwEndMfmaIndex = max(writer.barrierMfmaIndex - numMfmaBetweenLWandBarrier,0) if writer.numItersPLR else numMfmaPerIter*kernel["LoopIters"] - 1 + if kernel["DirectToLds"] and kernel["PrefetchGlobalRead"] == 2: + # DirectToLds + PGR=2 case, lwEndMfmaIndex must be after the end of local read (excluding local reads for next iter) + lrEnd = min(writer.barrierMfmaIndex - 1, writer.numMfmaForLR * (kernel["LoopIters"] - writer.numItersPLR)) + if writer.lwEndMfmaIndex < lrEnd: + writer.lwEndMfmaIndex = lrEnd + return numMfmaBetweenLWandBarrier + +def getLocalWriteMFMAStart(writer, kernel, tensorParametersA, tensorParametersB): + numMfmaPerIter = writer.numMfmaPerIter + ######### + # Get localWriteStart + ######### + if not kernel["1LDSBuffer"]: + # TODO: replace here for real number of globalReadIncInst + numGRIncInst = 12 if not kernel["StaggerU"] else 18 + numInstPerMfma = max(roundUp(writer.miLatencyLeft/2),1) + numMfmaToSched = roundUp(numGRIncInst/numInstPerMfma) + lwStartMfmaIndex = 1 + numMfmaToSched + else: + # for 1LDSB, we have to issue localwrites after localreads + if writer.numVgprBuffer == kernel["LoopIters"]: + if writer.numReadPerVectorA != 1 or writer.numReadPerVectorB !=1: + # fp16 or bf16, we read 1 element to vgprBuffer the other element to tempVgpr. + # since each iteration shares same tempVgpr, only read-to-vgprBuffer can + # be scheduled in the front of loop. + # localwrite have to start after last read-to-tempVgpr. + numHalfReads = (writer.numReadPerVectorA//2)*kernel["InnerUnroll"]*kernel["MIWaveTileA"] + (writer.numReadPerVectorB//2)*kernel["InnerUnroll"]*kernel["MIWaveTileB"] + numMfmaForHalfRead = 1 + latencyLeft = writer.miLatencyLeft + for i in range(numHalfReads): + latencyLeft -= 2 + if latencyLeft < 0: + numMfmaForHalfRead += 1 + latencyLeft = max(writer.miLatencyLeft - 2, 0) + lwStartMfmaIndex = numMfmaPerIter * (kernel["LoopIters"] - 1 - writer.numItersPLR) + numMfmaForHalfRead + else: + # we have enough vgprBuffer to schedule localReads in the front of loop + numMfmaForCurrentLoopLR = 1 + latencyLeft = writer.miLatencyLeft + for u in range(kernel["LoopIters"] - writer.numItersPLR): + doReadA = (u < kernel["LoopIters"] // writer.numIterPerCoalescedReadA - writer.numItersPLR) + doReadB = (u < kernel["LoopIters"] // writer.numIterPerCoalescedReadB - writer.numItersPLR) + # disable LocalRead if DirectToVgpr is enabled + doReadA = doReadA and (not kernel["DirectToVgprA"]) + doReadB = doReadB and (not kernel["DirectToVgprB"]) + # ds_read[A][0] + for i in range(writer.numReadPerVectorA * doReadA): + latencyLeft -= tensorParametersA["localReadInstruction"].issueLatency*2 + if latencyLeft < 0: + numMfmaForCurrentLoopLR += 1 + latencyLeft = max(writer.miLatencyLeft - tensorParametersA["localReadInstruction"].issueLatency*2,0) + # ds_read[B][0] + for i in range(writer.numReadPerVectorB * doReadB): + latencyLeft -= tensorParametersB["localReadInstruction"].issueLatency*2 + if latencyLeft < 0: + numMfmaForCurrentLoopLR += 1 + latencyLeft = max(writer.miLatencyLeft - tensorParametersB["localReadInstruction"].issueLatency*2,0) + # ds_read[A][1:] + for i in range((writer.numReadsPerIterA - writer.numReadPerVectorA) * doReadA): + latencyLeft -= tensorParametersA["localReadInstruction"].issueLatency*2 + if latencyLeft < 0: + numMfmaForCurrentLoopLR += 1 + latencyLeft = max(writer.miLatencyLeft - tensorParametersA["localReadInstruction"].issueLatency*2,0) + # ds_read[B][1:] + for i in range((writer.numReadsPerIterB - writer.numReadPerVectorB) * doReadB): + latencyLeft -= tensorParametersB["localReadInstruction"].issueLatency*2 + if latencyLeft < 0: + numMfmaForCurrentLoopLR += 1 + latencyLeft = max(writer.miLatencyLeft - tensorParametersB["localReadInstruction"].issueLatency*2,0) + lwStartMfmaIndex = numMfmaForCurrentLoopLR + else: + lwStartMfmaIndex = numMfmaPerIter * (kernel["LoopIters"] - 1 - writer.numItersPLR) + writer.numMfmaForLR + # to calculate number of mfma we need to wait before data arrive from lds to vgpr. + # latency: 40 quad-cycle for 4 word, 20 quad-cycle for 2 word, 10 quad-cycle for 1 word / half word + if writer.numIterPerCoalescedReadB > writer.numIterPerCoalescedReadA: + latencyForLR = roundUp(tensorParametersA["localReadInstruction"].blockWidth) * 10 + else: + latencyForLR = roundUp(tensorParametersB["localReadInstruction"].blockWidth) * 10 + latencyForLR -= max(latencyLeft,0) # remaining latency in mfma + while latencyForLR > 0: + latencyForLR -= writer.miLatency + lwStartMfmaIndex += 1 + + return lwStartMfmaIndex + +def getNumLocalWritePerMfma(writer, kernel, lwStartMfmaIndex): + ######### + # Get LocalWritePerMfma + ######### + if lwStartMfmaIndex > writer.lwEndMfmaIndex: + lwStartMfmaIndex = writer.lwEndMfmaIndex + numMfmaCanSched = writer.lwEndMfmaIndex - lwStartMfmaIndex + 1 + numLoadsA = kernel["DepthU"]*kernel["MacroTileA"]//kernel["GlobalLoadVectorWidthA"]//kernel["NumThreads"] + numLoadsB = kernel["DepthU"]*kernel["MacroTileB"]//kernel["GlobalLoadVectorWidthB"]//kernel["NumThreads"] + writesToSched = (numLoadsA + numLoadsB - 1) * PRECISION + oldValue = 0 + newValue = PRECISION + loop = 0 + # 1. number of padded writesToSched is (numWrites - 1) * 100 + 1 + # LW ---99--- LW ---99--- LW + # 2. we need to pad it to multiple of LWPM + # LW ---99--- LW ---99--- LW --?-- + # | ------- multiple of LWPM ---- | + # 3. if LWPM is not multiple of 100, we need extra empty instructions to schedule GR for PGR2 + # LW ---99--- LW ---99--- LW --?-- --?-- + # | ------- multiple of LWPM ---- |-LWPM-| + # 4. then we put GR into padded writesToSched + # put GR after LW + LWPM of empty inst, so that we can offset GR 1 mfma with LW if possible + # Ex. LWPM = 0.25 + # LW --24- GR ------74------ LW --24- GR ------74------ LW --24- GR --24- + # mfma--24-mfma--24-mfma--24-mfma--24-mfma--24-mfma--24-mfma--24-mfma--24-mfma + # we need LWPM to get precise LWPM + # so we iterate formula 10 times to get LWPM + while oldValue != newValue and loop < 10: + loop += 1 + oldValue = newValue + newValue = roundUp((writesToSched+1 + (oldValue - (writesToSched+1) % oldValue) + oldValue%PRECISION) / numMfmaCanSched) + return newValue + +def AssignGRPMandLWPM(writer, kernel, numLocalWriteModPerMfma): + ##### + # Assign GRPM and LWPM + ##### + # HOW THIS WORK + # padding each globalReadInstruction to 100 with empty instruction, + # each mfma will schedule intructions GRPM*100 times from padded globalReadInstruction. + # Ex. GRPM = 0.5 + # GR ---------99--------- GR --------99---------- GR + # mfma --49-- mfma --49-- mfma --49-- mfma --49-- mfma --49-- + writer.numGlobalReadInsPerMfma = roundUp(kernel["GlobalReadPerMfma"]*PRECISION) + + # HOW THIS WORK + # padding each globalReadInstruction to 100 with empty instruction, + # each mfma will schedule intructions GRPM*100 times from padded globalReadInstruction. + # Ex. LWPM = 0.5 + # LW ---------99--------- LW --------99---------- LW + # mfma --49-- mfma --49-- mfma --49-- mfma --49-- mfma --49-- + if kernel["PrefetchGlobalRead"] == 1: + # In PGR1: + # Larger LWPM can provide more latency to hide global read + # However, larger LWPM may cause mfma bubbles + # we set LWPM to 1 unless it requires larger LWPM to enable 1LDSB + if kernel["1LDSBuffer"]: + writer.numLocalWriteModPerMfma = max(numLocalWriteModPerMfma,PRECISION) + else: + writer.numLocalWriteModPerMfma = PRECISION + else: + writer.numLocalWriteModPerMfma = numLocalWriteModPerMfma + +def getScheduleParamMfma(writer): + numMfmaPerIter = writer.numMfmaPerIter + numGlobalReadInsPerIter = numMfmaPerIter * writer.numGlobalReadInsPerMfma + numLocalWriteModPerIter = numMfmaPerIter * writer.numLocalWriteModPerMfma + # if numGlobalReadInsPerMfma>1, we still want to schedule only 1 GlobalReadIncCode per mfma + # inserting empty CodeModule so that generator will schedule 1 GlobalReadIncCode 1 empty CodeModule if numGlobalReadInsPerMfma=2 + numEmptyGlobalReadIncCode = writer.numGlobalReadInsPerMfma - 1 + return numGlobalReadInsPerIter, numLocalWriteModPerIter, numEmptyGlobalReadIncCode + +def fixLocalWriteEndMfmaIndex(writer, kernel, tPA, tPB, globalReadIncACode, globalReadIncBCode, numMfmaBetweenLWandBarrier, uDu, lastLoop): + numMfmaPerIter = writer.numMfmaPerIter + # If numLocalWriteModPerMfma is not multiple of 100, + # last globalread will be scheduled at lwEndMfmaIndex, + # and last localwrite will be scheduled at lwEndMfmaIndex - 1 + # so we offset lwEndMfmaIndex by 1 mfma + if kernel["PrefetchGlobalRead"] == 2 and writer.numLocalWriteModPerMfma % PRECISION != 0: + numMfmaBetweenLWandBarrier -= 1 + def assignParamSplitLds(numMfmaBetweenLWandBarrier): + if not kernel.enabledSplitLDS: + return numMfmaBetweenLWandBarrier + # how many local reads in terms of mfma indices (height) + # total number of instructions (total) minus the instructions prefetched outside of loop (spent), divided by mfma bubble (width) + issueLatency = max(tPA["localReadInstruction"].issueLatency, tPB["localReadInstruction"].issueLatency) * 2 + width = writer.miLatencyLeft // issueLatency + width = max(width, 1) + spent = writer.numItersPLR * (writer.numReadsPerIterA + writer.numReadsPerIterB) + total = kernel["LoopIters"]//writer.numIterPerCoalescedReadA*writer.numReadsPerIterA + \ + kernel["LoopIters"]//writer.numIterPerCoalescedReadB*writer.numReadsPerIterB + height = int(ceil((total-spent)/width)) + # how many local writes + localWritesToSched = writer.localWriteACode.countType(LocalWriteInstruction) + \ + writer.localWriteBCode.countType(LocalWriteInstruction) + localWritesPerMfma = writer.numLocalWriteModPerMfma / PRECISION # was scaled by PRECISION + # _numMfmaBetweenLastLWandBarrier: a function of 'spacing', which is num of mfma instructions until local write starts + _numMfmaBetweenLastLWandBarrier = lambda spacing : writer.barrierMfmaIndex + 1 - ceil(localWritesToSched/localWritesPerMfma) - spacing + addrIncToSched = sum(1 for codemod in [globalReadIncACode, globalReadIncBCode] if len(codemod.items())) + if uDu < kernel["DepthULdsDivisor"] - 1: + if kernel["1LDSBuffer"] and kernel["PrefetchLocalRead"] > 1: + # space the stream of local writes so that 1st local write is scheduled after last local read, + # but give it 2 mfma's worth of headroom + spacing = 2 + height + else: + # can start ds_write/buffer_load as soon as loop starts, but give it 1 mfma's worth of headroom + spacing = 1 + else: + # query how much spacing we have by calling lambda(0), minus the original 'numMfmaBetweenLWandBarrier' + # we get the spacing that results in exactly 'numMfmaBetweenLWandBarrier' between last write and barrier + spacing = _numMfmaBetweenLastLWandBarrier(0) - numMfmaBetweenLWandBarrier + addrIncToSched - 1 + return max(0, _numMfmaBetweenLastLWandBarrier(spacing)) + + numMfmaBetweenLWandBarrier = assignParamSplitLds(numMfmaBetweenLWandBarrier) + writer.lwEndMfmaIndex = max(writer.barrierMfmaIndex - numMfmaBetweenLWandBarrier,0) if writer.numItersPLR else numMfmaPerIter*kernel["LoopIters"] - 1 + # adjust lwEndMfmaIndex for the following cases + # 1) PGR=2 + DirectToVgpr(DTV) + # 2) last loop enabled case + # In these cases, lwEndMfmaIndex needs to be < numMfmaPerIter * (kernel["LoopIters"] - 1) + # to schedule global read for DTV after lwEndMfmaIndex or execute PostLoop after StoreC in NoLoadLoop + # kernel["LoopIters"] has to be > 1 to make this logic work. + if kernel["LoopIters"] > 1 and \ + ((kernel["PrefetchGlobalRead"] == 2 and (kernel["DirectToVgprA"] or kernel["DirectToVgprB"])) or \ + lastLoop): + writer.lwEndMfmaIndex = min(writer.lwEndMfmaIndex, numMfmaPerIter * (kernel["LoopIters"] - 1) - 1) + if kernel["DirectToLds"] and kernel["PrefetchGlobalRead"] == 2: + # DirectToLds + PGR=2 case, lwEndMfmaIndex must be after the end of local read (excluding local reads for next iter) + lrEnd = min(writer.barrierMfmaIndex - 1, writer.numMfmaForLR * (kernel["LoopIters"] - writer.numItersPLR)) + if writer.lwEndMfmaIndex < lrEnd: + writer.lwEndMfmaIndex = lrEnd + localWriteEndIter = writer.lwEndMfmaIndex//numMfmaPerIter + localWriteEndIter = min(kernel["LoopIters"] - 1, localWriteEndIter) + assert localWriteEndIter < kernel["LoopIters"] + assert writer.lwEndMfmaIndex < numMfmaPerIter*kernel["LoopIters"] + return localWriteEndIter + +################################################################################ +################################################################################ +### +### Schedule Global Read +### +################################################################################ +################################################################################ + +def noSchedGlobalRead(writer, kernel, globalReadIncACode, globalReadIncBCode): + # put everything in the header: + writer.unrollLoopHeaderCode.add(writer.dtlsM0UpdateACode) + writer.unrollLoopHeaderCode.add(writer.globalReadACode) + writer.unrollLoopHeaderCode.add(writer.dtlsM0UpdateBCode) + writer.unrollLoopHeaderCode.add(writer.globalReadBCode) + writer.unrollLoopHeaderCode.add(globalReadIncACode) + writer.unrollLoopHeaderCode.add(globalReadIncBCode) + # Dummy + itemsGRToSchedLater = [] + lastLoadIter = 0 + return itemsGRToSchedLater, lastLoadIter + +def prepareGRInstToSched(writer, kernel): + writer.unrollLoopHeaderCode.add(writer.globalReadACode.header) + writer.unrollLoopHeaderCode.add(writer.globalReadBCode.header) + + # Add all loads from middle as individual schedulable items + # when using PGR2, put global read instruction right after corresponding localWrite instruction + if kernel["PrefetchGlobalRead"] == 2 or kernel.enabledSplitLDS: + itemsGRToSched = [] + itemsGRToSchedLater = list(writer.globalReadACode.middle.items()) + \ + list(writer.globalReadBCode.middle.items()) + itemsGRToSchedLaterDTV = [] + # PGR2 and DirectToVgpr case, schedule global read for DirectToVgpr separately after registers are used for mfma + if kernel["EnableMatrixInstruction"] and kernel["ScheduleIterAlg"] == 3: + if kernel["DirectToVgprA"] or kernel["DirectToVgprB"]: + itemsGRToSchedLater = list(writer.globalReadACode.middle.items()) # not DirectToVgpr (A has non-DirectToVgpr load) + itemsGRToSchedLaterDTV = list(writer.globalReadBCode.middle.items()) # DirectToVgpr (B has DirectToVgpr load) + # add to writer.perIterGlobalReadCodeDTV to schedule DirectToVgpr + while itemsGRToSchedLaterDTV: + itemGR = itemsGRToSchedLaterDTV.pop(0) + writer.perIterGlobalReadCodeDTV[kernel["LoopIters"] - 1].add(itemGR) + if kernel.enabledSetPrioSplitLDS and itemsGRToSchedLater: + itemsGRToSchedLater.insert(1, SSetPrior(prior=3, comment="top priority for load")) + itemsGRToSchedLater.insert(len(itemsGRToSchedLater), SSetPrior(prior=0)) + else: + itemsGRToSched = list(writer.globalReadACode.middle.items()) + \ + list(writer.globalReadBCode.middle.items()) + itemsGRToSchedLater = [] + + itemsGRToSchedTemp = [] + for i in range(len(itemsGRToSched)): + itemsGRToSchedTemp.append(itemsGRToSched.pop(0)) + for j in range(PRECISION-1): + itemsGRToSchedTemp.append(Module()) + itemsGRToSched = itemsGRToSchedTemp + return itemsGRToSched, itemsGRToSchedLater + + +def appendInstToSchedSIA3(writer, kernel, numEmptyGlobalReadIncCode, globalReadIncACode, globalReadIncBCode): + itemsGRIncToSched = [] + # for SIA3, we can break GlobalReadIncCode to avoid mfma bubbles + if kernel["PrefetchGlobalRead"] == 2: + # skip to schedule global read for PGR2 first mfma + for i in range(numEmptyGlobalReadIncCode+1): + imod = Module() + itemsGRIncToSched.append(imod) + numInst = globalReadIncACode.countType(Instruction) + globalReadIncBCode.countType(Instruction) + numInstPerMfma = max(roundUp(writer.miLatencyLeft/2),1) + + globalReadInc1 = globalReadIncACode.flatitems() + globalReadInc2 = globalReadIncBCode.flatitems() + if kernel["DirectToVgprA"]: + # swap the order of readInc for DTVA + globalReadInc1, globalReadInc2 = globalReadInc2, globalReadInc1 + globalReadIncItems = globalReadInc1 + globalReadInc2 + numMfmaToSched = roundUp(numInst/numInstPerMfma) + for j in range(numMfmaToSched): + imod = Module() + count = 0 + while globalReadIncItems and count < numInstPerMfma: + tempInst = globalReadIncItems.pop(0) + imod.add(tempInst) + if tempInst.countType(Instruction): + count += 1 + itemsGRIncToSched.append(imod) + for i in range(numEmptyGlobalReadIncCode): + imod = Module() + itemsGRIncToSched.append(imod) + return itemsGRIncToSched + +def appendInstToSchedDefault(numEmptyGlobalReadIncCode, globalReadIncACode, globalReadIncBCode): + itemsGRIncToSched = [] + itemsGRIncToSched.append(globalReadIncACode) + for i in range(numEmptyGlobalReadIncCode): + imod = Module() + itemsGRIncToSched.append(imod) + itemsGRIncToSched.append(globalReadIncBCode) + for i in range(numEmptyGlobalReadIncCode): + imod = Module() + itemsGRIncToSched.append(imod) + return itemsGRIncToSched + +def getSchedNumForIter0SIA3(writer, kernel, itemsGRToSched, itemsGRIncToSched, numGlobalReadInsPerIter): + numMfmaPerIter = writer.numMfmaPerIter + # Loop in PGR1: GlobalRead -> GlobalReadInc -> LocalWrite + # but GlobalReadInc shouldn't block LocalWrite so we count them out + # Loop in PGR2: GlobalReadInc -> LocalWrite/GlobalRead pair + # since LocalWrite/GlobalRead pair depends on GlobalReadInc, we count in only GlobalReadInc + if kernel["PrefetchGlobalRead"] == 2: + loadsToSched = len(itemsGRIncToSched) + else: + loadsToSched = len(itemsGRToSched) + + # Here is to adjust scheduling silently in order to have validation pass. + # Better way is to use larger globalReadPerMfma. + ## schedule more instructions at first iteration if no enough mfma to schedule globalRead + writer.grEndMfmaIndex = max(0, roundUp(loadsToSched/writer.numGlobalReadInsPerMfma) - 1) + if writer.grEndMfmaIndex > writer.lwEndMfmaIndex: + schedNumForIter0 = numGlobalReadInsPerIter + (writer.grEndMfmaIndex - writer.lwEndMfmaIndex) * writer.numGlobalReadInsPerMfma + writer.grEndMfmaIndex = writer.lwEndMfmaIndex + else: + schedNumForIter0 = numGlobalReadInsPerIter + if kernel["PrefetchGlobalRead"] == 1: + globalReadIncEndMfmaIndex = writer.grEndMfmaIndex + roundUp(len(itemsGRIncToSched)/writer.numGlobalReadInsPerMfma) + endIter = roundUp((globalReadIncEndMfmaIndex+1)/numMfmaPerIter) + else: + endIter = roundUp((writer.grEndMfmaIndex+1)/numMfmaPerIter) + ## schedule more instructions at first iteration if no enough mfma to schedule globalRead + globalReadInc + if endIter > kernel["LoopIters"]: + endIter = kernel["LoopIters"] + if kernel["PrefetchGlobalRead"] == 1: + schedNumForIter0 += (globalReadIncEndMfmaIndex+1 - kernel["LoopIters"]*numMfmaPerIter) * writer.numGlobalReadInsPerMfma + return schedNumForIter0, endIter + +# schedNumForIter0 SIA 1 or 2 +# distribute the instructions in itemsGRToSched evenly as possible to iterations: perIterGlobalReadCode[0,endIter) +# last one is perIterGlobalReadCode[endIter-1], +# Ideally: endIter <= localWriteEndIter, +# then put M0 updateCode (if any) and first 'schedNumForIter0' GR-inst in perIterGlobalReadCode[0] +# put every numGlobalReadInsPerIter GR-insts in perIterGlobalReadCode[1]~[endIter-1] +# corner case: endIter > localWriteEndIter, set endIter = localWriteEndIter,in this case, schedNumForIter0 will > 1 +# and perIterGlobalReadCode[0] would need to schedule more instructions +def getSchedNumForIter0Default(itemsGRToSched, itemsGRIncToSched, numGlobalReadInsPerIter, localWriteEndIter): + # reads and incs are scheduled in iters range(0..endIter) + endIter = roundUp((len(itemsGRToSched) + len(itemsGRIncToSched)) / numGlobalReadInsPerIter) + # FIXME: + # above formula precisely count number of GR + GRInc + # however it has regression issue with tuned yaml with default GRPM. + # below formula follows old logic to add 2 to the instruction count, so it may has larger schedNumForIter0 + # we should use above formula with GRPM tuning for better performance + # NOTE: both formula pass validation test + endIter = roundUp((len(itemsGRToSched) + len(itemsGRIncToSched) + 2*PRECISION) / numGlobalReadInsPerIter) + if endIter > localWriteEndIter: + # Front-load some of the buffer loads if we don't have enough loop iters: + # could use a different/smarter algorithm to space out the loads? + schedNumForIter0 = (endIter-(localWriteEndIter) + 1) * numGlobalReadInsPerIter + endIter = localWriteEndIter + else: + # schedule b2b for readCnt > 2 (True for bigger TT) + schedNumForIter0 = numGlobalReadInsPerIter + return schedNumForIter0, endIter + +def schedGlobalRead(writer, itemsGRToSched, itemsGRIncToSched, numGlobalReadInsPerIter, schedNumForIter0, endIter): + # insert dtlsM0UpdateACode dtlsM0UpdateBCode code + if writer.globalReadACode.middle.items(): + writer.globalReadACode.middle.items()[0].items().insert(0,writer.dtlsM0UpdateACode) + if writer.globalReadBCode.middle.items(): + writer.globalReadBCode.middle.items()[0].items().insert(0,writer.dtlsM0UpdateBCode) + + itemsGRToSched.extend(itemsGRIncToSched) + # append 'n' global load at a time + # append global load(S) first 'number of global load(s)' determined by schedNumForIter0 + for item in itemsGRToSched[:schedNumForIter0]: + writer.perIterGlobalReadCode[0].add(item) + itemsGRToSched = itemsGRToSched[schedNumForIter0:] # trim the scheduled GRs, do the rest in the following loop + + lastLoadIter = 0 + for u in range(1, endIter): + # append itemPerIter GR for each iteration, + # and trim the scheduled ones at the end of loop + itemPerIter = 1 * numGlobalReadInsPerIter + try: + for item in itemsGRToSched[:itemPerIter]: + writer.perIterGlobalReadCode[u].add(item) + lastLoadIter = u + itemsGRToSched = itemsGRToSched[itemPerIter:] + except IndexError: + break # itemsGRToSched is 0-length, no code left to schedule + + assert not itemsGRToSched # should have scheduled everything already, itemsGRToSched should be empty + + writer.perIterGlobalReadCode[endIter-1].add(writer.globalReadACode.footer) + writer.perIterGlobalReadCode[endIter-1].add(writer.globalReadBCode.footer) + return lastLoadIter + +################################################################################ +################################################################################ +### +### Schedule Local Write +### +################################################################################ +################################################################################ + +def noSchedLocalWrite(writer, kernel, tensorParametersA, tensorParametersB, localWriteEndIter): + # if no scheduleLocalWrite - just add writes to localWritelocalWriteEndIter + # If PGR=0, writes have to be done immediately following the loads - no opportunity to schedule + # so don't add to schedule, these will be added separately and before the first iter + if kernel["PrefetchGlobalRead"]: + # do we need a module here? That would prevent these from being scheduled + imod = writer.perIterLocalWriteCode[localWriteEndIter].add(Module()) + imod.add( + writer.wait(kernel, tensorParametersA, tensorParametersB, 0, -1, -1, \ + "1wait for global read")) + imod.addComment1("local write A") + imod.add(writer.localWriteACode) + imod.addComment1("local write B") + imod.add(writer.localWriteBCode) + +def prepareLWInstToSched(writer, kernel, numLocalWritesPerSched): + ################# + # create a plan # + ################# + itemsLWToSched = list(writer.localWriteACode.items()) + list(writer.localWriteBCode.items()) + if kernel["PrefetchGlobalRead"] == 2: + # PrefetchGlobalRead + DirectToLds case, need to add dummy list to insert global read + tmpList = [] + numItemsBeforeStoreC = 0 + numDummy = 0 + if kernel["DirectToLdsA"]: + numDummy += max(len(list(writer.globalReadACode.middle.items())) - numItemsBeforeStoreC, 0) + if kernel["DirectToLdsB"]: + # DirectToVgprA case, LDS load is actually in B. Need to get correct length + numReadB = len(list(writer.globalReadACode.middle.items())) if kernel["DirectToVgprA"] else len(list(writer.globalReadBCode.middle.items())) + numDummy += max(numReadB - numItemsBeforeStoreC, 0) + for i in range(numDummy): + tmpList.append(Module()) + # add dummy at the top of the list + itemsLWToSched = tmpList + itemsLWToSched + # extend localWrite by inserting empty Module + # See getNumLocalWritePerMfma for how this work + itemsLWToSchedTemp = [] + for i in range(len(itemsLWToSched)-1): + itemsLWToSchedTemp.append(itemsLWToSched.pop(0)) + for j in range(PRECISION-1): + itemsLWToSchedTemp.append(Module()) + if itemsLWToSched: + itemsLWToSchedTemp.append(itemsLWToSched.pop(0)) + for i in range(numLocalWritesPerSched + numLocalWritesPerSched % PRECISION - len(itemsLWToSchedTemp) % numLocalWritesPerSched): + itemsLWToSchedTemp.append(Module()) + itemsLWToSched = itemsLWToSchedTemp + # This counts the number of modules which contain a ds_write + # Scheduler below keeps all writes in the same module in same iteration + # so this is better match to what it is trying to do + # numWritesToSched = sum(1 for item in itemsLWToSched if item.countType(LocalWriteInstruction)) + numWritesToSched = len(itemsLWToSched) + return itemsLWToSched, numWritesToSched + +def assignLWSchedIndexSIA3(writer, kernel, numLocalWritesPerSched, localWriteEndIter, numWritesToSched): + numMfmaPerIter = writer.numMfmaPerIter + writer.lwStartMfmaIndex = writer.lwEndMfmaIndex - max(1,roundUp(numWritesToSched/numLocalWritesPerSched)) + 1 + if writer.lwStartMfmaIndex < writer.grEndMfmaIndex: + writer.lwStartMfmaIndex = writer.grEndMfmaIndex + # DirectToLds + PGR=2 case, lwStart must be after all local reads are done + if kernel["DirectToLds"] and kernel["PrefetchGlobalRead"] == 2: + lrEnd = min(writer.lwEndMfmaIndex, writer.numMfmaForLR * (kernel["LoopIters"] - writer.numItersPLR)) + if writer.lwStartMfmaIndex < lrEnd: + writer.lwStartMfmaIndex = lrEnd + startIter = writer.lwStartMfmaIndex//numMfmaPerIter + assert startIter < localWriteEndIter+1 # startIter should be at or before the endIter + return startIter + +def assignLWSchedIndexDefault(writer, kernel, numLocalWritesPerSched, localWriteEndIter, lastLoadIter, numWritesToSched): + startIter = localWriteEndIter - roundUp(numWritesToSched/numLocalWritesPerSched) + 1 + # - can't move a write past the load it depends on + # as a simplification, don't move writes past any loads + if startIter < lastLoadIter: + startIter = lastLoadIter + return startIter + +def getReadsToWait(writer, kernel): + readsToWait = len(list(writer.localWriteACode.items())) + len(list(writer.localWriteBCode.items())) + readsToWaitDTV = 0 + # add waitcnt for DirectToVgpr. Delaying wait for DirectToVgpr global read + if kernel["DirectToVgprA"] or kernel["DirectToVgprB"]: + # DirectToVgprA case, actual A load is in writer.globalReadBCode (due to swap). + # Need to check writer.globalReadBCode + readsToWaitDTV += len(list(writer.globalReadBCode.middle.items())) + readsToWaitNGLL = readsToWait + return readsToWait, readsToWaitDTV, readsToWaitNGLL + +def schedLocalWrite(writer, kernel, numLocalWriteModPerIter, numLocalWritesPerSched, localWriteEndIter, \ + itemsGRToSchedLater, itemsLWToSched, startIter, readsToWait, readsToWaitDTV, readsToWaitNGLL, \ + uDu, firstIter, lastLc, maxVmcnt, startIterItem = None): + # schedule here + localwriteCnt = 0 + for u in range(startIter, localWriteEndIter+1): + if u==(localWriteEndIter): + itemPerIter = len(itemsLWToSched) # schedule all remaining activity + else: + itemPerIter = numLocalWriteModPerIter + # if localwrite is not multiple of numLocalWriteModPerIter, fill last iteration first. + # make sure numLocalWriteModPerIter is enough to schedule localwrite + # TODO: if numLocalWriteModPerIter is not enough to schedule localwrite, need smarter way to distribute localWrite + if u == startIter and startIterItem: + itemPerIter = startIterItem + + for item in itemsLWToSched[:itemPerIter]: + # Use a module to ensure these pieces stay together in the sub-iter scheduler + imod = Module("LocalWriteMod%u"%u) + imodNGLL = Module("LocalWriteMod%u"%u) + writesPerItem = item.countType(LocalWriteInstruction) + readsToWaitAdjustForStoreC = 0 + if writesPerItem: + imod.addComment0("sched write - iter %u writesPerItem=%u"%(u,writesPerItem)) + imodNGLL.addComment0("sched write - iter %u writesPerItem=%u"%(u,writesPerItem)) + # if writesPerItem>1 this indicates multiple LocalWrites in the same module + # this happens in some transpose cases. Here the first write needs to wait + # for the associated global read to finish, then the remaining writes can flow + # TODO - can schedule these writes across iters, should figure this out above + readsToWait = readsToWait - 1 + readsToWaitNGLL = readsToWaitNGLL - 1 + if uDu < kernel["DepthULdsDivisor"]-1: + imod.addComment0("no wait vmcnt except for in the last subLdsLoop") + else: + imod.add(SWaitCnt(lgkmcnt=-1, \ + vmcnt=min(maxVmcnt, readsToWait + readsToWaitDTV + readsToWaitAdjustForStoreC), vscnt=-1, \ + comment="wait for global read before writing to local")) + imodNGLL.add(SWaitCnt(lgkmcnt=-1, \ + vmcnt=min(maxVmcnt, readsToWaitNGLL + readsToWaitDTV + readsToWaitAdjustForStoreC), vscnt=-1, \ + comment="wait for global read before writing to local")) + # PK and StoreCUnroll is removed so you cannot find any HolderContainer in s_waitcnt + if kernel["PrefetchGlobalRead"]==2: + hasHolder, wcList = hasHolderInWaitCnt(item) + if hasHolder: + readsToWaitAdjust = readsToWait + readsToWaitDTV + if kernel["PrefetchGlobalRead"]==2: + # PGR=2 special cases + if (not kernel["ProblemType"]["UseBeta"]): + # no Load C case + if not firstIter: + # PGR=2 and not firstIter case, HolderContainer includes num of storeC from previous Iter + readsToWaitAdjust += readsToWaitAdjustForStoreC + if kernel["NoLdsWriteCode"] and kernel["PrefetchGlobalRead"]!=2: + # DirectToLds or DirectToVgpr for both A and B case, use the number of global read for both A and B as vmcnt (only for PGR=1) + readsToWaitAdjust = len(list(writer.globalReadACode.middle.items())) + len(list(writer.globalReadBCode.middle.items())) + for wc in wcList: + replaceHolder(wc, (readsToWaitAdjust)) + + imod.add(item) + # schedule global instruction that need to be scheduled later + if localwriteCnt % PRECISION == (numLocalWritesPerSched % PRECISION): + reads = 0 + while itemsGRToSchedLater: + itemGR = itemsGRToSchedLater[0] + readsInc = itemGR.countType(GlobalReadInstruction) + reads = reads + readsInc + if reads > 1: + break + # PK and StoreCUnroll is removed so you cannot find any HolderContainer in s_waitcnt + hasHolder, wcList = hasHolderInWaitCnt(itemGR) + if hasHolder: + for wc in wcList: + replaceHolder(wc, (readsToWait)) + imod.add(itemGR) + else: + imod.add(itemGR) + readsToWait = readsToWait + readsInc # GR instruction increments vmcnt + itemsGRToSchedLater.pop(0) + localwriteCnt += 1 + writer.perIterLocalWriteCode[u].add(imod) + + imodNGLL.add(copy.deepcopy(item)) + if lastLc: + # local write code for NGLL should be updated at the last lc + # in init acc opt case, the last inner loop generated is not for the last lc. + # in that case, local write code for NGLL is not as expected. + writer.perIterLocalWriteCodeNGLL[u].add(imodNGLL) + + itemsLWToSched = itemsLWToSched[itemPerIter:] + + # should never run out of items to schedule + assert not itemsLWToSched # should have scheduled everthing already + +################################################################################ +################################################################################ +### +### Helper function +### +################################################################################ +################################################################################ + +def hasHolderInWaitCnt(module: Item): + wcList = [] + hasHolder = False + if isinstance(module, Module): + for item in module.items(): + tmpHasHolder, tmpList = hasHolderInWaitCnt(item) + hasHolder = hasHolder or tmpHasHolder + wcList.extend(tmpList) + elif isinstance(module, SWaitCnt): + wcList.append(module) + if isinstance(module.lgkmcnt, HolderContainer) or \ + isinstance(module.vmcnt, HolderContainer) or \ + isinstance(module.vscnt, HolderContainer): + hasHolder = True + return hasHolder, wcList diff --git a/tensilelite/Tensile/Components/ShiftVectorComponents.py b/tensilelite/Tensile/Components/ShiftVectorComponents.py new file mode 100644 index 0000000000..5f1dce6605 --- /dev/null +++ b/tensilelite/Tensile/Components/ShiftVectorComponents.py @@ -0,0 +1,309 @@ +################################################################################ +# Copyright 2021-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import Label, Module, VCC, DSModifiers, \ + DSBPermuteB32, SBranch, SCBranchVCCNZ, \ + SMovB32, SMovB64, SNop, \ + SOrSaveExecB32, SOrSaveExecB64, SWaitCnt, \ + VAccvgprReadB32, VAccvgprWriteB32, VAddCOU32, \ + VAndB32, VCmpEQU32, VCmpLtU32, VCmpXEqU32, \ + VCndMaskB32, VMovB32, VMulI32I24, VLShiftLeftB32, \ + VLShiftRightB32, VSubU32, \ + RegisterPoolResource, staticMultiply, vectorStaticDivide, \ + vectorStaticRemainder, vgpr, sgpr, accvgpr, log2 +from ..Component import ShiftVectorComponents + +class ShiftVectorComponentsMFMA(ShiftVectorComponents): + kernel = {"EnableMatrixInstruction": True} + + """ + Shift Vector Components d0,1 + """ + def __call__(self, writer, kernel, tP): + """ when we enable shift ptr with vectorwidth(2), we shift global read on edge block when size % vectorwidth != 0. + For example if M size == 3 vector width == 2, we want to do global read for [0-1] and [2-3]. + But 3 is not in memory object, so we shift to do global read [0-1] and [1-2]. + So buffer become [0, 1, 1, 2], assume result in register is same as input [0, 1, 1, 2] + We need to shift it back to [0, 1, 2]. + + In MFMA outputs, We have numContinuousOutput(4) for each thread. + We have numThreadInWave(64) threads. + number of thread in N is sames as kernel["MatrixInstN"] (32) + number of thread in M is numThreadInWave/numOutputThreads1 = 2 + stride of continuous output for each thread (numSubOutputPerWave0) is numOutputThreads0 * numContinuousOutput, (8). + we have numSubOutputGroupsPerWave0 which is 4 (kernel[tP["mt"]](64) // numSubOutputPerWave0(8)) + + So we do shift back by below algorithm. + 1. check if M_size % GlobalLoadVectorWidth != 0, return if == 0 + 2. decide which subgroup we need to shift, M_size(3) means 3/8 = group 0 + 3. decide which thread we need to shift, we have different groups of thread, (0-31) for first group, (32-63) for second group. + 4. decide which shift block (subTile1) we want to shift. for ex [0-1], [1-2], we want to shift second subtile + """ + + # TODO: use this for non SourceSwap for B? + # this part can support non SourceSwap for B + # But let non SourceSwap for B go original shiftptr path + if (not kernel["SourceSwap"]) and tP["isB"]: + return Module("ShiftVectorComponentsMFMA (Empty)") + + # common parameter + regPerElem = kernel["MIRegPerOut"] + glvw = tP["glvw"] + numThreadInWave = writer.kernel["WavefrontSize"] + accImOffset = writer.AccVgprImagNumOffset(kernel) + vectorWidth = kernel["VectorWidth"] if (kernel["SourceSwap"] and tP["isA"]) else 1 + + # use to handle MatrixInst 4x4 + matrixInstM = kernel["MatrixInstM"] * kernel["MatrixInstBM"] if (kernel["MatrixInstM"] == 4) else kernel["MatrixInstM"] + matrixInstN = kernel["MatrixInstN"] * kernel["MatrixInstBN"] if (kernel["MatrixInstN"] == 4) else kernel["MatrixInstN"] + matrixInstBM = 1 if (kernel["MatrixInstM"] == 4) else kernel["MatrixInstBM"] + matrixInstBN = 1 if (kernel["MatrixInstN"] == 4) else kernel["MatrixInstBN"] + + # unify process for dimension M/N + matrixInstCoal = matrixInstM if tP["isA"] else matrixInstN + matrixInstPrep = matrixInstN if tP["isA"] else matrixInstM + matrixInstBCoal = matrixInstBM if tP["isA"] else matrixInstBN + matrixInstBPrep = matrixInstBN if tP["isA"] else matrixInstBM + miWaveGroupCoal = kernel["MIWaveGroup"][0] if tP["isA"] else kernel["MIWaveGroup"][1] + miWGIdStride = numThreadInWave if tP["isA"] else (numThreadInWave * kernel["MIWaveGroup"][0]) + miWaveTitleCoal = kernel["MIWaveTile"][0] if tP["isA"] else kernel["MIWaveTile"][1] + miWaveTitlePrep = kernel["MIWaveTile"][1] if tP["isA"] else kernel["MIWaveTile"][0] + + # unify process for SourceSwap and non-SourceSwap + conThInProcDim = kernel["SourceSwap"] ^ tP["isB"] # continuous threads in processed dimension(Coalesced dimension) + + threadInterval = 1 if conThInProcDim else matrixInstPrep + numThreadInCoal = matrixInstCoal if conThInProcDim else (numThreadInWave // matrixInstPrep) + + numContOutCoal = 1 if conThInProcDim else kernel["MIOutputVectorWidth"] + allContOutCoal = numContOutCoal * vectorWidth + + OutBlocksInMI = (matrixInstCoal * matrixInstPrep) // numThreadInWave // numContOutCoal + OutBlocksInMI = 1 if conThInProcDim else OutBlocksInMI + + subMBShapeCoal = (matrixInstCoal * vectorWidth) if conThInProcDim else ((numThreadInWave // matrixInstPrep) * numContOutCoal) + MBShapeCoal = subMBShapeCoal * OutBlocksInMI + MIBShapeCoal = MBShapeCoal * matrixInstBCoal + WGShapeCoal = MIBShapeCoal * miWaveGroupCoal + miOuterTTCoal = miWaveTitleCoal // vectorWidth + + numOutputsPrep = (matrixInstCoal * matrixInstPrep // numThreadInWave) if conThInProcDim else 1 + numOutputsPrep = numOutputsPrep * matrixInstBPrep * miWaveTitlePrep + complexMultiplier = 2 if kernel["ProblemType"]["DataType"].isComplex() else 1 + + # unify process for dimension M/N + regStrideCoal = 1 if tP["isA"] else numOutputsPrep + regStridePrep = miOuterTTCoal * matrixInstBCoal * OutBlocksInMI * allContOutCoal if tP["isA"] else 1 + + + # labels for shiftptr + glvwLabels = [] + MBblockLabels = [] + VWBlockLabels = [] + for i in range(0, glvw): # grvw block + r = (i+1) % glvw # r = [1,2,3,...,glvw-1, 0], the last one glvwLabels[glvw-1] stores for r=0 -> no shift + comment = "end shift0" if i == glvw-1 else "" + label = Label(writer.labels.getName("ShiftVectorComponents%u_GLVW%u" % (tP["idx"], r) ), comment) + glvwLabels.append(label) + subMBLabels = [] + subVWBlockLabels = [] + for mb in range(0, OutBlocksInMI * matrixInstBCoal * miOuterTTCoal): # unit block of each thread + label = Label(writer.labels.getName("ShiftVectorComponents%u_GLVW%u_BM%u" % (tP["idx"], r, mb)), "") + subMBLabels.append(label) + sub2VWBlockLabels = [] + for vw in range(0, max(1, allContOutCoal//glvw)): # vw block of glvw + label = Label(writer.labels.getName("ShiftVectorComponents%u_GLVW%u_BM%u_VW%u" % (tP["idx"], r, mb, vw)), "") + sub2VWBlockLabels.append(label) + subVWBlockLabels.append(sub2VWBlockLabels) + MBblockLabels.append(subMBLabels) + VWBlockLabels.append(subVWBlockLabels) + + with writer.allocTmpSgpr(writer.laneSGPRCount) as tmpSgprInfo: + # wgMT value + tmpSgpr = tmpSgprInfo.idx + tmpVgpr = writer.vgprPool.checkOutAligned(2,2) + tmpVgprRes = RegisterPoolResource(tmpVgpr, 2) + dummy = writer.vgprPool.checkOut(1) + wgMT = writer.vgprPool.checkOut(1) + wg = tP["wg"] + + module = Module("ShiftVectorComponentsMFMA") + + # get M size of edge block + mtReg = writer.vgprPool.checkOut(1) + module.add(VMovB32(dst=vgpr(wgMT), src=sgpr(wg))) + module.add(VMulI32I24(dst=vgpr(wgMT), src0=hex(-kernel[tP["mt"]]), src1=vgpr(wgMT), comment="wg*MT")) + module.add(VAddCOU32(dst=vgpr(wgMT), dst1=VCC(), src0=sgpr("SizesFree+%u"%tP["idx"]), src1=vgpr(wgMT), comment="wgMT = Size - wg*MT")) + module.add(VMovB32(dst=vgpr(mtReg), src=hex(kernel[tP["mt"]]), comment="MT")) + module.add(VCmpLtU32(dst=sgpr(tmpSgpr,writer.laneSGPRCount), src0=vgpr(wgMT), src1=vgpr(mtReg), comment="wgMT < MT")) + module.add(VCndMaskB32(dst=vgpr(wgMT), src0=vgpr(mtReg), src1=vgpr(wgMT), src2=sgpr(tmpSgpr,writer.laneSGPRCount), comment="wgMT = (wgMT < MT) ? wgMT : MT" )) + + # identify which wave have to process + wReg = writer.vgprPool.checkOut(1) + sReg = writer.vgprPool.checkOut(1) + module.add(vectorStaticDivide(wReg, "Serial", miWGIdStride, tmpVgprRes)) + module.add(vectorStaticRemainder(dummy, wReg, wReg, miWaveGroupCoal, tmpVgprRes, tmpSgprInfo)) + module.add(vectorStaticDivide(sReg, wgMT, MIBShapeCoal, tmpVgprRes)) + module.add(vectorStaticRemainder(dummy, sReg, sReg, miWaveGroupCoal, tmpVgprRes, tmpSgprInfo)) + module.add(VCmpEQU32(dst=sgpr(tmpSgpr,writer.laneSGPRCount), src0=vgpr(sReg), src1=vgpr(wReg), comment="wave_id == block_belong_to_wave?" )) + module.add(VCndMaskB32(dst=vgpr(wgMT), src0=vgpr(mtReg), src1=vgpr(wgMT), src2=sgpr(tmpSgpr,writer.laneSGPRCount), comment="wgMT = (wgMT < MT) ? wgMT : MT" )) + writer.vgprPool.checkIn(mtReg) + writer.vgprPool.checkIn(sReg) + + # mbReg: which mb block meed to shift, mb(matrixInstM*VectorWidth) + module.addComment1("mbReg: which mb block need to shift, mb(matrixInstCoal(%u) * VectorWidth(%u))" % (matrixInstCoal, vectorWidth)) + mbReg = writer.vgprPool.checkOut(1) + tReg = writer.vgprPool.checkOut(1) + module.add(vectorStaticDivide(mbReg, wgMT, subMBShapeCoal, tmpVgprRes)) + module.add(staticMultiply(vgpr(tReg), vgpr(wReg), (matrixInstBCoal * OutBlocksInMI), tmpSgprInfo)) + module.add(VSubU32(dst=vgpr(mbReg), src0=vgpr(mbReg), src1=vgpr(tReg))) + writer.vgprPool.checkIn(tReg) + + # gbReg: glvw block id + module.addComment1("gbReg: glvw block id") + gbReg = writer.vgprPool.checkOut(1) + module.add(vectorStaticDivide(gbReg, wgMT, glvw, tmpVgprRes)) + + # tgbReg: thread in glvw block + module.addComment1("tgbReg: glvw block id") + tgbReg = writer.vgprPool.checkOut(1) + module.add(vectorStaticDivide(tgbReg, "Serial", threadInterval, tmpVgprRes)) + module.add(vectorStaticRemainder(dummy, tgbReg, tgbReg, numThreadInCoal, tmpVgprRes, tmpSgprInfo)) + module.add(staticMultiply(vgpr(tgbReg), vgpr(tgbReg), allContOutCoal, tmpSgprInfo)) + module.add(vectorStaticDivide(tgbReg, tgbReg, glvw, tmpVgprRes)) + module.add(staticMultiply(vgpr(wReg), vgpr(wReg), MIBShapeCoal//glvw, tmpSgprInfo)) + module.add(VAddCOU32(dst=vgpr(tgbReg), dst1=VCC(), src0=vgpr(wReg), src1=vgpr(tgbReg), comment="tgbReg = (tid_coal * continOut) / GLVW")) + module.add(VSubU32(dst=vgpr(gbReg), src0=vgpr(gbReg), src1=vgpr(tgbReg))) + writer.vgprPool.checkIn(wReg) + writer.vgprPool.checkIn(tgbReg) + + # vw block of glvw + module.addComment1("vwReg: glvw in which vw block?") + vwReg = writer.vgprPool.checkOut(1) + module.add(VAndB32(dst=vgpr(vwReg), src0=allContOutCoal-1, src1=vgpr(wgMT), comment="permute register between threads")) + module.add(VLShiftRightB32(dst=vgpr(vwReg), shiftHex=log2(glvw), src=vgpr(vwReg), comment="permute register between threads")) + + # rReg : reminder of M_size % vectorwidth + # decide to jump to block which handle this case, M_size % vector width + module.addComment1("rReg : reminder of M_size % GlobalLoadVectorWidth") + rReg = writer.vgprPool.checkOut(1) + module.add(vectorStaticRemainder(dummy, rReg, wgMT, glvw, tmpVgprRes, tmpSgprInfo)) + for r in range(1, glvw): + module.add(VCmpEQU32(dst=VCC(), src0=vgpr(rReg), src1=hex(r), comment="wgMT%%VW == %u"%r )) + module.add(SCBranchVCCNZ(labelName=glvwLabels[(r-1)].getLabelName(), comment="branch to shift d%u r=%u"%(tP["idx"], r))) + module.add(SBranch(labelName=glvwLabels[glvw-1].getLabelName(), comment="no shifting" )) + writer.vgprPool.checkIn(rReg) + + _, arch2acc = writer.AccToArchMapper(kernel) + + # blocks for handle M_size % vector width + for r in range(1, glvw): + module.addComment2("shift d%u r=%u"%(tP["idx"], r)) + module.add(glvwLabels[r-1]) + for tt in range(0, miOuterTTCoal): + for bm in range(0, matrixInstBCoal): + for ob in range(0, OutBlocksInMI): + label = ob + OutBlocksInMI * (bm + matrixInstBCoal * tt) + target = ob + OutBlocksInMI * (bm + matrixInstBCoal * miWaveGroupCoal * tt) + module.add(VCmpEQU32(dst=VCC(), src0=vgpr(mbReg), src1=hex(target))) + module.add(SCBranchVCCNZ(labelName=MBblockLabels[r-1][label].getLabelName(), comment="branch to shift d%u r%u mb%u" % (tP["idx"], r, label))) + + for r in range(1, glvw): + for mb in range(0, miOuterTTCoal * matrixInstBCoal * OutBlocksInMI): + module.addComment2("shift d%u r=%u mb=%u"%(tP["idx"], r, mb)) + MBblockLabels[r-1][mb].comment = "r%u mb%u"%(r, mb) + module.add(MBblockLabels[r-1][mb]) + for vw in range(0, max(1, allContOutCoal//glvw)): + module.add(VCmpEQU32(dst=VCC(), src0=vgpr(vwReg), src1=hex(vw))) + module.add(SCBranchVCCNZ(labelName=VWBlockLabels[r-1][mb][vw].getLabelName(), comment="branch to shift d%u r%u mb%u vw%u" % (tP["idx"], r, mb, vw))) + + # blocks for handle M_size % vector width + tReg = writer.vgprPool.checkOut(min(glvw, allContOutCoal)) + for r in range(1, glvw): + for tt in range(0, miOuterTTCoal): + for bm in range(0, matrixInstBCoal): + for ob in range(0, OutBlocksInMI): + mb = ob + OutBlocksInMI * (bm + matrixInstBCoal * tt) + for vw in range(0, max(1, allContOutCoal//glvw)): + module.addComment2("shift d%u r=%u mb=%u vw%d"%(tP["idx"], r, mb, vw)) + VWBlockLabels[r-1][mb][vw].comment = "r%u mb%u vw%u"%(r, mb, vw) + module.add(VWBlockLabels[r-1][mb][vw]) + module.add(SMovB32(dst=sgpr(tmpSgpr), src=(((ob*subMBShapeCoal + bm*MBShapeCoal + tt*WGShapeCoal) // glvw) + vw))) + module.add(VCmpXEqU32(dst=sgpr(tmpSgpr, writer.laneSGPRCount), src0=vgpr(gbReg), src1=sgpr(tmpSgpr), comment="is thread in edge glvw region" )) + module.add(VAndB32(dst=vgpr(tmpVgpr), src0=kernel["WavefrontSize"]-1, src1=vgpr("Serial"), comment="permute register between threads")) + module.add(VLShiftLeftB32(dst=vgpr(tmpVgpr), shiftHex=log2(writer.bpr), src=vgpr(tmpVgpr), comment="permute register between threads")) + + for ot in range(numOutputsPrep): + for c in range(complexMultiplier): + for nr in range(regPerElem): + vgprOffsetForSCIU = 0 + copyInst = VAccvgprReadB32 if not kernel["MIArchVgpr"] else VMovB32 + for e in range(min(r, allContOutCoal)): + src = (e+(glvw-r)) % allContOutCoal + srcVgpr = (src + (vw * glvw) + allContOutCoal * mb) * regStrideCoal + srcVgpr = srcVgpr + ot * regStridePrep + srcVgpr = arch2acc[srcVgpr] * regPerElem + nr + c * accImOffset + vgprOffsetForSCIU + srcVal = accvgpr(srcVgpr) if not kernel["MIArchVgpr"] else vgpr(srcVgpr) + module.add(copyInst(dst=vgpr(tReg+e), src=srcVal, comment="glvw %u mb %u tt1 %u r %u" % (r, mb, ot, nr))) + + if not kernel["MIArchVgpr"]: + module.add(SNop(waitState=1, comment="v_accvgpr read vgpr after write vgpr: 2 wait states")) + + needWait = False + for e in range(min(r, allContOutCoal)): + crossThread = (e+(glvw-r)) // allContOutCoal + if crossThread != 0: + ds = DSModifiers(na=1, offset=crossThread*threadInterval*4) + module.add(DSBPermuteB32(dst=vgpr(tReg+e), src0=vgpr(tmpVgpr), src1=vgpr(tReg+e), ds=ds, comment="permute edge values")) + needWait = True + + if needWait: + module.add(SWaitCnt(waitAll=True, comment="wait for swizzle operation")) + + copyInst = VAccvgprWriteB32 if not kernel["MIArchVgpr"] else VMovB32 + for e in range(min(r, allContOutCoal)): + dstVgpr = (e + (vw * glvw) + allContOutCoal * mb) * regStrideCoal + dstVgpr = dstVgpr + ot * regStridePrep + dstVgpr = arch2acc[dstVgpr] * regPerElem + nr + c * accImOffset + vgprOffsetForSCIU + dstStr = accvgpr(dstVgpr) if not kernel["MIArchVgpr"] else vgpr(dstVgpr) + module.add(copyInst(dst=dstStr, src=vgpr(tReg+e))) + + # end shift reset mask and jump out + all1mask = "0xFFFFFFFF" if (kernel["WavefrontSize"] == 32) else "0xFFFFFFFFFFFFFFFF" + SMovBX = SMovB64 if kernel["WavefrontSize"] == 64 else SMovB32 + SOrSaveExecBX = SOrSaveExecB64 if kernel["WavefrontSize"] == 64 else SOrSaveExecB32 + module.add(SMovBX(dst=sgpr(tmpSgpr, writer.laneSGPRCount), src=all1mask, comment="to restore all threads active")) + module.add(SOrSaveExecBX(dst=VCC(), src=sgpr(tmpSgpr,writer.laneSGPRCount), comment="all threads active")) + module.add(SBranch(labelName=glvwLabels[glvw-1].getLabelName(), comment="done shifting" )) + module.addSpaceLine() + + module.add(glvwLabels[glvw-1]) + writer.vgprPool.checkIn(tReg) + + # checkin scratch vgprs + writer.vgprPool.checkIn(tmpVgpr) + writer.vgprPool.checkIn(wgMT) + writer.vgprPool.checkIn(dummy) + writer.vgprPool.checkIn(gbReg) + writer.vgprPool.checkIn(vwReg) + writer.vgprPool.checkIn(mbReg) + + return module diff --git a/tensilelite/Tensile/Components/Signature.py b/tensilelite/Tensile/Components/Signature.py new file mode 100644 index 0000000000..9646638396 --- /dev/null +++ b/tensilelite/Tensile/Components/Signature.py @@ -0,0 +1,156 @@ +################################################################################ +# Copyright 2020-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..Component import Signature +from ..Common import globalParameters +from ..TensileInstructions import SignatureBase +from ..TensileInstructions import SignatureValueKind as SVK + +from math import ceil + +class SignatureCOV3(Signature): + kernel = {"CodeObjectVersion": "V3"} + + def __call__(self, writer) -> SignatureBase: + kernel = writer.kernel + + # kern arg size + kernArgReg = 0 + kernArgReg += 3*writer.rpga + kernArgReg += max(1,int(writer.bpeAB/4)) # alpha + if kernel["ProblemType"]["UseBeta"]: + kernArgReg += max(1,int(writer.bpeCexternal/4)) # beta + kernArgReg += kernel["ProblemType"]["NumIndicesC"] # strides + kernArgReg += kernel["ProblemType"]["NumIndicesC"] # strides + kernArgReg += len(kernel["ProblemType"]["IndexAssignmentsA"]) # strides + kernArgReg += len(kernel["ProblemType"]["IndexAssignmentsB"]) # strides + if not kernel["ProblemType"]["UseInitialStridesAB"]: + kernArgReg -= 2 # strides + if not kernel["ProblemType"]["UseInitialStridesCD"]: + kernArgReg -= 2 # strides + kernArgReg += kernel["ProblemType"]["NumIndicesSummation"] + kernArgReg += kernel["ProblemType"]["NumIndicesC"] + if globalParameters["DebugKernel"]: + kernArgReg += writer.rpga # debug buffer + # kernArgBytes = kernArgReg * 4 # bytes/reg + + group_segment_size = kernel["LdsNumElements"] * writer.bpeAB + + sgprWgZ = 1 if kernel["ProblemType"]["NumIndicesC"] > 2 else 0 + signature = SignatureBase(kernelName=writer.kernelName, + codeObjectVersion="v3", + groupSegmentSize=group_segment_size, + sgprWorkGroup=[1, 1, sgprWgZ], + vgprWorkItem=0, + flatWorkGroupSize=(kernel["SubGroup0"] * kernel["SubGroup1"] * kernel["LocalSplitU"])) + + srcValueType = kernel["ProblemType"]["DataType"].toNameAbbrev() + dstValueType = kernel["ProblemType"]["DestDataType"].toNameAbbrev() + cptValueType = kernel["ProblemType"]["ComputeDataType"].toNameAbbrev() + actValueType = kernel["ProblemType"]["ActivationComputeDataType"].toNameAbbrev() + + if globalParameters["DebugKernel"]: + signature.addArg("AddressDbg", SVK.SIG_GLOBALBUFFER, "struct", "generic") + signature.addArg("sizeC", SVK.SIG_VALUE, "u64") + signature.addArg("sizeA", SVK.SIG_VALUE, "u64") + signature.addArg("sizeB", SVK.SIG_VALUE, "u64") + signature.addArg( "D", SVK.SIG_GLOBALBUFFER, dstValueType, "generic") + signature.addArg( "C", SVK.SIG_GLOBALBUFFER, dstValueType, "generic") + signature.addArg( "A", SVK.SIG_GLOBALBUFFER, srcValueType, "generic") + signature.addArg( "B", SVK.SIG_GLOBALBUFFER, srcValueType, "generic") + + # Note: We use packed f16 if alpha and beta are f16 + if kernel["ProblemType"]["ComputeDataType"].isHalf(): + cptValueType = 'pkf16' + signature.addArg( "alpha", SVK.SIG_VALUE, cptValueType) + if kernel["ProblemType"]["UseBeta"]: + signature.addArg("beta", SVK.SIG_VALUE, cptValueType) + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + signature.addArg("bias", SVK.SIG_GLOBALBUFFER, srcValueType, "generic") + + if ((kernel["ProblemType"]["ActivationType"] != 'none') and (kernel["GlobalSplitU"] == 1) \ + and kernel["ActivationFused"]): + if kernel["ProblemType"]["ActivationComputeDataType"].isHalf(): + actValueType = 'pkf16' + for name in kernel["ProblemType"]["ActivationType"].getAdditionalArgStringList(): + signature.addArg( name, SVK.SIG_VALUE, actValueType) + if kernel["ProblemType"]["ActivationType"] == 'all': + signature.addArg( "activationType", SVK.SIG_VALUE, "u32") + + for i in range(0, writer.numSgprStridesD): + signature.addArg( "strideD%u"%i, SVK.SIG_VALUE, "u32") + + for i in range(0, writer.numSgprStridesC): + signature.addArg( "strideC%u"%i, SVK.SIG_VALUE, "u32") + + for i in range(0, writer.numSgprStridesA): + signature.addArg( "strideA%u"%i, SVK.SIG_VALUE, "u32") + + for i in range(0, writer.numSgprStridesB): + signature.addArg( "strideB%u"%i, SVK.SIG_VALUE, "u32") + + for i in range(0, writer.numSgprSizesFree): + signature.addArg( "SizesFree%u"%i, SVK.SIG_VALUE, "u32") + + for i in range(0, writer.numSgprSizesSum): + signature.addArg( "SizesSum%u"%i, SVK.SIG_VALUE, "u32") + + for idxChar in kernel["PackedC0IdxChars"][:-1]: + signature.addArg("MagicNumberSize%s"%idxChar, SVK.SIG_VALUE, "u32") + signature.addArg( "MagicShiftSize%s"%idxChar, SVK.SIG_VALUE, "u32") + + signature.addArg( "OrigStaggerUIter", SVK.SIG_VALUE, "i32") + + signature.addArg( "NumWorkGroups0", SVK.SIG_VALUE, "u32") + signature.addArg( "NumWorkGroups1", SVK.SIG_VALUE, "u32") + + signature.addArg( "NumFullBlocks", SVK.SIG_VALUE, "u32") + signature.addArg( "WgmRemainder1", SVK.SIG_VALUE, "u32") + signature.addArg( "MagicNumberWgmRemainder1", SVK.SIG_VALUE, "u32") + + signature.addArg( "OffsetD", SVK.SIG_VALUE, "u32") + signature.addArg( "OffsetC", SVK.SIG_VALUE, "u32") + signature.addArg( "OffsetA", SVK.SIG_VALUE, "u32") + signature.addArg( "OffsetB", SVK.SIG_VALUE, "u32") + + signature.addArg( "padding", SVK.SIG_VALUE, "u32") + + self.addOptConfigComment(signature, + tt=[kernel["ThreadTile0"], kernel["ThreadTile1"]], + sg=[kernel["SubGroup0"], kernel["SubGroup1"]], + vw=kernel["VectorWidth"], + glvwA=kernel["GlobalLoadVectorWidthA"], + glvwB=kernel["GlobalLoadVectorWidthB"], + d2lA=kernel["DirectToLdsA"], + d2lB=kernel["DirectToLdsB"], + useSgprForGRO=kernel["_UseSgprForGRO"]) + + return signature + + def addOptConfigComment(self, signature: SignatureBase, tt, sg, vw, glvwA, glvwB, d2lA, d2lB, useSgprForGRO): + signature.addDescriptionTopic("Optimizations and Config:") + signature.addDescriptionBlock("ThreadTile= %u x %u" % (tt[0], tt[1]) ) + signature.addDescriptionBlock("SubGroup= %u x %u" % (sg[0], sg[1]) ) + signature.addDescriptionBlock("VectorWidth=%u" % vw ) + signature.addDescriptionBlock("GlobalLoadVectorWidthA=%u, GlobalLoadVectorWidthB=%u" % (glvwA, glvwB) ) + signature.addDescriptionBlock("DirectToLdsA=%s" % d2lA ) + signature.addDescriptionBlock("DirectToLdsB=%s" % d2lB ) + signature.addDescriptionBlock("UseSgprForGRO=%s" % useSgprForGRO ) diff --git a/tensilelite/Tensile/Components/__init__.py b/tensilelite/Tensile/Components/__init__.py new file mode 100644 index 0000000000..4fa4e796e7 --- /dev/null +++ b/tensilelite/Tensile/Components/__init__.py @@ -0,0 +1,45 @@ +################################################################################ +# Copyright 2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +def use(): pass + +# This should list all of the files in the 'Components' directory, without extensions. +__all__ = [ + "MAC_BF16_HPA", + "MAC_F16", + "MAC_F16_HPA", + "MAC_F32", + "MAC_F32C", + "MAC_F64", + "MAC_F64C", + "MAC_I8_HPA", + "MAC_I8X4", + "Priority", + "Signature", + "LocalRead", + "ShiftVectorComponents", + "ComputeStoreVgprs", + "NotLocalFullTileElements", + "LraTileAssignment", + "PackData", + "SIA", + "GlobalWriteBatch" + ] diff --git a/tensilelite/Tensile/Configuration.py b/tensilelite/Tensile/Configuration.py new file mode 100644 index 0000000000..99046e94e9 --- /dev/null +++ b/tensilelite/Tensile/Configuration.py @@ -0,0 +1,917 @@ +from copy import deepcopy +import ast + +class ReadWriteTransformDict(dict): # dicts take a mapping or iterable as their optional first argument + """ + The focus of this class is to allow explicit customization of + reading and writing to attributes. + Users of this class might restrict read or write access to particular + variables, or perform some customization on those functionalities. + """ + + __slots__ = () # no __dict__ - that would be redundant + + readTransformFuncKey = '_readTransformFunc' + writeTransformFuncKey = '_writeTransformFunc' + + def __init__(self, readTransformFunc = None, writeTransformFunc = None): + super().__init__() + if readTransformFunc is not None and callable(readTransformFunc): + self.setReadTransform(readTransformFunc) + if writeTransformFunc is not None and callable(writeTransformFunc): + self.setWriteTransform(writeTransformFunc) + + def __getitem__(self, key): + return self.readWithTransform(key) if self.hasReadTransform() else self.readNoTransform(key) + + def __setitem__(self, key, value): + self.writeWithTransform(key, value) if self.hasWriteTransform() else self.writeNoTransform(key, value) + + def __getattr__(self, key): + return self[key] + + def __setattr__(self, key, value): + self[key] = value + + def __copy__(self): # don't delegate w/ super - dict.copy() -> dict :( + cls = self.__class__ + result = cls.__new__(cls) + result.update(self) + return result + + def __deepcopy__(self, memo): # don't delegate w/ super - dict.copy() -> dict :( + cls = self.__class__ + result = cls.__new__(cls) + memo[id(self)] = result + for key in self.keys(): + result.writeNoTransform(key, deepcopy(self.readNoTransform(key))) + return result + + def __repr__(self): + return "\n".join(self.__toPrettyLines()) + + def __toPrettyLines(self): + """ + Embedded values are sometimes hard to read and understand + with so many layers, so add some tabs and line breaks + to help distinguish hierarchy trees of objects in the dict. + """ + prettyLines = [] + prettyLines += ["<{0}(".format(type(self).__name__)] + for key, value in self.items(): + if isinstance(value, ReadWriteTransformDict): + valuePrettyLines = value.__toPrettyLines() + prettyLines += ["\t{0}: {1}".format(key, valuePrettyLines.pop(0))] + for line in valuePrettyLines: + prettyLines += ["\t{0}".format(line)] + else: + prettyLines += ["\t{0}: {1}".format(key, value)] + prettyLines += [")>"] + return prettyLines + + def get(self, key, default=None): + try: + return self[key] + except: + return default + + def set(self, key, value): + try: + self[key] = value + except: + pass + + # Transform tests + def hasReadTransform(self): + return self.readTransformFuncKey in self + + def hasWriteTransform(self): + return self.writeTransformFuncKey in self + + # Omit transform + def readNoTransform(self, name): + return super().__getitem__(name) + + def writeNoTransform(self, name, value): + super().__setitem__(name, value) + + # Use transform + def readWithTransform(self, name): + return self.getReadTransform()(self, name) + + def writeWithTransform(self, name, value): + self.getWriteTransform()(self, name, value) + + # Transform get/set + def getReadTransform(self): + return self.readNoTransform(self.readTransformFuncKey) if self.hasReadTransform() else None + + def setReadTransform(self, readTransformFunc): + self.writeNoTransform(self.readTransformFuncKey, readTransformFunc) + if readTransformFunc is None: + self.pop(self.readTransformFuncKey, None) + + def getWriteTransform(self): + return self.readNoTransform(self.writeTransformFuncKey) if self.hasWriteTransform() else None + + def setWriteTransform(self, writeTransformFunc): + self.writeNoTransform(self.writeTransformFuncKey, writeTransformFunc) + if writeTransformFunc is None: + self.pop(self.writeTransformFuncKey, None) + + def toDict(self): + return dict(zip(self.keys(), self.values())) + + @staticmethod + def flattenDict(obj, prefix = "", separator="."): + """ + Sometimes it is favorable to reduce to a single hierarchy + dict for readability or debugging. + """ + result = ReadWriteTransformDict() + for key in obj.keys(): + attribute = obj.readNoTransform(key) + attribute = attribute.readNoTransform("value") if isinstance(attribute, ReadWriteTransformDict) else attribute + if isinstance(attribute, dict): + subDict = ReadWriteTransformDict.toFlattenedDict(attribute, key, separator) + for subKey in subDict.keys(): + newKey = separator.join([prefix, subKey]) if prefix != "" else subKey + result[newKey] = deepcopy(subDict[subKey]) + else: + newKey = separator.join([prefix, key]) if prefix != "" else key + result[newKey] = deepcopy(obj[key]) + return result + + def toFlattenedDict(self, prefix = "", separator="."): + return ReadWriteTransformDict.flattenDict(self, prefix, separator) + +class Parameter(ReadWriteTransformDict): + """ + The Parameter class will be used as the main + container for configuration properties. + For this class we want to group configurations + by name, value, default value and descriptions. + No other attributes may be set. + All operators and reverse operators are overloaded + to use value attribute as the operand as this is + usually the one we are interested in. + """ + __slots__ = () + + # Simplification of creating attributes + def createAttr(self, name, value): + super().writeNoTransform(name, value) + + def __init__(self, name, initialValue, defaultValue=None, description=""): + self.createAttr("type", type(initialValue)) + self.createAttr("name", name) + self.createAttr("value", initialValue) + self.createAttr("defaultValue", defaultValue) + self.createAttr("description", description) + +# if defaultValue is None: +# self.createAttr("defaultValue", initialValue) +# else: +# self.createAttr("defaultValue", defaultValue) + + # Restrict writing to only known fields + def writeXForm(self, key, value): + if key not in self or key == "type": + raise AttributeError("Cannot write attribute: {0}".format(key)) + else: + # Ensure incoming values are same type to allow assignment + objType = self.readNoTransform("type") + incomingType = type(value) + if key == "value" and objType is not incomingType: + raise AttributeError("Type preservation: stored {0} != incoming {1}".format(objType, incomingType)) + else: + self.writeNoTransform(key, value) + + super().__init__(None, writeXForm) + + # Boolean ops + def __lt__(self, rhs): + if isinstance(rhs, Parameter): + return self.value < rhs.value + else: + return self.value < rhs + + def __rlt__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value < self.value + else: + return lhs < self.value + + def __le__(self, rhs): + if isinstance(rhs, Parameter): + return self.value <= rhs.value + else: + return self.value <= rhs + + def __rle__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value <= self.value + else: + return lhs <= self.value + + def __eq__(self, rhs): + if isinstance(rhs, Parameter): + return self.value == rhs.value + else: + return self.value == rhs + + def __req__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value == self.value + else: + return lhs == self.value + + def __ne__(self, rhs): + if isinstance(rhs, Parameter): + return self.value != rhs.value + else: + return self.value != rhs + + def __rne__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value != self.value + else: + return lhs != self.value + + def __gt__(self, rhs): + if isinstance(rhs, Parameter): + return self.value > rhs.value + else: + return self.value > rhs + + def __rgt__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value > self.value + else: + return lhs > self.value + + def __ge__(self, rhs): + if isinstance(rhs, Parameter): + return self.value >= rhs.value + else: + return self.value >= rhs + + def __rge__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value >= self.value + else: + return lhs >= self.value + + # Binary Ops with their reverses + def __add__(self, rhs): + if isinstance(rhs, Parameter): + return self.value + rhs.value + else: + return self.value + rhs + + def __radd__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value + self.value + else: + return lhs + self.value + + def __sub__(self, rhs): + if isinstance(rhs, Parameter): + return self.value - rhs.value + else: + return self.value - rhs + + def __rsub__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value - self.value + else: + return lhs - self.value + + def __mul__(self, rhs): + if isinstance(rhs, Parameter): + return self.value * rhs.value + else: + return self.value * rhs + + def __rmul__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value * self.value + else: + return lhs * self.value + + def __truediv__(self, rhs): + if isinstance(rhs, Parameter): + return self.value / rhs.value + else: + return self.value / rhs + + def __rtruediv__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value / self.value + else: + return lhs / self.value + + def __floordiv__(self, rhs): + if isinstance(rhs, Parameter): + return self.value // rhs.value + else: + return self.value // rhs + + def __rfloordiv__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value // self.value + else: + return lhs // self.value + + def __mod__(self, rhs): + if isinstance(rhs, Parameter): + return self.value % rhs.value + else: + return self.value % rhs + + def __rmod__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value % self.value + else: + return lhs % self.value + + def __pow__(self, rhs): + if isinstance(rhs, Parameter): + return self.value ** rhs.value + else: + return self.value ** rhs + + def __rpow__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value ** self.value + else: + return lhs ** self.value + + def __rshift__(self, rhs): + if isinstance(rhs, Parameter): + return self.value >> rhs.value + else: + return self.value >> rhs + + def __rrshift__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value >> self.value + else: + return lhs >> self.value + + def __lshift__(self, rhs): + if isinstance(rhs, Parameter): + return self.value << rhs.value + else: + return self.value << rhs + + def __rlshift__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value << self.value + else: + return lhs << self.value + + def __and__(self, rhs): + if isinstance(rhs, Parameter): + return self.value & rhs.value + else: + return self.value & rhs + + def __rand__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value & self.value + else: + return lhs & self.value + + def __or__(self, rhs): + if isinstance(rhs, Parameter): + return self.value | rhs.value + else: + return self.value | rhs + + def __ror__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value | self.value + else: + return lhs | self.value + + def __xor__(self, rhs): + if isinstance(rhs, Parameter): + return self.value ^ rhs.value + else: + return self.value ^ rhs + + def __rxor__(self, lhs): + if isinstance(lhs, Parameter): + return lhs.value ^ self.value + else: + return lhs ^ self.value + + def __bool__(self): + return bool(self.value) + + # Unary ops + def __neg__(self): + return -self.value + + def __pos__(self): + return +self.value + + def __invert__(self): + return ~self.value + + # Interface + def resetToDefault(self): + self.value = self.defaultValue + return self.value + + def getValue(self): + return self.value + + def getDefault(self): + return self.defaultValue + + def getDescription(self): + return self.description + +class CallableParameter(Parameter): + """ + This class is a Parameter who gets its value + dynamically at read time. It stores a function + that gets called with self as context, which the + result will replace the value attribute. + This is useful for retrieving values that are not + constants and must be retrieved from another function + calls + """ + __slots__ = () + + class BadFunc(Exception): + pass + + def __init__(self, name, callFunc, description=""): + super().__init__(name, 0, 0, description) + self.createAttr("callFunc", callFunc) + + # Before each read, call on self to update value + rXForm = self.getReadTransform() + def readXForm(obj, key): + if key == "value": + obj.writeNoTransform(key, obj.__call__()) + return rXForm(obj, key) if rXForm is not None else obj.readNoTransform(key) + self.setReadTransform(readXForm) + + # Prevent external writing to value attribute + # Will be updated by reads as callable is updated. + wXForm = self.getWriteTransform() + def writeXForm(obj, key, value): + if key == "value": + pass + else: + wXForm(obj, key, value) if wXForm is not None else obj.writeNoTransform(key, value) + self.setWriteTransform(writeXForm) + + def __call__(self): + return self.callFunc(self) + + @classmethod + def createBinaryOp(cls, lhs, rhs, op): + """ + This function will build a binary operation + function to call with two operands that are + bound to the parameter + """ + FuncMap = { + "And" : lambda lhs, rhs : lhs and rhs, + "Or" : lambda lhs, rhs : lhs or rhs, + "Lt" : lambda lhs, rhs : lhs < rhs, + "LtE" : lambda lhs, rhs : lhs <= rhs, + "Eq" : lambda lhs, rhs : lhs == rhs, + "NotEq" : lambda lhs, rhs : lhs != rhs, + "Gt" : lambda lhs, rhs : lhs > rhs, + "GtE" : lambda lhs, rhs : lhs >= rhs, + "Mult" : lambda lhs, rhs : lhs * rhs, + "Pow" : lambda lhs, rhs : lhs ** rhs, + "Div" : lambda lhs, rhs : lhs / rhs, + "FloorDiv" : lambda lhs, rhs : lhs // rhs, + "Mod" : lambda lhs, rhs : lhs % rhs, + "Add" : lambda lhs, rhs : lhs + rhs, + "Sub" : lambda lhs, rhs : lhs - rhs, + "BitAnd" : lambda lhs, rhs : lhs & rhs, + "BitOr" : lambda lhs, rhs : lhs | rhs, + "BitXor" : lambda lhs, rhs : lhs ^ rhs, + "LShift" : lambda lhs, rhs : lhs << rhs, + "RShift" : lambda lhs, rhs : lhs >> rhs, + "min" : lambda lhs, rhs : min(lhs, rhs), + "max" : lambda lhs, rhs : max(lhs, rhs), + } + + # Incoming op is a name + if isinstance(op, str): + assert op in FuncMap, "Missing operation in funcMap: {0}".format(op) + name = op + func = FuncMap[op] + # Incoming op assumed as a function + else: + name = "CustomBinaryOp" + func = op + + # Verify that the function is callable with 2 params + try: + func(lhs, rhs) + except Exception as ex: + raise CallableParameter.BadFunc from ex + + # Capture function and attached values + opKey = next((key for key in FuncMap if FuncMap[key] == op), None) + if opKey is not None and opKey in ["And", "Or", "Lt", "Le", "Eq", "NotEq", "Gt", "Ge"]: + callFunc = lambda obj : bool(func(obj.lhs, obj.rhs)) + else: + callFunc = lambda obj : func(obj.lhs, obj.rhs) + + binOp = cls(name, callFunc, "Binary operaton with two operands") + binOp.createAttr("lhs", lhs) + binOp.createAttr("rhs", rhs) + + return binOp + + @classmethod + def createUnaryOp(cls, rhs, op): + """ + This function will build a unary operation + function to call with one operand that is + bound to the parameter + """ + + FuncMap = { + "Not" : lambda val : not val, + "Invert" : lambda val : ~val, + "USub" : lambda val : -val, + "UAdd" : lambda val : +val, + "None" : lambda val : val + } + + # Incoming op is a name + if isinstance(op, str): + assert op in FuncMap, "Missing operation in funcMap: {0}".format(op) + name = op + func = FuncMap[op] + # Incoming op assumed as a function + else: + name = "CustomUnaryOp" + func = op + + # Verify that the function is callable with 1 param + try: + func(rhs) + except Exception as ex: + raise CallableParameter.BadFunc from ex + + # Capture function and attached values + opKey = next((key for key in FuncMap if FuncMap[key] == op), None) + if opKey is not None and opKey in ["Not"]: + callFunc = lambda obj : bool(func(obj.rhs)) + else: + callFunc = lambda obj : func(obj.rhs) + + unOp = cls(name, callFunc, "Unary operaton with one operand") + unOp.createAttr("rhs", rhs) + + return unOp + +class ExpressionEvaluator(object): + """ + This class is a recursive visitor of an ast module tree. + The ast tree is generated from a text expression (e.g. a > 5). + Given a context object to resolve names to values, the expression + evaluator will return a chain of callable parameters that can be + invoked to get the result of the expression. + Example: + context = {a : 10} + tree = ast.parse("a > 5", mode='exec') # ast library + exprEval = ExpressionEvaluator.evaluate(tree, context) + result = exprEval.__call__() # = True + """ + + __slots__ = () + + def evaluate(self, node, namesContext): + + #print("Type: {0}".format(type(node).__name__)) + #print(node) + + nodeType = type(node).__name__ + + # Root Level 0 node (required) + if nodeType == "Module": + # fields: ('body') + assert len(node.body) == 1, "Expecting only one expression" + return self.evaluate(node.body[0], namesContext) + + # Level 1 node (required) + elif nodeType == "Expr": + # fields: ('value') + result = self.evaluate(node.value, namesContext) + return result.value if hasattr(result, "value") else result + + # Binary operations nodes which take 2 operands + elif nodeType == "BinOp": + # fields: ('op', 'left', 'right') + lhs = self.evaluate(node.left, namesContext) + rhs = self.evaluate(node.right, namesContext) + op = type(node.op).__name__ + return CallableParameter.createBinaryOp(lhs, rhs, op) + + elif nodeType == "BoolOp": + # fields: ('op', 'values') + lhs = self.evaluate(node.values[0], namesContext) + rhs = self.evaluate(node.values[1], namesContext) + op = type(node.op).__name__ + return CallableParameter.createBinaryOp(lhs, rhs, op) + + elif nodeType == "Compare": + # fields: ('left', 'ops', 'comparators') + lhs = self.evaluate(node.left, namesContext) + for i in range(len(node.ops)): + rhs = self.evaluate(node.comparators[i], namesContext) + op = type(node.ops[i]).__name__ + result = CallableParameter.createBinaryOp(lhs, rhs, op) + lhs = result + return lhs + + # Unary operations which take 1 operand + elif nodeType == "UnaryOp": + # fields: ('op', 'operand') + value = self.evaluate(node.operand, namesContext) + op = type(node.op).__name__ + return CallableParameter.createUnaryOp(value, op) + + elif nodeType == "Call": + # fields: ('func', 'args', 'keywords') + if len(node.args) == 2: + lhs = self.evaluate(node.args[0], namesContext) + rhs = self.evaluate(node.args[1], namesContext) + # For the op, give empty context so we get a string + # from the name node. Then pass it into the factory + op = self.evaluate(node.func, {}) + return CallableParameter.createBinaryOp(lhs, rhs, op) + + elif len(node.args) == 1: + rhs = self.evaluate(node.args[0], namesContext) + # For the op, give empty context so we get a string + # from the name node. Then pass it into the factory + op = self.evaluate(node.func, {}) + return CallableParameter.createUnaryOp(rhs, op) + + assert 0, "Unknown function call with {0} parameters".format(len(node.args)) + + elif nodeType == "Assign": + # fields: ('targets', 'value') + + value = self.evaluate(node.value, namesContext) + valueToAssign = value.value if hasattr(value, "value") else value + + for target in node.targets: + targetType = type(target).__name__ + + # If we are assigning to a name, use + # current context as write object. + if targetType == "Name": + assignObj = namesContext + assignAttr = target.id + + # If we are assigning to an attribute, + # get the embedded context to write to + elif targetType == "Attribute": + assignObj = self.evaluate(target, namesContext) + assignAttr = target.attr + + else: assert 0, "Don't know how to handle target node type: {0}".format(targetType) + + # The assignment value might be an expression, + # so if it is callable, then evaluate it + assignObj[assignAttr] = valueToAssign + + return valueToAssign + + # Leaf node types + elif nodeType == "Name": + # fields: ('id') + if node.id in namesContext: + return namesContext[node.id] + else: + print("No context for named variable: {0}".format(node.id)) + return str(node.id) + + elif nodeType == "Attribute": + # fields: ('value', 'attr', 'ctx') + + # node.attr has the leaf attribute. We need to recurse and get + # the leaf attribute container so we can access the attribute. + # node.value has the next chained node, which is either another Attribute + # or Name node + newContext = self.evaluate(node.value, namesContext) + assert node.attr in newContext, "No attribute for named variable: {0}".format(node.attr) + + # For storing context, return the target value's container so we can + # setattr on it. See: Assign case. + # Otherwise return the actual stored value + return newContext if type(node.ctx).__name__ == "Store" else newContext[node.attr] + + elif nodeType == "IfExp": + # fields ('test', 'body', 'orelse') + theTest = self.evaluate(node.test, namesContext) + trueCond = self.evaluate(node.body, namesContext) + falseCond = self.evaluate(node.orelse, namesContext) + return trueCond if theTest() else falseCond + + elif nodeType == "Num": + # fields ('n') + return node.n + + elif nodeType == "Constant": + # fields ('kind', 'value') + return node.value + + elif nodeType == "Str": + # fields ('s') + return node.s + + assert 0, "Unknown node type: {0}".format(nodeType) + +# Everything stored as a Parameter object, which has +# value, default value and type attributes +class ProjectConfig(ReadWriteTransformDict): + """ + Adds support for hierarchical configuration designs. + Allows grouping of configurations together. + Also allows dynamic constraints. + + Each Config entry is a Parameter type object that + encapsulates attributes of Name, Type, Value, DefaultValue + and Description as seen above. + + Example: + + Program: + Network: + IP: X.X.X.X + PORT: XXXX + MaxPort: YYYY + UI: + WindowSize: 2000 + Antialiasing: True + + # Creation patterns: + Program = ProjectConfig() + networkConfig = Program.createSection("Network") + networkConfig.createValue("IP", X.X.X.X, "This is the IP we connect to") + networkConfig.createValue("PORT", XXXX, "Port we want to access") + networkConfig.createValue("PORT", YYYY, "Maximum port value") + + uiConfig = Program.createSection("UI") + uiConfig.createValue("WindowSize", 2000) + uiConfig.createValue("Antialiasing", True) + + # Access patterns: + IP = Program.Network.IP + PORT = Program["Network"]["IP"] + networkConfig = Program["Network"] + + # Constraints assertions: e.g. limit port numbers or antialiasing. + Program.addConstraint("Program.Network.PORT >= MaxPort") + Program.addConstraint("False if Program.UI.WindowSize > 2000 and Program.UI.Antialiasing else True") + + # Overrides: e.g. enforce port numbers + Program.addConstraint("Program.Network.PORT = Program.Network.MaxPort if Program.Network.PORT > Program.Network.MaxPort") + + """ + + __slots__ = () + + ContainerType = Parameter + SubConfigType : 'ProjectConfig' + ConstraintsKey = "_Constraints" + + def __init__(self): + + # Sub-configs will be stored inside ContainerType, + # so when reading them as an attribute, forward the config + # stored in the container's value + def readXForm(obj, key): + + attribute = obj.readNoTransform(key) + + # Attribute is a Container, which has a "value" + # and should be forwarded to get around awkward + # value forwarding. We are mainly interested + # in accessing the value attribute, so we can + # now access it with config["Attrib"] instead + # of config["Attrib"]["value"] + if isinstance(attribute, self.ContainerType): + return attribute.readNoTransform("value") + else: + return attribute + + def writeXForm(obj, key, value): + if key in super(obj.__class__, obj).keys() and isinstance(obj.readNoTransform(key), self.ContainerType): + obj.readNoTransform(key).writeNoTransform("value", value) + else: + obj.writeNoTransform(key, value) + + super().__init__(readXForm, writeXForm) + + def __contains__(self, key): + return True if super().__contains__(key) else self.toFlattenedDict().__contains__(key) + + def __getContainer(self, key): + # In case we are accessing with keys such as [a.b.c] + # Then we need to recurse to get the embedded container. + # Assumes that a and b for example are embedded configs + if "." in key: + try: + levels = key.split(".") + currentValue = self + for level in levels[:-1]: + currentValue = currentValue[level] + return currentValue.readNoTransform(levels[-1]) + except: + pass + + # Fallback, in case there is a "." in the key + return self.readNoTransform(key) + + def __getitem__(self, key): + + if "." in key: + try: + levels = key.split(".") + currentValue = self + for level in levels: + currentValue = currentValue[level] + return currentValue + except: + pass + + # Fallback is to check if the full key lives + # in the top level, in case it just happens + # to have a '.' in it. + return super().__getitem__(key) + + def __setitem__(self, key, value): + # In case we are accessing with keys such as [a.b.c] + # Then we need to recurse to get the embedded value. + # Assumes that a and b for example are embedded configs + if "." in key: + try: + levels = key.split(".") + currentValue = self + for level in levels[:-1]: + currentValue = currentValue[level] + setattr(currentValue, levels[-1], value) + return + except: + pass + + # Fallback is to check if the full key lives + # in the top level, in case it just happens + # to have a '.' in it. + super().__setitem__(key, value) + + # Simplification of creating attributes + def createValue(self, name, value, defaultValue = None, description=""): + self[name] = self.ContainerType(name, value, defaultValue, description) + return self[name] + + def createSection(self, name): + return self.createValue(name, ProjectConfig()) + + def resetToDefaults(self): + for key in self.keys(): + attribute = self.readNoTransform(key) + if isinstance(attribute, ProjectConfig): + attribute.resetToDefaults() + elif isinstance(attribute, self.ContainerType): + attribute.writeNoTransform("value", attribute.readNoTransform("defaultValue")) + + def addConstraint(self, expressionStr): + if self.ConstraintsKey not in self: + self.createSection(self.ConstraintsKey) + + self[self.ConstraintsKey].createValue(expressionStr, ast.parse(expressionStr, mode='exec')) + + def checkConstraints(self): + result = True + if self.ConstraintsKey in self: + constraints = self.readNoTransform(self.ConstraintsKey).value + for expression, tree in [(x, y) for (x, y) in constraints.items() if "_" not in x]: + result = ExpressionEvaluator().evaluate(tree.value, self) + value = result.value if isinstance(result, Parameter) else result + assert value, "Constraint evaluation failed: {0}".format(expression) + return value + + def getDefaultValue(self, name): + return self.__getContainer(name).getDefault() + + def getDescription(self, name): + return self.__getContainer(name).getDescription() diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py new file mode 100644 index 0000000000..a95c7b007b --- /dev/null +++ b/tensilelite/Tensile/Contractions.py @@ -0,0 +1,517 @@ +################################################################################# +# Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Activation import ActivationType +from .Common import printExit +from .TensileInstructions import DataType +from . import Hardware +from . import Properties +from .SolutionStructs import Solution as OriginalSolution +from .Utils import state, state_key_ordering + +from . import Common +from . Common import globalParameters + +@state_key_ordering +class FreeIndex: + StateKeys = ['isA', 'i', 'c', 'd'] + + def __init__(self, isA, i=None, c=None, d=None): + self.isA = isA + self.i = i # index into A or B (depending on isA) + self.c = c + self.d = d + +@state_key_ordering +class BatchIndex: + StateKeys = ['a', 'b', 'c', 'd'] + def __init__(self, a=None, b=None, c=None, d=None): + self.a = a + self.b = b + self.c = c + self.d = d + +@state_key_ordering +class BoundIndex: + StateKeys = ['a', 'b', 'aMirror', 'bMirror'] + def __init__(self, a=None, b=None, aMirror=False, bMirror=False): + self.a = a + self.b = b + self.aMirror = aMirror + self.bMirror = bMirror + + +class ProblemType: + StateKeys = ['operationIdentifier', 'aType', 'bType', 'cType', 'dType', + 'useBeta', 'useBias', 'highPrecisionAccumulate', + 'useInitialStridesAB', 'useInitialStridesCD', 'stridedBatched', + 'activationType', 'activationHPA'] + @classmethod + def FromOriginalState(cls, d): + indices = [None]*d['TotalIndices'] + freeIndices = [] + batchIndices = [] + boundIndices = [] + + for i in d['IndicesSummation']: + bi = BoundIndex(aMirror=('MirrorDimsA' in d and i in d['MirrorDimsA']), + bMirror=('MirrorDimsB' in d and i in d['MirrorDimsB'])) + indices[i] = bi + boundIndices.append(bi) + + for i in range(0,d["NumIndicesC"]): + if i in d['IndicesBatch']: + bi = BatchIndex(c=i, d=i) + indices[i] = bi + batchIndices.append(bi) + else: + assert i in d['IndicesFree'] + if i in d['IndexAssignmentsA']: + fi = FreeIndex(isA=True, i=d["IndexAssignmentsA"].index(i), c=i, d=i) + elif i in d['IndexAssignmentsB']: + fi = FreeIndex(isA=False, i=d["IndexAssignmentsB"].index(i), c=i, d=i) + else: + raise RuntimeError("free index %u not in ia or ib"%i) + indices[i] = fi + freeIndices.append(fi) + + for ia, ic in enumerate(d['IndexAssignmentsA']): + indices[ic].a = ia + + for ib, ic in enumerate(d['IndexAssignmentsB']): + indices[ic].b = ib + + for idx in indices: + assert idx is not None + idxState = state(idx) + for (key, value) in list(idxState.items()): + assert value is not None + + rv = cls() + rv.indices = indices + rv.freeIndices = freeIndices + rv.batchIndices = batchIndices + rv.boundIndices = boundIndices + rv.aDims = len(d['IndexAssignmentsA']) + rv.bDims = len(d['IndexAssignmentsB']) + rv.cDims = d['NumIndicesC'] + rv.dDims = rv.cDims + + rv.aConjugate = d['ComplexConjugateA'] + rv.bConjugate = d['ComplexConjugateB'] + + srcType = DataType(d['DataType']) + dstType = DataType(d['DestDataType']) if 'DestDataType' in d else srcType + computeType = DataType(d['ComputeDataType']) if 'ComputeDataType' in d else dstType + + rv.aType = srcType + rv.bType = srcType + rv.cType = dstType + rv.dType = dstType + # we already checked the src/dst/compute types are supported and well-assigned in SolutionStruct + rv.alphaType = computeType + rv.betaType = computeType + + rv.highPrecisionAccumulate = False + if 'HighPrecisionAccumulate' in d: + rv.highPrecisionAccumulate = d['HighPrecisionAccumulate'] + + rv.useInitialStridesAB = False + if 'UseInitialStridesAB' in d: + rv.useInitialStridesAB = d['UseInitialStridesAB'] + rv.useInitialStridesCD = False + if 'UseInitialStridesCD' in d: + rv.useInitialStridesCD = d['UseInitialStridesCD'] + + rv.stridedBatched = True + if 'StridedBatched' in d: + rv.stridedBatched = d['StridedBatched'] + + rv.setConstStrideA = [] + if 'SetConstStrideA' in d: + rv.setConstStrideA = d['SetConstStrideA'] + rv.setConstStrideB = [] + if 'SetConstStrideB' in d: + rv.setConstStrideB = d['SetConstStrideB'] + + rv.mirrorDimsA = d['MirrorDimsA'] if 'MirrorDimsA' in d else [] + rv.mirrorDimsB = d['MirrorDimsB'] if 'MirrorDimsB' in d else [] + + rv.useBeta = True + if 'UseBeta' in d: + rv.useBeta = d['UseBeta'] + + rv.useBias = False + if 'UseBias' in d: + rv.useBias = d['UseBias'] + + rv.batched = d['Batched'] + + rv.activationType = ActivationType('none') + if 'ActivationType' in d: + rv.activationType = d['ActivationType'] + rv.activationHPA = False + if 'ActivationHPA' in d: + rv.activationHPA = d['ActivationHPA'] + if 'ActivationComputeDataType' in d: + rv.activationComputeDataType = DataType(d['ActivationComputeDataType']) + else: + rv.activationComputeDataType = DataType(d['ComputeDataType'] if rv.activationHPA else \ + d['DestDataType']) + + return rv + + def __init__(self, freeIndices=None, batchIndices=None, boundIndices=None, aDims=None, bDims=None, cDims=None, dDims=None): + self.freeIndices = freeIndices + self.batchIndices = batchIndices + self.boundIndices = boundIndices + self.aDims = aDims + self.bDims = bDims + self.cDims = cDims + self.dDims = dDims + + @property + def indexNames(self): + aNames = ['_'] * self.aDims + bNames = ['_'] * self.bDims + cNames = ['_'] * self.cDims + + allIndexNames = 'ijklmnopqrstuvwxyz' + index = 0 + + dNames = list([allIndexNames[index+i] for i in range(self.cDims)]) + index += len(dNames) + + sumNames = list([allIndexNames[index+i] for i in range(len(self.boundIndices))]) + index += len(sumNames) + + for free in self.freeIndices: + if free.isA: + aNames[free.i ] = dNames[free.d] + else: + bNames[free.i ] = dNames[free.d] + cNames[free.c] = dNames[free.d] + + for batch in self.batchIndices: + name = dNames[batch.d] + aNames[batch.a] = name + bNames[batch.b] = name + cNames[batch.c] = name + + for i, bound in enumerate(self.boundIndices): + name = sumNames[i] + aNames[bound.a] = name.upper() if bound.aMirror else name + bNames[bound.b] = name.upper() if bound.bMirror else name + + aNames = ''.join(aNames) + bNames = ''.join(bNames) + cNames = ''.join(cNames) + dNames = ''.join(dNames) + sumNames = ''.join(sumNames) + + return (aNames, bNames, cNames, dNames, sumNames) + + @property + def operationIdentifier(self): + (aNames, bNames, cNames, dNames, sumNames) = self.indexNames + + aOp = 'C' if self.aConjugate else '' + bOp = 'C' if self.bConjugate else '' + + return '_'.join(['Contraction', sumNames, + 'A' + aNames + aOp, + 'B' + bNames + bOp, + 'C' + cNames, + 'D' + dNames]) + + def predicates(self, includeBatch=False, includeOperation=False, includeType=False): + predicates = [] + + #if includeBatch and not self.batched: + # predicates.append(ProblemPredicate("BatchSizeEqual", index=0, value=1)) + + if includeOperation: + predicates.append(ProblemPredicate("OperationIdentifierEqual", value=self.operationIdentifier)) + if not self.useBeta: + predicates.append(ProblemPredicate("BetaZero")) + predicates.append(ProblemPredicate("UseBias", value=self.useBias)) + predicates.append(ProblemPredicate("Activation", value=self.activationType)) + if self.activationType == 'all': + enumList = ActivationType.getEnumStrList(self.activationComputeDataType) + wlStr = "" + for actType in enumList: + wlStr += ActivationType(actType).toEnum() + "," + wlStr = wlStr.rstrip(',') + predicates.append(ProblemPredicate("ActivationEnumWhiteList", value=wlStr)) + predicates.append(ProblemPredicate("StridedBatched", value=self.stridedBatched)) + + if includeType: + predicates.append(ProblemPredicate("TypesEqual", value=(self.aType, self.bType, self.cType, self.dType))) + predicates.append(ProblemPredicate("HighPrecisionAccumulate", value=self.highPrecisionAccumulate)) + predicates.append(ProblemPredicate("ActivationHPA", value=self.activationHPA)) + + return predicates + +def extractDimPredicate(cls, key, value, predicateName): + """ + Extract the predicate for AssertStrideEqual* + Value is a dictionary + """ + predicates = [] + for pos,val in value.items(): + if val != -1: + predicates.append(cls(predicateName, index=pos, value=val)) + if len(predicates) == 1: + return predicates[0] + elif len(predicates) > 1: + return cls.And(predicates) + +class ProblemPredicate(Properties.Predicate): + @classmethod + def FromOriginalKeyPair(cls, pair): + (key, value) = pair + if key.endswith('Multiple'): + if value == 1: + return None + + if key == "AssertFree0ElementMultiple": + tag = "Free0SizeMultiple" + index = 0 + elif key == "AssertFree1ElementMultiple": + tag = "Free1SizeMultiple" + index = 0 + elif key == "AssertSummationElementMultiple": + tag = "BoundSizeMultiple" + index = -1 + else: + raise RuntimeError("Unknown Multiple Value: {}".format(key)) + + return cls(tag, index=index, value=value) + + if key == "_WorkspaceSizePerElemC" and value > 0: + return cls("WorkspaceCheck", index=0, value=value) + + if key.startswith('Assert'): + raise RuntimeError("Unknown assertion key: {}".format(key)) + + if key == "Fp16AltImpl": + return cls("Fp16AltImpl") if value != False else None + + @classmethod + def CompoundPredicates(cls, state, problemType): + rv = [] + + if not problemType.aType.isInt8x4(): + # calculate the minimum supported free dimension size + TLUA = state['ProblemType']['TLUA'] + TLUB = state['ProblemType']['TLUB'] + minFree0 = state['GlobalLoadVectorWidthA'] if TLUA else 1 + minFree1 = state['GlobalLoadVectorWidthB'] if TLUB else 1 + rv += [cls('LeadingFree0SizesGreaterOrEqual', value=minFree0)] + rv += [cls('LeadingFree1SizesGreaterOrEqual', value=minFree1)] + + if len(state["PackedC0IndicesX"]) > 1: + rv += [cls("CDStridesEqual")] + + if "KernelLanguage" in state: + rv += [cls("KernelLanguageCompatible", value=state["KernelLanguage"])] + + if ('GlobalSplitU' in state) and (state['GlobalSplitU'] > 1): + if ('_GlobalAccumulation' not in state) or (state['_GlobalAccumulation'] != 'MultipleBuffer'): + rv += [cls("DeterministicMode", value = False)] + + if ("MatrixInstruction" in state and state["MatrixInstruction"]) or \ + ("EnableMatrixInstruction" in state and state["EnableMatrixInstruction"] is True): + rv += [cls("ArithmeticUnitCompatible", value="MFMA")] + else: + printExit("TensileLite does not support VALU.") + + # if bufferload is performed, we output some predication info for host side, + # to prevent from some extremely large problems from launching and causing bufferload offset limit < 2^32 + # thoses cases will not satisfy the assertion thus won't use the kernel. + # See Common.py for more details, we will need four values: + # TODO - haven't been fully tested for FP16 and BF, need to verify the false-positive + if 'BufferLoad' in state and state['BufferLoad'] == True: + TLUA = state['ProblemType']['TLUA'] + TLUB = state['ProblemType']['TLUB'] + MayShiftA = TLUA and state['AssertFree0ElementMultiple'] < state['GlobalLoadVectorWidthA'] + MayShiftB = TLUB and state['AssertFree1ElementMultiple'] < state['GlobalLoadVectorWidthB'] + subrv={} + subrv['ShiftPtrElemB'] = state['GlobalLoadVectorWidthB'] if MayShiftB else 0 + subrv['ShiftPtrElemA'] = state['GlobalLoadVectorWidthA'] if MayShiftA else 0 + subrv['DUorMT1'] = state['DepthU'] if TLUB else state['MacroTile1'] + subrv['DUorMT0'] = state['DepthU'] if TLUA else state['MacroTile0'] + # value is also a dict for better readibility, client side need to handel the serialization + rv += [cls('BufferLoadOffsetLimitCheck', value=subrv)] + + # When doing globol write, may need to load matrix C if beta !=0 + if 'BufferLoad' in state and state['BufferLoad'] == True: + rv += [cls('BufferLoadOffsetLimitCheck_Beta', value=state['MacroTile1'])] + + # similiar check is applied for bufferstore, + # for bufferstore offset, test if the bot-right offset < 2^32, + # it should be StrideA*MT1, so we need to output MT1 and use the StrideA of problem in host-side for predication + if 'BufferStore' in state and state['BufferStore'] == True: + rv += [cls('BufferStoreOffsetLimitCheck', value=state['MacroTile1'])] + + if '_GlobalAccumulation' in state and state['_GlobalAccumulation'] != None: + value = globalParameters['MinKForGSU'] * state['GlobalSplitU'] + rv += [cls('GlobalSplitUCheckMinK', value=value)] + + return rv + + @classmethod + def FromOriginalState(cls, d, problemType, morePreds=[]): + problemTypePreds = problemType.predicates(True, True, True) + compoundPreds = cls.CompoundPredicates(d, problemType) + extraPreds = problemTypePreds + compoundPreds + morePreds + + return super().FromOriginalState(d, extraPreds) + +class SizeMapping: + StateKeys = ['workGroup', + 'macroTile', + 'threadTile', + 'depthU', + 'staggerU', + 'globalSplitU', + 'staggerStrideShift', + 'workGroupMapping', + 'packBatchDims', + 'magicDivAlg', + 'sourceKernel', + 'globalAccumulation', + 'workspaceSizePerElemC', + 'activationFused' + ] + + @classmethod + def FromOriginalState(cls, d): + globalAccum = 0 + if d['_GlobalAccumulation'] == 'SingleBuffer': + globalAccum = 1 + if d['_GlobalAccumulation'] == 'MultipleBuffer': + globalAccum = 2 + return cls(workGroup = d['WorkGroup'], + macroTile = cls.ReadOriginalMacroTile(d), + threadTile = d['ThreadTile'], + workGroupMapping = d['WorkGroupMapping'], + staggerU = d['StaggerU'] if 'StaggerU' in d else 0, + depthU = d['DepthU'], + globalSplitU = d['GlobalSplitU'], + staggerStrideShift = d['_staggerStrideShift'] if '_staggerStrideShift' in d else 0, + packBatchDims = 0, + magicDivAlg = d.get('MagicDivAlg', 1), + sourceKernel = d['KernelLanguage'] == 'Source', + globalAccumulation = globalAccum, + workspaceSizePerElemC = d['_WorkspaceSizePerElemC'], + activationFused = d['ActivationFused'] + ) + + @classmethod + def ReadOriginalMacroTile(cls, d): + rv = [1,1,1] + rv[0] = d['MacroTile0'] + rv[1] = d['MacroTile1'] + return rv + + def __init__(self, **kwargs): + for (key, value) in list(kwargs.items()): + setattr(self, key, value) + +class Solution: + StateKeys = ['name', + 'problemType', + 'hardwarePredicate', + 'problemPredicate', + 'sizeMapping', + 'debugKernel', + 'info', + 'index', + 'ideals', + 'linearModel'] + HiddenKeys = ['originalSolution'] + + @classmethod + def FromSolutionStruct(cls, solution, deviceInfo=None): + return cls.FromOriginalState(solution._state) + + @classmethod + def FromOriginalState(cls, d, deviceInfo=None): + rv = cls() + + + if 'SolutionNameMin' in d: + rv.name = d['SolutionNameMin'] + + rv.problemType = ProblemType.FromOriginalState(d['ProblemType']) + + rv.problemPredicate = ProblemPredicate.FromOriginalState(d, rv.problemType) + + if 'DebugKernel' in d: + rv.debugKernel = d['DebugKernel'] + + if 'SolutionIndex' in d: + rv.index = d['SolutionIndex'] + + rv.info = cls.ReadOriginalInfo(d) + + rv.sizeMapping = SizeMapping.FromOriginalState(d) + if 'Ideals' in d: + rv.ideals = d['Ideals'] + else: + rv.ideals = {} + + if 'LinearModel' in d: + rv.linearModel = d['LinearModel'] + else: + rv.linearModel = {} + + if 'ISA' not in d: + if d['KernelLanguage'] == 'Assembly': + d['ISA'] = Common.gfxArch(deviceInfo[1]) + else: + d['ISA'] = [0,0,0] + + rv.originalSolution = OriginalSolution(d) + + return rv + + @classmethod + def ReadOriginalInfo(cls, d): + return dict([(key, str(value)) for (key, value) in list(d.items()) if key != 'ProblemType']) + + def __init__(self, **kwargs): + self.name = None + self.problemType = None + self.hardwarePredicate = Hardware.HardwarePredicate('TruePred') + self.problemPredicate = ProblemPredicate('TruePred') + self.sizeMapping = None + self.debugKernel = False + self.info = {} + self.index = None + self.ideals = {} + + for key, value in kwargs: + if key not in Solution.StateKeys and key not in Solution.HiddenKeys: + raise KeyError("{0} is not a property of Solution.".format(key)) + + setattr(self, key, value) diff --git a/tensilelite/Tensile/CustomKernels.py b/tensilelite/Tensile/CustomKernels.py new file mode 100644 index 0000000000..6ebbf30f6d --- /dev/null +++ b/tensilelite/Tensile/CustomKernels.py @@ -0,0 +1,62 @@ +################################################################################ +# Copyright 2020-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Common import globalParameters + +import yaml + +import os + +def isCustomKernelConfig(config): + return "CustomKernelName" in config and config["CustomKernelName"] + +def getCustomKernelFilepath(name, directory=globalParameters["CustomKernelDirectory"]): + return os.path.join(directory, (name + ".s")) + +def getAllCustomKernelNames(directory=globalParameters["CustomKernelDirectory"]): + return [fname[:-2] for fname in os.listdir(directory) if fname.endswith(".s")] + +def getCustomKernelContents(name, directory=globalParameters["CustomKernelDirectory"]): + try: + with open(getCustomKernelFilepath(name, directory)) as f: + return f.read() + except: + raise RuntimeError("Failed to find custom kernel: {}".format(os.path.join(directory, name))) + +def getCustomKernelConfigAndAssembly(name, directory=globalParameters["CustomKernelDirectory"]): + contents = getCustomKernelContents(name, directory) + config = "\n" #Yaml configuration properties + assembly = "" + inConfig = False + for line in contents.splitlines(): + if line == "---": inConfig = True #Beginning of yaml section + elif line == "...": inConfig = False #End of yaml section + elif inConfig: config += line + "\n" + else : assembly += line + "\n"; config += "\n" #Second statement to keep line numbers consistent for yaml errors + + return (config, assembly) + +def getCustomKernelConfig(name, directory=globalParameters["CustomKernelDirectory"]): + rawConfig, _ = getCustomKernelConfigAndAssembly(name, directory) + try: + return yaml.safe_load(rawConfig)["custom.config"] + except yaml.scanner.ScannerError as e: + raise RuntimeError("Failed to read configuration for custom kernel: {0}\nDetails:\n{1}".format(name, e)) diff --git a/tensilelite/Tensile/CustomKernels/DGEMM_Aldebaran_NN_MT128x128x16_MI16x16x4x1_GRVW2_SU4_SUS128_WGM4.s b/tensilelite/Tensile/CustomKernels/DGEMM_Aldebaran_NN_MT128x128x16_MI16x16x4x1_GRVW2_SU4_SUS128_WGM4.s new file mode 100644 index 0000000000..e00a9ff3d5 --- /dev/null +++ b/tensilelite/Tensile/CustomKernels/DGEMM_Aldebaran_NN_MT128x128x16_MI16x16x4x1_GRVW2_SU4_SUS128_WGM4.s @@ -0,0 +1,10841 @@ + + +/******************************************/ +/* Function Prefix */ +/******************************************/ + + + +/******************************************/ +/* Begin Kernel */ +/******************************************/ + +// Component.Signature.SignatureCOV3 +.amdgcn_target "amdgcn-amd-amdhsa--gfx90a" +.text +.protected DGEMM_Aldebaran_NN_MT128x128x16_MI16x16x4x1_GRVW2_SU4_SUS128_WGM4 +.globl DGEMM_Aldebaran_NN_MT128x128x16_MI16x16x4x1_GRVW2_SU4_SUS128_WGM4 +.p2align 8 +.type DGEMM_Aldebaran_NN_MT128x128x16_MI16x16x4x1_GRVW2_SU4_SUS128_WGM4,@function +.section .rodata,#alloc +.p2align 6 +.amdhsa_kernel DGEMM_Aldebaran_NN_MT128x128x16_MI16x16x4x1_GRVW2_SU4_SUS128_WGM4 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_accum_offset 256 // accvgpr offset + .amdhsa_next_free_vgpr 256 // vgprs + .amdhsa_next_free_sgpr 73 // sgprs + .amdhsa_group_segment_fixed_size 32768 // lds bytes + .amdhsa_private_segment_fixed_size 0 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_system_sgpr_workgroup_id_y 1 + .amdhsa_system_sgpr_workgroup_id_z 1 + .amdhsa_system_vgpr_workitem_id 0 +.end_amdhsa_kernel +.text + +/******************************************/ +/* Optimizations and Config: */ +/******************************************/ +/* ThreadTile= 8 x 8 */ +/* SubGroup= 16 x 16 */ +/* VectorWidth=2 */ +/* GlobalLoadVectorWidthA=2, GlobalLoadVectorWidthB=2 */ +/* DirectToLdsA=False */ +/* DirectToLdsB=False */ +/* UseSgprForGRO=False */ +.amdgpu_metadata +--- +custom.config: + ProblemType: + OperationType: GEMM + DataType: D + TransposeA: False + TransposeB: False + UseBeta: True + Batched: True + MatrixInstruction: [ 16, 16, 4, 1 ] + ThreadTile: [ 2, 128 ] + WorkGroup: [ 64, 4, 1 ] + DepthU: 16 + VectorWidth: 2 + SourceSwap: 1 + GlobalReadVectorWidth: 2 + StaggerUStride: 128 + StaggerU: 4 + WorkGroupMapping: 4 + AssertSizeMultiple: {3: 32} +amdhsa.version: + - 1 + - 0 +amdhsa.kernels: + - .name: DGEMM_Aldebaran_NN_MT128x128x16_MI16x16x4x1_GRVW2_SU4_SUS128_WGM4 + .symbol: 'DGEMM_Aldebaran_NN_MT128x128x16_MI16x16x4x1_GRVW2_SU4_SUS128_WGM4.kd' + .language: OpenCL C + .language_version: + - 2 + - 0 + .args: + - .name: sizeC + .size: 8 + .offset: 0 + .value_kind: by_value + .value_type: u64 + - .name: sizeA + .size: 8 + .offset: 8 + .value_kind: by_value + .value_type: u64 + - .name: sizeB + .size: 8 + .offset: 16 + .value_kind: by_value + .value_type: u64 + - .name: D + .size: 8 + .offset: 24 + .value_kind: global_buffer + .value_type: f64 + .address_space: generic + - .name: C + .size: 8 + .offset: 32 + .value_kind: global_buffer + .value_type: f64 + .address_space: generic + - .name: A + .size: 8 + .offset: 40 + .value_kind: global_buffer + .value_type: f64 + .address_space: generic + - .name: B + .size: 8 + .offset: 48 + .value_kind: global_buffer + .value_type: f64 + .address_space: generic + - .name: alpha + .size: 8 + .offset: 56 + .value_kind: by_value + .value_type: f64 + - .name: beta + .size: 8 + .offset: 64 + .value_kind: by_value + .value_type: f64 + - .name: strideD0 + .size: 4 + .offset: 72 + .value_kind: by_value + .value_type: u32 + - .name: strideD1 + .size: 4 + .offset: 76 + .value_kind: by_value + .value_type: u32 + - .name: strideC0 + .size: 4 + .offset: 80 + .value_kind: by_value + .value_type: u32 + - .name: strideC1 + .size: 4 + .offset: 84 + .value_kind: by_value + .value_type: u32 + - .name: strideA0 + .size: 4 + .offset: 88 + .value_kind: by_value + .value_type: u32 + - .name: strideA1 + .size: 4 + .offset: 92 + .value_kind: by_value + .value_type: u32 + - .name: strideB0 + .size: 4 + .offset: 96 + .value_kind: by_value + .value_type: u32 + - .name: strideB1 + .size: 4 + .offset: 100 + .value_kind: by_value + .value_type: u32 + - .name: SizesFree0 + .size: 4 + .offset: 104 + .value_kind: by_value + .value_type: u32 + - .name: SizesFree1 + .size: 4 + .offset: 108 + .value_kind: by_value + .value_type: u32 + - .name: SizesFree2 + .size: 4 + .offset: 112 + .value_kind: by_value + .value_type: u32 + - .name: SizesSum0 + .size: 4 + .offset: 116 + .value_kind: by_value + .value_type: u32 + - .name: OrigStaggerUIter + .size: 4 + .offset: 120 + .value_kind: by_value + .value_type: i32 + - .name: NumWorkGroups0 + .size: 4 + .offset: 124 + .value_kind: by_value + .value_type: u32 + - .name: NumWorkGroups1 + .size: 4 + .offset: 128 + .value_kind: by_value + .value_type: u32 + - .name: NumFullBlocks + .size: 4 + .offset: 132 + .value_kind: by_value + .value_type: u32 + - .name: WgmRemainder1 + .size: 4 + .offset: 136 + .value_kind: by_value + .value_type: u32 + - .name: MagicNumberWgmRemainder1 + .size: 4 + .offset: 140 + .value_kind: by_value + .value_type: u32 + - .name: OffsetD + .size: 4 + .offset: 144 + .value_kind: by_value + .value_type: u32 + - .name: OffsetC + .size: 4 + .offset: 148 + .value_kind: by_value + .value_type: u32 + - .name: OffsetA + .size: 4 + .offset: 152 + .value_kind: by_value + .value_type: u32 + - .name: OffsetB + .size: 4 + .offset: 156 + .value_kind: by_value + .value_type: u32 + - .name: padding + .size: 4 + .offset: 160 + .value_kind: by_value + .value_type: u32 + .group_segment_fixed_size: 32768 + .kernarg_segment_align: 8 + .kernarg_segment_size: 168 + .max_flat_workgroup_size: 256 + .private_segment_fixed_size: 0 + .sgpr_count: 73 + .sgpr_spill_count: 0 + .vgpr_count: 256 + .vgpr_spill_count: 0 + .wavefront_size: 64 +... +.end_amdgpu_metadata +DGEMM_Aldebaran_NN_MT128x128x16_MI16x16x4x1_GRVW2_SU4_SUS128_WGM4: + +/******************************************/ +/* Asm syntax workarounds */ +/******************************************/ +.macro _v_add_co_u32 dst:req, cc:req, src0:req, src1:req, dpp= + v_add_co_u32 \dst, \cc, \src0, \src1 \dpp +.endm + +.macro _v_add_u32 dst:req, src0:req, src1:req, dpp= + v_add_u32 \dst, \src0, \src1 \dpp +.endm + +.macro _v_add_i32 dst:req, src0:req, src1:req, dpp= + v_add_i32 \dst, \src0, \src1 \dpp +.endm + +.macro _v_addc_co_u32 dst:req, ccOut:req, src0:req, ccIn:req, src1:req, dpp= + v_addc_co_u32 \dst, \ccOut, \src0, \ccIn, \src1 \dpp +.endm + +.macro _v_sub_co_u32 dst:req, cc:req, src0:req, src1:req, dpp= + v_sub_co_u32 \dst, \cc, \src0, \src1 \dpp +.endm + +.macro _v_sub_u32 dst:req, src0:req, src1:req, dpp= + v_sub_u32 \dst, \src0, \src1 \dpp +.endm + +.macro _v_sub_i32 dst:req, src0:req, src1:req, dpp= + v_sub_i32 \dst, \src0, \src1 \dpp +.endm + +.macro _v_add_lshl_u32 dst:req, src0:req, src1:req, shiftCnt:req + v_add_lshl_u32 \dst, \src0, \src1, \shiftCnt +.endm + +.macro _v_lshl_add_u32 dst:req, src0:req, src1:req, shiftCnt:req + v_lshl_add_u32 \dst, \src0, \src1, \shiftCnt +.endm + +.macro _v_lshl_or_b32 dst:req, src0:req, shiftCnt:req, src1:req + v_lshl_or_b32 \dst, \src0, \shiftCnt, \src1 +.endm + +.macro _v_cmpx_lt_i16 dst, src0, src1= + v_cmpx_lt_i16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_lt_i32 dst, src0, src1= + v_cmpx_lt_i32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_lt_i64 dst, src0, src1= + v_cmpx_lt_i64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_lt_u16 dst, src0, src1= + v_cmpx_lt_u16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_lt_u32 dst, src0, src1= + v_cmpx_lt_u32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_lt_u64 dst, src0, src1= + v_cmpx_lt_u64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_eq_i16 dst, src0, src1= + v_cmpx_eq_i16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_eq_i32 dst, src0, src1= + v_cmpx_eq_i32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_eq_i64 dst, src0, src1= + v_cmpx_eq_i64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_eq_u16 dst, src0, src1= + v_cmpx_eq_u16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_eq_u32 dst, src0, src1= + v_cmpx_eq_u32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_eq_u64 dst, src0, src1= + v_cmpx_eq_u64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_le_i16 dst, src0, src1= + v_cmpx_le_i16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_le_i32 dst, src0, src1= + v_cmpx_le_i32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_le_i64 dst, src0, src1= + v_cmpx_le_i64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_le_u16 dst, src0, src1= + v_cmpx_le_u16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_le_u32 dst, src0, src1= + v_cmpx_le_u32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_le_u64 dst, src0, src1= + v_cmpx_le_u64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_gt_i16 dst, src0, src1= + v_cmpx_gt_i16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_gt_i32 dst, src0, src1= + v_cmpx_gt_i32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_gt_i64 dst, src0, src1= + v_cmpx_gt_i64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_gt_u16 dst, src0, src1= + v_cmpx_gt_u16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_gt_u32 dst, src0, src1= + v_cmpx_gt_u32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_gt_u64 dst, src0, src1= + v_cmpx_gt_u64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ne_i16 dst, src0, src1= + v_cmpx_ne_i16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ne_i32 dst, src0, src1= + v_cmpx_ne_i32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ne_i64 dst, src0, src1= + v_cmpx_ne_i64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ne_u16 dst, src0, src1= + v_cmpx_ne_u16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ne_u32 dst, src0, src1= + v_cmpx_ne_u32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ne_u64 dst, src0, src1= + v_cmpx_ne_u64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_lg_i16 dst, src0, src1= + v_cmpx_lg_i16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_lg_i32 dst, src0, src1= + v_cmpx_lg_i32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_lg_i64 dst, src0, src1= + v_cmpx_lg_i64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_lg_u16 dst, src0, src1= + v_cmpx_lg_u16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_lg_u32 dst, src0, src1= + v_cmpx_lg_u32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_lg_u64 dst, src0, src1= + v_cmpx_lg_u64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ge_i16 dst, src0, src1= + v_cmpx_ge_i16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ge_i32 dst, src0, src1= + v_cmpx_ge_i32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ge_i64 dst, src0, src1= + v_cmpx_ge_i64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ge_u16 dst, src0, src1= + v_cmpx_ge_u16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ge_u32 dst, src0, src1= + v_cmpx_ge_u32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_ge_u64 dst, src0, src1= + v_cmpx_ge_u64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_o_i16 dst, src0, src1= + v_cmpx_o_i16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_o_i32 dst, src0, src1= + v_cmpx_o_i32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_o_i64 dst, src0, src1= + v_cmpx_o_i64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_o_u16 dst, src0, src1= + v_cmpx_o_u16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_o_u32 dst, src0, src1= + v_cmpx_o_u32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_o_u64 dst, src0, src1= + v_cmpx_o_u64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_u_i16 dst, src0, src1= + v_cmpx_u_i16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_u_i32 dst, src0, src1= + v_cmpx_u_i32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_u_i64 dst, src0, src1= + v_cmpx_u_i64 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_u_u16 dst, src0, src1= + v_cmpx_u_u16 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_u_u32 dst, src0, src1= + v_cmpx_u_u32 \dst, \src0, \src1 +.endm + +.macro _v_cmpx_u_u64 dst, src0, src1= + v_cmpx_u_u64 \dst, \src0, \src1 +.endm +.macro _v_mac_f32 c:req, a:req, b:req + v_mac_f32 \c, \a, \b +.endmacro + +/******************************************/ +/* Magic div and mod functions */ +/******************************************/ +.macro V_MAGIC_DIV dstIdx:req, dividend:req, magicNumber:req, magicShift:req, magicA:req + v_mul_hi_u32 v[\dstIdx+1], \dividend, \magicNumber + v_mul_lo_u32 v[\dstIdx+0], \dividend, \magicA + _v_add_u32 v[\dstIdx+0], v[\dstIdx+0], v[\dstIdx+1] + v_lshrrev_b32 v[\dstIdx+0], \magicShift, v[\dstIdx+0] +.endm + +/******************************************/ +/* VGPR Assignments */ +/******************************************/ +/* ValuC range: [0-128), serializedStore enabled */ +.set vgprValuC, 0 +/* ValuA/B Xn=PLR buffer idx */ +.set vgprValuA_X0_I0, 128 +.set vgprValuA_X1_I0, 132 +.set vgprValuA_X2_I0, 136 +.set vgprValuA_X3_I0, 140 +.set vgprValuB_X0_I0, 144 +.set vgprValuB_X1_I0, 160 +.set vgprValuB_X2_I0, 176 +.set vgprValuB_X3_I0, 192 +.set vgprLocalWriteAddrA, 208 +.set vgprLocalWriteAddrB, 209 +.set vgprGlobalReadOffsetA, 210 +.set vgprGlobalReadOffsetB, 214 +.set vgprG2LA, 218 +.set vgprValuA_X0_I1, 218 +.set vgprValuA_X1_I1, 222 +.set vgprValuA_X2_I1, 226 +.set vgprValuA_X3_I1, 230 +.set vgprG2LB, 234 +.set vgprLocalReadAddrA, 250 +.set vgprLocalReadAddrB, 251 +.set vgprSerial, 252 +/* Num VGPR=256 */ +/* Num AccVGPR=0 */ + +/******************************************/ +/* SGPR Assignments */ +/******************************************/ +.set sgprKernArgAddress, 0 +.set sgprWorkGroup0, 2 +.set sgprWorkGroup1, 3 +.set sgprWorkGroup2, 4 +.set sgprLoopCounterL, 5 +.set sgprOrigLoopCounter, 6 +.set sgprSrdA, 8 +.set sgprSrdB, 12 +.set sgprSrdD, 16 +.set sgprSrdC, 20 +.set sgprTensor2dSizeA, 24 +.set sgprTensor2dSizeB, 26 +.set sgprAddressD, 28 +.set sgprAddressC, 30 +.set sgprAddressA, 32 +.set sgprAddressB, 34 +.set sgprAlpha, 36 +.set sgprBeta, 38 +.set sgprStridesD, 40 +.set sgprStridesC, 42 +.set sgprStridesA, 44 +.set sgprStridesB, 46 +.set sgprSizesFree, 48 +.set sgprSizesSum, 51 +.set sgprOrigStaggerUIter, 52 +.set sgprNumWorkGroups0, 53 +.set sgprNumWorkGroups1, 54 +.set sgprNumFullBlocks, 55 +.set sgprWgmRemainder1, 56 +.set sgprMagicNumberWgmRemainder1, 57 +.set sgprOffsetD, 58 +.set sgprOffsetC, 59 +.set sgprOffsetA, 60 +.set sgprOffsetB, 61 +.set sgprShadowLimitA, 58 +.set sgprShadowLimitB, 60 +.set sgprStaggerUIter, 7 +.set sgprWrapUA, 62 +.set sgprWrapUB, 64 +.set sgprGlobalReadIncsA, 66 +.set sgprGlobalReadIncsB, 67 +/* max SGPR=73 */ + +/* Size Assignments */ +.set sgprSizeI, sgprSizesFree+0 +.set sgprSizeJ, sgprSizesFree+1 +.set sgprSizeK, sgprSizesFree+2 +.set sgprSizeL, sgprSizesSum+0 + +/* Stride Assignments */ +.set constStrideD0I, 1 +.set sgprStrideD1J, sgprStridesD+0 +.set sgprStrideDK, sgprStridesD+1 +.set constStrideC0I, 1 +.set sgprStrideC1J, sgprStridesC+0 +.set sgprStrideCK, sgprStridesC+1 +.set constStrideA0I, 1 +.set sgprStrideAL, sgprStridesA+0 +.set sgprStrideAK, sgprStridesA+1 +.set constStrideBL, 1 +.set sgprStrideB1J, sgprStridesB+0 +.set sgprStrideBK, sgprStridesB+1 + +.set MT0, 128 +.set MT1, 128 +.set DepthU, 16 +.set GSU, 1 +.set BpeA, 8 +.set BpeALog2, 3 +.set BpeB, 8 +.set BpeBLog2, 3 +/* Number of elements to shift-left SRD */ +.set SrdShiftLeftA, 2 +.set SrdShiftLeftB, 2 +/* 2GB limit - set offsets to -1 to exceed this and clamp */ +.set BufferLimit, 0xffffffff +.set BufferOOB, 0x80000000 + +/******************************************/ +/* Bits 127:96 of SRD. */ +/* hex: 0x00020000 */ +/* dst_sel_x (3b): 0 */ +/* dst_sel_y (3b): 0 */ +/* dst_sel_z (3b): 0 */ +/* dst_sel_w (3b): 0 */ +/* num_format (3b): 0 */ +/* data_format (4b): 4 */ +/* user_vm_enable (1b): 0 */ +/* user_vm_mode (1b): 0 */ +/* index_stride (2b): 0 */ +/* add_tid_enable (1b): 0 */ +/* _unusedA (3b): 0 */ +/* nv (1b): 0 */ +/* _unusedB (2b): 0 */ +/* type (2b): 0 */ +/******************************************/ +.set Srd127_96, 0x00020000 + +/* Global Offset A */ +.macro GLOBAL_OFFSET_A vgprAddr:req vgprOffset0I:req vgprOffsetL:req vgprTmp:req +v_mul_lo_u32 v[\vgprTmp+0], s[sgprStrideAL], v[\vgprOffsetL] // mul d1 lower +_v_add_co_u32 v[\vgprAddr+0], vcc, v[\vgprOffset0I], v[\vgprTmp+0] // accumulate K lower +_v_add_u32 v[\vgprAddr+0], 0x2, v[\vgprAddr+0] // add prepad for pointer shift +v_lshlrev_b32 v[\vgprAddr+0], 0x3, v[\vgprAddr+0] // offset *= bytes/element +.endm + +/* Global Offset B */ +.macro GLOBAL_OFFSET_B vgprAddr:req vgprOffsetL:req vgprOffset1J:req vgprTmp:req +v_mul_lo_u32 v[\vgprTmp+0], s[sgprStrideB1J], v[\vgprOffset1J] // mul d1 lower +_v_add_co_u32 v[\vgprAddr+0], vcc, v[\vgprOffsetL], v[\vgprTmp+0] // accumulate K lower +_v_add_u32 v[\vgprAddr+0], 0x2, v[\vgprAddr+0] // add prepad for pointer shift +v_lshlrev_b32 v[\vgprAddr+0], 0x3, v[\vgprAddr+0] // offset *= bytes/element +.endm + +/******************************************/ +/* Dynamic Scalar Divide: vQuotient=vDividend/vDivisor; vRemainder=vDividend%vDivisor; */ +/******************************************/ +.macro DYNAMIC_VECTOR_DIVIDE vQuotient vRemainder vDividend vDivisor vTmp0 vTmp1 sTmp +v_cvt_f32_u32 v[\vQuotient], v[\vDivisor] // +v_rcp_f32 v[\vQuotient], v[\vQuotient] // +v_mul_f32 v[\vQuotient], 0x4f800000, v[\vQuotient] // +v_cvt_u32_f32 v[\vQuotient], v[\vQuotient] // +v_mul_lo_u32 v[\vRemainder], v[\vDivisor], v[\vQuotient] // +v_mul_hi_u32 v[\vTmp0], v[\vDivisor], v[\vQuotient] // +_v_sub_co_u32 v[\vTmp1], vcc, 0x0, v[\vRemainder] // +v_cmp_ne_i32 s[\sTmp:\sTmp+1], 0x0, v[\vTmp0] // +v_cndmask_b32 v[\vRemainder], v[\vTmp1], v[\vRemainder], s[\sTmp:\sTmp+1] // +v_mul_hi_u32 v[\vRemainder], v[\vRemainder], v[\vQuotient] // +_v_sub_co_u32 v[\vTmp0], vcc, v[\vQuotient], v[\vRemainder] // +_v_add_co_u32 v[\vQuotient], vcc, v[\vQuotient], v[\vRemainder] // +v_cndmask_b32 v[\vQuotient], v[\vQuotient], v[\vTmp0], s[\sTmp:\sTmp+1] // +v_mul_hi_u32 v[\vQuotient], v[\vQuotient], v[\vDividend] // +v_mul_lo_u32 v[\vRemainder], v[\vQuotient], v[\vDivisor] // +_v_sub_co_u32 v[\vTmp0], vcc, v[\vDividend], v[\vRemainder] // +v_cmp_ge_u32 s[\sTmp:\sTmp+1], v[\vDividend], v[\vRemainder] // +_v_add_co_u32 v[\vRemainder], vcc, 0x1, v[\vQuotient] // +_v_add_co_u32 v[\vTmp1], vcc, -1, v[\vQuotient] // +v_cmp_le_u32 vcc, v[\vDivisor], v[\vTmp0] // +s_and_b64 vcc, s[\sTmp:\sTmp+1], vcc // +v_cndmask_b32 v[\vQuotient], v[\vQuotient], v[\vRemainder], vcc // +v_cndmask_b32 v[\vQuotient], v[\vTmp1], v[\vQuotient], s[\sTmp:\sTmp+1] // +v_cmp_ne_i32 vcc, 0x0, v[\vDivisor] // +v_cndmask_b32 v[\vQuotient], -1, v[\vQuotient], vcc // final result +v_mul_lo_u32 v[\vRemainder], v[\vQuotient], v[\vDivisor] // +_v_sub_co_u32 v[\vRemainder], vcc, v[\vDividend], v[\vRemainder] // final result +.endm + + + +/******************************************/ +/* Allocate Resources */ +/******************************************/ + +s_setprio 3 // optimization store +s_mov_b32 m0, 0x9000 // LDS clamp at 36864 bytes +v_mov_b32 v[vgprSerial], v0 // thread serial id + +/* Load Kernel Args */ +s_load_dwordx16 s[24:39], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x8 // +s_load_dwordx16 s[40:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x48 // +s_load_dwordx4 s[56:59], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x88 // +s_load_dwordx2 s[60:61], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x98 // +s_waitcnt lgkmcnt(0) // wait for 160 bytes of kern args +s_lshl_b32 s[sgprOffsetD], s[sgprOffsetD], 0x3 // elements offset to bytes offset +s_add_u32 s[sgprAddressD+0], s[sgprAddressD+0], s[sgprOffsetD] // add offset to buffer address +s_addc_u32 s[sgprAddressD+1], s[sgprAddressD+1], 0 // add offset to buffer address +s_lshl_b32 s[sgprOffsetC], s[sgprOffsetC], 0x3 // elements offset to bytes offset +s_add_u32 s[sgprAddressC+0], s[sgprAddressC+0], s[sgprOffsetC] // add offset to buffer address +s_addc_u32 s[sgprAddressC+1], s[sgprAddressC+1], 0 // add offset to buffer address +s_lshl_b32 s[sgprOffsetA], s[sgprOffsetA], 0x3 // elements offset to bytes offset +s_add_u32 s[sgprAddressA+0], s[sgprAddressA+0], s[sgprOffsetA] // add offset to buffer address +s_addc_u32 s[sgprAddressA+1], s[sgprAddressA+1], 0 // add offset to buffer address +s_lshl_b32 s[sgprOffsetB], s[sgprOffsetB], 0x3 // elements offset to bytes offset +s_add_u32 s[sgprAddressB+0], s[sgprAddressB+0], s[sgprOffsetB] // add offset to buffer address +s_addc_u32 s[sgprAddressB+1], s[sgprAddressB+1], 0 // add offset to buffer address + +.set OffsetD, UNDEF +.set OffsetC, UNDEF +.set OffsetA, UNDEF +.set OffsetB, UNDEF + +/* Short circuit condition if Alpha == 0, then sumDims=0 */ +v_cmp_eq_f64 vcc, s[sgprAlpha:sgprAlpha+1], 0.0 // Alpha == 0.0 ? +s_cbranch_vccz label_AlphaNonZero // branch if Alpha != 0 +s_mov_b32 s[sgprSizesSum+0], 0x0 // Set summation dim=0 if Alpha == 0 +label_AlphaNonZero: + + +/******************************************/ +/* Local Read Addresses */ +/******************************************/ + + +/* local read addresses: tile assignments a/b */ + +/*lr0I*/ +v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) + // 1. N offset: nOffset = nIdx * nStride(1) (multiplier is 1, do nothing) +v_lshrrev_b32 v0, 4, v2 // 2. block offset: bnIdx = wtid / dividedForBlkId(16) +v_and_b32 v0, 0, v0 // 2. block offset: bnIdx = bnIdx % num1DBlocks(1) +v_lshlrev_b32 v0, 0x4, v0 // 2. block offset: bnOffset = bnIdx * strideBlock(16) +_v_add_u32 v1, v0, v1 // 3. add N and block offset: bnOffset = block and N offset +v_lshlrev_b32 v1, 0x1, v1 // 3. apply VectorWidth: bnOffset = bnOffset * vw(2) +v_lshrrev_b32 v2, 4, v2 // 4. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v2, 0x8, v2 // 4. K offset: lrKOffset = kIdx * mStride(256) +_v_add_u32 v1, v2, v1 // 5. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v0, 6, v[vgprSerial] // 6. wave offset in N dimen: wtid = tid / dividedForWaveId(64) +v_and_b32 v0, 3, v0 // 6. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) +v_lshlrev_b32 v0, 0x5, v0 // 6. wave offset in M dimen: wOffset = wtid0 * W0Stride(32) +_v_add_u32 v1, v0, v1 // 7. final local read offset: flrOffset = lrOffset + WOffset +/*lr1J*/ +v_and_b32 v3, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v2, 15, v3 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v2, 0x4, v2 // 1. N offset: nOffset = nIdx * nStride(16) +v_lshrrev_b32 v0, 4, v3 // 2. block offset: bnIdx = wtid / dividedForBlkId(16) +v_and_b32 v0, 0, v0 // 2. block offset: bnIdx = bnIdx % num1DBlocks(1) +v_lshlrev_b32 v0, 0x8, v0 // 2. block offset: bnOffset = bnIdx * strideBlock(256) +_v_add_u32 v2, v0, v2 // 3. add N and block offset: bnOffset = block and N offset + // 3. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) +v_lshrrev_b32 v3, 4, v3 // 4. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v3, 0x1, v3 // 4. K offset: lrKOffset = kIdx * mStride(2) +_v_add_u32 v2, v3, v2 // 5. offset in wave: lrOffset = bnOffset + lrKOffset + + +/* local read addresses: final offsets a */ + +// v_lshrrev_b32 v0, 8, v[vgprSerial] // LSU offset: sgid = Serial / subGroup(256) +// s_mov_b32 s68, 128 // LSU offset: stirde = MT0(128) + PAD0(0) +// v_mul_lo_u32 v0, s68, v0 // LSU offset: lsuoffset = sgid*(MT0+PAD) +// _v_add_lshl_u32 v[vgprLocalReadAddrA], v0, v1, 0x3 // Final Offset: offset = (lro0*VW+lsuoffset)*bpe + + +/* local read addresses: final offsets b */ + +v_lshrrev_b32 v0, 8, v[vgprSerial] // LSU offset: sgid = Serial / subGroup(256) +s_mov_b32 s68, 128 // LSU offset: stirde = MT1(128) + PAD1(0) +v_mul_lo_u32 v0, s68, v0 // LSU offset: lsuoffset = sgid*(MT1+PAD) +_v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v2, 0x3 // Final Offset: offset = (lro1*VW+lsuoffset)*bpe +v_lshrrev_b32 v1, 7, v[vgprLocalReadAddrB] // Final Offset: padding 4 per block 128 +v_lshlrev_b32 v1, 0x5, v1 // Final Offset: padding 4 per block 128 +_v_add_u32 v[vgprLocalReadAddrB], v1, v[vgprLocalReadAddrB] // Final Offset: add padding 4 per block 128 + + +/* local read addresses: declare addresses a */ + +/* N/A */ + + +/* local read addresses: declare addresses b */ + +// _v_add_co_u32 v[vgprLocalReadAddrB+0], vcc, 0x4000, v[vgprLocalReadAddrB+0] // += LdsOffsetB (lower) + + + +/******************************************/ +/* Begin setupNewTile, isPap=False */ +/******************************************/ + + +/* global read addresses: work-group */ + +/* graWorkGroup mapping */ +s_mov_b32 s71, 0x20000001L // magic number for WGM==4 +s_mul_hi_u32 s69, s[sgprWorkGroup1], s71 // s_magic mul +s_mul_i32 s68, s[sgprWorkGroup1], s71 // s_magic mul +s_lshr_b64 s[68:69], s[68:69], 31 // sMagicDiv +s_mul_i32 s69, s68, 4 // quotient * non-magic divisor +s_sub_u32 s69, s[sgprWorkGroup1], s69 // WorkGroup1=remainder +s_mul_i32 s69, s69, s[sgprNumWorkGroups0] // (wg1 % WGM)*nwg0 +s_add_u32 s69, s69, s[sgprWorkGroup0] // wgSerial = wg0 + (wg1 % WGM)*nwg0 +s_cmp_ge_u32 s68, s[sgprNumFullBlocks] // blockId >= numFullBlocks ? +s_cmov_b32 s71, s[sgprMagicNumberWgmRemainder1] // +s_cselect_b32 s70, s[sgprWgmRemainder1], 4 // +s_mul_hi_u32 s3, s69, s71 // s_magic mul +s_mul_i32 s2, s69, s71 // s_magic mul +s_lshr_b64 s[2:3], s[2:3], 31 // sMagicDiv +s_mul_i32 s[sgprWorkGroup1], s[sgprWorkGroup0], s70 // quotient * non-magic divisor +s_sub_u32 s[sgprWorkGroup1], s69, s[sgprWorkGroup1] // WorkGroup1=remainder +s_mul_i32 s68, s68, 4 // blockId * WGM +s_add_u32 s[sgprWorkGroup1], s[sgprWorkGroup1], s68 // wg1 += blockId * WGM + + +/* global read addresses: tile offset assignment a */ + +/* LVCA = 64 */ +/* v0 = (local)groA-tile = serial%LVCA (note (wgA*MTA) will be added to SRD) */ +/* v1 = groA-unroll = serial/LVCA */ +v_and_b32 v0, 15, v[vgprSerial] // v0 = v[vgprSerial] % 16 +v_lshlrev_b32 v0, 0x1, v0 // v0 = v0 * 2 +v_lshrrev_b32 v1, 6, v[vgprSerial] // v1 = v[vgprSerial] / 64 +v_lshlrev_b32 v1, 5, v1 // v1 = v1 * 32 +v_add_u32 v0, v0, v1 // v0 = v0 + v1 + +v_and_b32 v1, 63, v[vgprSerial] // v1 = (v[vgprSerial] % 64) / 16 +v_lshrrev_b32 v1, 4, v1 // v1 = (v[vgprSerial] % 64) / 16 +v_lshlrev_b32 v1, 1, v1 // v1 = v1 * 2 + +// v_lshrrev_b32 v1, 6, v[vgprSerial] // v1 = v[vgprSerial] / 64 +// v_and_b32 v0, 63, v[vgprSerial] // v0 = v[vgprSerial] % 64 +// /* gro-tile *= glvw */ +// v_lshlrev_b32 v0, 0x1, v0 // v0 = v0 * 2 + + +/* global read addresses: tile offset assignment b */ + +/* LVCB = 8 */ +/* v2 = (local)groB-tile = serial/LVCB (note (wgB*MTB) will be added to SRD) */ +/* v3 = groB-unroll = serial%LVCB */ +v_and_b32 v6, 63, v[vgprSerial] // v6 = v[vgprSerial] % 64 +v_lshrrev_b32 v2, 3, v6 // v2 = v6 / 8 +v_and_b32 v3, 7, v6 // v3 = v6 % 8 +v_readfirstlane_b32 s68, v[vgprSerial] // WaveIdxWavefrontWidth +s_lshr_b32 s68, s68, 0x6 // WaveId +s_mul_i32 s68, s68, 32 // Global Read Wave: each wave loads continuous lsp(8)*nrp(4) columns +_v_add_u32 v2, s68, v2 // Global Read Wave: add back to cloumn index +/* gro-unroll *= glvw */ +v_lshlrev_b32 v3, 0x1, v3 // v3 = v3 * 2 + + +/* global read addresses: unroll assignment a */ + +/* v1 */ + + +/* global read addresses: unroll assignment b */ + +/* v3 */ + + +/* global read addresses: other free assignments */ + +/* s[sgprWorkGroup2] */ + + +/* global read addresses: tile offsets a */ + +v_mov_b32 v4, v0 // groA0I_0 + + +/* global read addresses: tile offsets b */ + +v_mov_b32 v5, v2 // groB1J_0 +_v_add_co_u32 v6, vcc, 8, v5 // groB1J_1 += LSPB +_v_add_co_u32 v7, vcc, 8, v6 // groB1J_2 += LSPB +_v_add_co_u32 v8, vcc, 8, v7 // groB1J_3 += LSPB + + +/* global read addresses: unroll offsets a */ + +v_mov_b32 v9, v1 // groAL_0 +_v_add_co_u32 v10, vcc, 1, v9 // groAL_1 + LSPA +_v_add_co_u32 v11, vcc, 8, v9 // groAL_2 + LSPA +_v_add_co_u32 v12, vcc, 9, v9 // groAL_3 + LSPA + + +/* global read addresses: unroll offsets b */ + +v_mov_b32 v13, v3 // groBL_0 + + +/* global read addresses: shift a */ + +s_mul_i32 s68, s[sgprWorkGroup0], 128 // WorkGroup[01] * MT +s_sub_u32 s68, s[sgprSizeI], s68 // edge = Size0I - WG*MT +s_sub_u32 s68, s68, 2 // edge -= margin(2) +v_mov_b32 v14, s68 // edge vgpr = Size0I- WG*MT - margin(2) +_v_add_co_u32 v15, vcc, v14, 2 // shiftedEdge = edge + srdShiftLeft(2) +_v_add_co_u32 v16, vcc, v4, 2 // shiftedOffset = offset + srdShiftLeft(2) +v_cmp_lt_u32 s[68:69], v16, v15 // shiftedOffset < shiftedEdge +v_cndmask_b32 v4, v14, v4, s[68:69] // offset = (offset < edge) ? offset(v4) : edge(v14) + + +/* global read addresses: final offsets a */ + +GLOBAL_OFFSET_A vgprGlobalReadOffsetA+0, 4, 9, 14 // gROA_0_0_0_0 +GLOBAL_OFFSET_A vgprGlobalReadOffsetA+1, 4, 10, 14 // gROA_0_0_1_0 +GLOBAL_OFFSET_A vgprGlobalReadOffsetA+2, 4, 11, 14 // gROA_0_0_2_0 +GLOBAL_OFFSET_A vgprGlobalReadOffsetA+3, 4, 12, 14 // gROA_0_0_3_0 + + +/* global read addresses: final offsets b */ + +GLOBAL_OFFSET_B vgprGlobalReadOffsetB+0, 13, 5, 9 // gROB_0_0_0_0 +GLOBAL_OFFSET_B vgprGlobalReadOffsetB+1, 13, 6, 9 // gROB_0_0_1_0 +GLOBAL_OFFSET_B vgprGlobalReadOffsetB+2, 13, 7, 9 // gROB_0_0_2_0 +GLOBAL_OFFSET_B vgprGlobalReadOffsetB+3, 13, 8, 9 // gROB_0_0_3_0 + + +/* global read addresses: addresses a */ + +/* max read offset = size[n] * stride[n-1] */ +s_mul_hi_u32 s71, s[sgprWorkGroup0], 128 // WorkGroup[01] * MT +s_mul_i32 s70, s[sgprWorkGroup0], 128 // WorkGroup[01] * MT +s_sub_u32 s[sgprShadowLimitA+0], s[sgprTensor2dSizeA], s70 // sub tileStart +s_subb_u32 s[sgprShadowLimitA+1], s[sgprTensor2dSizeA+1], s71 // sub tileStart +s_lshl_b64 s[sgprShadowLimitA:sgprShadowLimitA+1], s[sgprShadowLimitA:sgprShadowLimitA+1], 0x3 // Set limit to use bytes +s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], 16 // extend limit for pre-pad +s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], 0 // extend limit for pre-pad +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +s_mul_hi_u32 s69, s[sgprStrideAK], s[sgprWorkGroup2] // Stride*WG +s_mul_i32 s68, s[sgprStrideAK], s[sgprWorkGroup2] // Stride*WG +s_add_u32 s70, s70, s68 // accum wg term to tilestart +s_addc_u32 s71, s71, s69 // accum wg term to tilestart +s_lshl_b64 s[70:71], s[70:71], 0x3 // tileStart *= BPE +s_add_u32 s[sgprSrdA+0], s[sgprAddressA+0], s70 // SRD base = Address+ tileStart0 +s_addc_u32 s[sgprSrdA+1], s[sgprAddressA+1], s71 // SRD base = Address+ tileStart1 +s_sub_u32 s[sgprSrdA+0], s[sgprSrdA+0], 16 // pre-pad to make room for possible pointer shift +s_subb_u32 s[sgprSrdA+1], s[sgprSrdA+1], 0 // pre-pad to make room for possible pointer shift +s_mov_b32 s[sgprSrdA+3], Srd127_96 // Set bits 127_96 in SRD + + +/* global read addresses: addresses b */ + +/* max read offset = size[n] * stride[n-1] */ +s_mul_hi_u32 s71, s[sgprWorkGroup1], 128 // WorkGroup[01] * MT +s_mul_i32 s70, s[sgprWorkGroup1], 128 // WorkGroup[01] * MT +s_mul_hi_u32 s71, s70, s[sgprStrideB1J] // tlu=0, scaled tile-offset by stride +s_mul_i32 s70, s70, s[sgprStrideB1J] // tlu=0, scaled tile-offset by stride +s_sub_u32 s[sgprShadowLimitB+0], s[sgprTensor2dSizeB], s70 // sub tileStart +s_subb_u32 s[sgprShadowLimitB+1], s[sgprTensor2dSizeB+1], s71 // sub tileStart +s_lshl_b64 s[sgprShadowLimitB:sgprShadowLimitB+1], s[sgprShadowLimitB:sgprShadowLimitB+1], 0x3 // Set limit to use bytes +s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], 16 // extend limit for pre-pad +s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], 0 // extend limit for pre-pad +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +s_mul_hi_u32 s69, s[sgprStrideBK], s[sgprWorkGroup2] // Stride*WG +s_mul_i32 s68, s[sgprStrideBK], s[sgprWorkGroup2] // Stride*WG +s_add_u32 s70, s70, s68 // accum wg term to tilestart +s_addc_u32 s71, s71, s69 // accum wg term to tilestart +s_lshl_b64 s[70:71], s[70:71], 0x3 // tileStart *= BPE +s_add_u32 s[sgprSrdB+0], s[sgprAddressB+0], s70 // SRD base = Address+ tileStart0 +s_addc_u32 s[sgprSrdB+1], s[sgprAddressB+1], s71 // SRD base = Address+ tileStart1 +s_sub_u32 s[sgprSrdB+0], s[sgprSrdB+0], 16 // pre-pad to make room for possible pointer shift +s_subb_u32 s[sgprSrdB+1], s[sgprSrdB+1], 0 // pre-pad to make room for possible pointer shift +s_mov_b32 s[sgprSrdB+3], Srd127_96 // Set bits 127_96 in SRD + + +/* global read addresses: increments a */ + +s_mul_i32 s[sgprGlobalReadIncsA+0], DepthU*BpeA, s[sgprStrideAL] // incrA unrollIdx) + + +/* global read addresses: increments b */ + +s_mov_b32 s[sgprGlobalReadIncsB+0], DepthU*BpeB // incrB (unrollIdx) + + +/******************************************/ +/* Local Write Addresses */ +/******************************************/ + +/* lwaTileAssignmentA = v0 */ + +/* lwaTileAssignmentB = v2 */ + +/* lwaUnrollAssignmentA = v1 */ + +/* lwaUnrollAssignmentB = v3 */ + + +/* local write addresses: first offset a */ + +// v_mul_u32_u24 v[vgprLocalWriteAddrA], 0x80, v1 // lwAL**(MTA + PAD) +// _v_add_lshl_u32 v[vgprLocalWriteAddrA], v0, v[vgprLocalWriteAddrA], 0x3 // lwFOA = (lwAA + lwAL*(MT0I+PAD))*bpe + + +/* local write addresses: first offset b */ + +v_mul_u32_u24 v[vgprLocalWriteAddrB], 0x10, v2 // lwBL**(DepthU_Compute + PAD) +_v_add_lshl_u32 v[vgprLocalWriteAddrB], v3, v[vgprLocalWriteAddrB], 0x3 // lwFOB = (lwBB + lwBL*(DepthU+PAD))*bpe +v_lshrrev_b32 v3, 7, v[vgprLocalWriteAddrB] // padding 4 per block 128 +v_lshlrev_b32 v3, 0x5, v3 // padding 4 per block 128 +_v_add_u32 v[vgprLocalWriteAddrB], v3, v[vgprLocalWriteAddrB] // add padding 4 per block 128 +// _v_add_co_u32 v[vgprLocalWriteAddrB], vcc, 0x4000, v[vgprLocalWriteAddrB] // lwFOB = lwB1J + lwBL*MT1J + LDS_OFFSET_B=2048*8 + + + + + + + +/* declare loop num iterations */ + + +s_lshr_b32 s[sgprLoopCounterL], s[sgprSizesSum+0], 4 // s[sgprLoopCounterL] = s[sgprSizesSum+0] / 16 +s_mov_b32 s[sgprOrigLoopCounter], s[sgprLoopCounterL] // copy loop counter + +s_and_b32 s[sgprStaggerUIter], s[sgprOrigStaggerUIter], s[sgprWorkGroup0] // Compute actual stagger start for this tile +s_lshl_b32 s[sgprStaggerUIter], s[sgprStaggerUIter], 0 // shift by StaggerUStride + + +/* SRDs += (StaggerUIter) * GlobalReadIncsA+0 */ +s_mul_hi_i32 s69, s[sgprStaggerUIter], s[sgprGlobalReadIncsA+0] // stagger byte offset +s_mul_i32 s68, s[sgprStaggerUIter], s[sgprGlobalReadIncsA+0] // stagger byte offset +s_mul_hi_i32 s[sgprWrapUA+1], s[sgprLoopCounterL], s[sgprGlobalReadIncsA+0] // Number of bytes accessed by the unroll loop +s_mul_i32 s[sgprWrapUA+0], s[sgprLoopCounterL], s[sgprGlobalReadIncsA+0] // Number of bytes accessed by the unroll loop +s_sub_u32 s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0], s[sgprWrapUA+0] // remove one iteration +s_subb_u32 s[sgprWrapUA+1], 0, s[sgprWrapUA+1] // remove one iteration +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s68 // limit -= inc) +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0] // Move shadow to real if we are within 2^32 + + +/* SRDs += (StaggerUIter) * GlobalReadIncsB+0 */ +s_mul_hi_i32 s69, s[sgprStaggerUIter], s[sgprGlobalReadIncsB+0] // stagger byte offset +s_mul_i32 s68, s[sgprStaggerUIter], s[sgprGlobalReadIncsB+0] // stagger byte offset +s_mul_hi_i32 s[sgprWrapUB+1], s[sgprLoopCounterL], s[sgprGlobalReadIncsB+0] // Number of bytes accessed by the unroll loop +s_mul_i32 s[sgprWrapUB+0], s[sgprLoopCounterL], s[sgprGlobalReadIncsB+0] // Number of bytes accessed by the unroll loop +s_sub_u32 s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0], s[sgprWrapUB+0] // remove one iteration +s_subb_u32 s[sgprWrapUB+1], 0, s[sgprWrapUB+1] // remove one iteration +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s68 // limit -= inc) +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0] // Move shadow to real if we are within 2^32 +s_add_u32 s[sgprStaggerUIter], s[sgprStaggerUIter], 2 // Subtract (PGR-1); StaggerUIter now contains target iteration to wrap + +/* local read addresses: init pointers a */ + + +/* localReadInitPointers */ + +/* local read addresses: init pointers b */ + + +/* localReadInitPointers */ + + +/* prefetch: global -> local */ + +s_cmp_eq_u32 s[sgprLoopCounterL], 0 // at last iteration? +// s_setprio 0 // optimization store +s_cbranch_scc1 ShadowInitStart_9 // skip to ShadowInitStart iter b/c numIter==0 + +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+1], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+2], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+3], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_3_0 + +buffer_load_dwordx4 v[vgprValuA_X0_I0+0:vgprValuA_X0_I0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X0_I0+4:vgprValuA_X0_I0+4+3], v[vgprGlobalReadOffsetA+1], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X0_I0+8:vgprValuA_X0_I0+8+3], v[vgprGlobalReadOffsetA+2], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprValuA_X0_I0+12:vgprValuA_X0_I0+12+3], v[vgprGlobalReadOffsetA+3], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_3_0 + + + + +/* global read inc A loopL */ +s_add_u32 s70, s[sgprLoopCounterL], 1 // remove pf(1) +s_cmp_eq_u32 s[sgprStaggerUIter], s70 // Is this wrapIter? (pf) +s_cselect_b32 s68, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +s_cselect_b32 s69, s[sgprWrapUA+1], 0 // incUpper <- ? +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s68 // limit -= inc) +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0] // Move shadow to real if we are within 2^32 + +/* global read inc B loopL */ +s_add_u32 s70, s[sgprLoopCounterL], 1 // remove pf(1) +s_cmp_eq_u32 s[sgprStaggerUIter], s70 // Is this wrapIter? (pf) +s_cselect_b32 s68, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +s_cselect_b32 s69, s[sgprWrapUB+1], 0 // incUpper <- ? +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s68 // limit -= inc) +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0] // Move shadow to real if we are within 2^32 + + +/******************************************/ +/* End setupNewTile, isPap=False */ +/******************************************/ + +ShadowInitStart_9: // + +s_mov_b32 s[sgprSrdD+0], s[sgprAddressD+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdD+1], s[sgprAddressD+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdD+2], 0x80000000 // +s_mov_b32 s[sgprSrdD+3], Srd127_96 // Set bits 127_96 in post-loop SRD + +s_mov_b32 s[sgprSrdC+0], s[sgprAddressC+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdC+1], s[sgprAddressC+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdC+2], 0x80000000 // +s_mov_b32 s[sgprSrdC+3], Srd127_96 // Set bits 127_96 in post-loop SRD + + +s_mul_i32 s70, MT1, s[sgprWorkGroup1] // <- wg1*MT1 +s_mul_hi_u32 s69, s70, s[sgprStrideC1J] // CScale s70 by Stride +s_mul_i32 s68, s70, s[sgprStrideC1J] // CScale s70 by Stride +s_lshl_b64 s[68:69], s[68:69], 3 // scale by bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s68 // add lo to SRD +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], s69 // add hi to SRD +s_mul_hi_u32 s69, s70, s[sgprStrideD1J] // Scale s70 by Stride +s_mul_i32 s68, s70, s[sgprStrideD1J] // Scale s70 by Stride +s_lshl_b64 s[68:69], s[68:69], 3 // scale by bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s68 // add lo to SRD +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], s69 // add hi to SRD + +s_mul_hi_u32 s69, s[sgprWorkGroup2], s[sgprStrideCK] // CScale s[sgprWorkGroup2] by Stride +s_mul_i32 s68, s[sgprWorkGroup2], s[sgprStrideCK] // CScale s[sgprWorkGroup2] by Stride +s_lshl_b64 s[68:69], s[68:69], 3 // scale by bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s68 // add lo to SRD +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], s69 // add hi to SRD +s_mul_hi_u32 s69, s[sgprWorkGroup2], s[sgprStrideDK] // Scale s[sgprWorkGroup2] by Stride +s_mul_i32 s68, s[sgprWorkGroup2], s[sgprStrideDK] // Scale s[sgprWorkGroup2] by Stride +s_lshl_b64 s[68:69], s[68:69], 3 // scale by bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s68 // add lo to SRD +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], s69 // add hi to SRD + + + +/* initC: remove C-tile 0-128 from pool */ +v_mov_b32 v208, 15728640 // set out-of-bound addr +ds_read_b32 v[vgprValuC+0], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+1], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+2], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+3], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+4], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+5], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+6], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+7], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+8], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+9], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+10], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+11], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+12], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+13], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+14], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+15], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+16], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+17], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+18], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+19], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+20], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+21], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+22], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+23], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+24], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+25], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+26], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+27], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+28], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+29], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+30], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+31], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+32], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+33], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+34], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+35], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+36], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+37], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+38], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+39], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+40], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+41], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+42], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+43], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+44], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+45], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+46], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+47], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+48], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+49], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+50], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+51], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+52], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+53], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+54], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+55], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+56], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+57], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+58], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+59], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+60], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+61], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+62], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+63], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+64], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+65], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+66], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+67], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+68], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+69], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+70], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+71], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+72], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+73], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+74], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+75], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+76], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+77], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+78], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+79], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+80], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+81], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+82], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+83], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+84], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+85], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+86], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+87], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+88], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+89], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+90], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+91], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+92], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+93], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+94], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+95], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+96], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+97], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+98], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+99], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+100], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+101], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+102], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+103], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+104], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+105], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+106], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+107], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+108], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+109], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+110], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+111], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+112], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+113], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+114], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+115], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+116], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+117], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+118], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+119], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+120], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+121], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+122], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+123], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+124], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+125], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+126], v208, offset:0 // initC +ds_read_b32 v[vgprValuC+127], v208, offset:0 // initC + +/* initC: remove AB-tile 128-208 from pool */ + +s_cmp_eq_u32 s[sgprLoopCounterL], 0 // at last iteration? + +/* after InitC, skip to end of prefetch last iter if numIter==0 */ +s_cbranch_scc0 label_NoBranch_10 // Only branch on scc1 +s_getpc_B64 s[68:69] // addr of next instr +s_add_i32 s70, PrefetchGlobalLastIterEnd_4, 0x4 // target branch offset +s_cmp_ge_i32 s70, 0x0 // check positive or negative +s_cbranch_scc1 label_Positive_11 // jump when positive +s_abs_i32 s70, s70 // abs offset +s_sub_u32 s68, s68, s70 // sub target branch offset +s_subb_u32 s69, s69, 0 // sub high and carry +s_setpc_b64 s[68:69] // branch to PrefetchGlobalLastIterEnd_4 +label_Positive_11: +s_add_u32 s68, s68, s70 // add target branch offset +s_addc_u32 s69, s69, 0 // add high and carry +s_setpc_b64 s[68:69] // branch to PrefetchGlobalLastIterEnd_4 +label_NoBranch_10: + +s_waitcnt vmcnt(4) // lgkmcnt=-1 vmcnt=08wait for global read + +/* local write b */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:1280 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 1280 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:2560 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 2560 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:3840 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 3840 + + +/* local write swap a */ + + + +/* local write swap b */ + + + + +s_cmp_eq_u32 s[sgprLoopCounterL] 0x1 // PGR=2 but only 1 loop +s_cbranch_scc1 label_0012 // PGR=2 but only 1 loop + + +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+1], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+2], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+3], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_3_0 + +buffer_load_dwordx4 v[vgprValuA_X0_I1+0:vgprValuA_X0_I1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X0_I1+4:vgprValuA_X0_I1+4+3], v[vgprGlobalReadOffsetA+1], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X0_I1+8:vgprValuA_X0_I1+8+3], v[vgprGlobalReadOffsetA+2], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprValuA_X0_I1+12:vgprValuA_X0_I1+12+3], v[vgprGlobalReadOffsetA+3], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_3_0 + +label_0012: // + +s_waitcnt lgkmcnt(0) // lgkmcnt=0 vmcnt=-10prefetch wait for local write + +// Skip force waitcnt0 +s_barrier // + + +/* local read prefetch b */ + +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 + + +/* local read inc a */ + +/* N/A, lro->128 */ +/* self.localReadDoCntA 1 self.localReadDoCntB 1 */ + + +/* local read inc b */ + +/* N/A, lro->8 */ +/* self.localReadDoCntA 1 self.localReadDoCntB 1 */ + + + +/******************************************/ +/* Unrolled Loop(s) - Begin */ +/******************************************/ + +openLoopL_13: +s_cmp_eq_u32 s[sgprLoopCounterL], 0x1 // LoopCounterL < EndCounter +s_cbranch_scc1 label_0014 // PGR=2 but only 1 loop, toPGR1 +s_cmp_le_u32 s[sgprLoopCounterL], 0x2 // LoopCounterL < EndCounter +s_cbranch_scc1 LoopEndL_2 // do not enter LoopL +LoopBeginL_1: + + +/******************************************/ +/* Unrolled Loop 1/2 - Begin */ +/******************************************/ + + +/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ + + + + + +/* iter 0 */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) vmcnt(8) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[0:7] +/* mfmaIndex:1 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* global read inc A loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +s_cselect_b32 s68, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +s_cselect_b32 s69, s[sgprWrapUA+1], 0 // incUpper <- ? +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[8:15] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s68 // limit -= inc) +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[16:23] +/* mfmaIndex:3 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0] // Move shadow to real if we are within 2^32 +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[24:31] +/* mfmaIndex:4 */ +s_setprio 0 // store optimization +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* global read inc B loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +s_cselect_b32 s68, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +s_cselect_b32 s69, s[sgprWrapUB+1], 0 // incUpper <- ? +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[32:39] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s68 // limit -= inc) +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[40:47] +/* mfmaIndex:6 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0] // Move shadow to real if we are within 2^32 +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[48:55] +/* mfmaIndex:7 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[56:63] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[64:71] +/* mfmaIndex:9 */ +/* localReadsVacancy: letencyLeft 1 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[72:79] +/* mfmaIndex:10 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[80:87] +/* mfmaIndex:11 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[88:95] +/* mfmaIndex:12 */ +s_setprio 3 // store optimization +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[96:103] +/* mfmaIndex:13 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[104:111] +/* mfmaIndex:14 */ +/* 1 LDS buffer: read-sync-write */ +s_waitcnt lgkmcnt(0) // +s_barrier // +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[112:119] +/* mfmaIndex:15 */ +s_waitcnt vmcnt(7) // lgkmcnt=-1 vmcnt=7wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[120:127] +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_0_0 +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=8 */ + + +/* iter 1 */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:16 */ +/* localReadsVacancy: letencyLeft 5 */ +s_waitcnt lgkmcnt(1) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=1, new=1 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[0:7] +/* mfmaIndex:17 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[8:15] +/* mfmaIndex:18 */ +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[16:23] +/* mfmaIndex:19 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[24:31] +/* mfmaIndex:20 */ +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[32:39] +/* mfmaIndex:21 */ +s_waitcnt vmcnt(7) // lgkmcnt=-1 vmcnt=7wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:1280 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 1280 +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[40:47] +/* mfmaIndex:22 */ +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+1], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[48:55] +/* mfmaIndex:23 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[56:63] +/* mfmaIndex:24 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[64:71] +/* mfmaIndex:25 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[72:79] +/* mfmaIndex:26 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[80:87] +/* mfmaIndex:27 */ +s_waitcnt vmcnt(7) // lgkmcnt=-1 vmcnt=7wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:2560 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 2560 +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[88:95] +/* mfmaIndex:28 */ +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+2], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_2_0 +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[96:103] +/* mfmaIndex:29 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[104:111] +/* mfmaIndex:30 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[112:119] +/* mfmaIndex:31 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=2 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:32 */ +s_waitcnt lgkmcnt(3) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=4 newLW=4 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[0:7] +/* mfmaIndex:33 */ +s_waitcnt vmcnt(7) // lgkmcnt=-1 vmcnt=7wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:3840 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 3840 +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[8:15] +/* mfmaIndex:34 */ +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+3], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_3_0 +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[16:23] +/* mfmaIndex:35 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[24:31] +/* mfmaIndex:36 */ +s_setprio 0 // store optimization +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[32:39] +/* mfmaIndex:37 */ +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[40:47] +/* mfmaIndex:38 */ +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[48:55] +/* mfmaIndex:39 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[56:63] +/* mfmaIndex:40 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[64:71] +/* mfmaIndex:41 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[72:79] +/* mfmaIndex:42 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[80:87] +/* mfmaIndex:43 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[88:95] +/* mfmaIndex:44 */ +s_setprio 3 // store optimization +s_waitcnt lgkmcnt(0) // lgkmcnt=0 vmcnt=-13wait for local write +s_barrier // +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[96:103] +/* mfmaIndex:45 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[104:111] +/* mfmaIndex:46 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[112:119] +/* mfmaIndex:47 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=1 numReadsIterA=3 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + + +/* iter 3 (swap and reset local write pointers iteration) */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:48 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +s_waitcnt lgkmcnt(4) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=4 newLW=4 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[0:7] +/* mfmaIndex:49 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[8:15] +/* mfmaIndex:50 */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[16:23] +/* mfmaIndex:51 */ +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[24:31] +/* mfmaIndex:52 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0+0:vgprValuA_X0_I0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_0_0 +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[32:39] +/* mfmaIndex:53 */ +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[40:47] +/* mfmaIndex:54 */ +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[48:55] +/* mfmaIndex:55 */ +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[56:63] +/* mfmaIndex:56 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0+4:vgprValuA_X0_I0+4+3], v[vgprGlobalReadOffsetA+1], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[64:71] +/* mfmaIndex:57 */ +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[72:79] +/* mfmaIndex:58 */ +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[80:87] +/* mfmaIndex:59 */ +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[88:95] +/* mfmaIndex:60 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0+8:vgprValuA_X0_I0+8+3], v[vgprGlobalReadOffsetA+2], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_2_0 +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[96:103] +/* mfmaIndex:61 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[104:111] +/* mfmaIndex:62 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[112:119] +/* mfmaIndex:63 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[120:127] +buffer_load_dwordx4 v[vgprValuA_X0_I0+12:vgprValuA_X0_I0+12+3], v[vgprGlobalReadOffsetA+3], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_3_0 +/* numPrefetchIter=1 */ +/* dataAtIterA=2 numReadsIterA=3 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=8 */ + + + + +/******************************************/ +/* Unrolled Loop - End */ +/******************************************/ + +s_sub_u32 s[sgprLoopCounterL], s[sgprLoopCounterL], 1 // dec counterL + +/******************************************/ +/* Unrolled Loop 2/2 - Begin */ +/******************************************/ + + +/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ + + + + + +/* iter 0 */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) vmcnt(8) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[0:7] +/* mfmaIndex:1 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* global read inc A loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +s_cselect_b32 s68, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +s_cselect_b32 s69, s[sgprWrapUA+1], 0 // incUpper <- ? +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[8:15] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s68 // limit -= inc) +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[16:23] +/* mfmaIndex:3 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0] // Move shadow to real if we are within 2^32 +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[24:31] +/* mfmaIndex:4 */ +s_setprio 0 // store optimization +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* global read inc B loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +s_cselect_b32 s68, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +s_cselect_b32 s69, s[sgprWrapUB+1], 0 // incUpper <- ? +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[32:39] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s68 // limit -= inc) +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[40:47] +/* mfmaIndex:6 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0] // Move shadow to real if we are within 2^32 +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[48:55] +/* mfmaIndex:7 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[56:63] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[64:71] +/* mfmaIndex:9 */ +/* localReadsVacancy: letencyLeft 1 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[72:79] +/* mfmaIndex:10 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[80:87] +/* mfmaIndex:11 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[88:95] +/* mfmaIndex:12 */ +s_setprio 3 // store optimization +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[96:103] +/* mfmaIndex:13 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[104:111] +/* mfmaIndex:14 */ +/* 1 LDS buffer: read-sync-write */ +s_waitcnt lgkmcnt(0) // +s_barrier // +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[112:119] +/* mfmaIndex:15 */ +s_waitcnt vmcnt(7) // lgkmcnt=-1 vmcnt=7wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[120:127] +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_0_0 +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=8 */ + + +/* iter 1 */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:16 */ +/* localReadsVacancy: letencyLeft 5 */ +s_waitcnt lgkmcnt(1) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=1, new=1 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[0:7] +/* mfmaIndex:17 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[8:15] +/* mfmaIndex:18 */ +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[16:23] +/* mfmaIndex:19 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[24:31] +/* mfmaIndex:20 */ +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[32:39] +/* mfmaIndex:21 */ +s_waitcnt vmcnt(7) // lgkmcnt=-1 vmcnt=7wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:1280 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 1280 +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[40:47] +/* mfmaIndex:22 */ +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+1], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[48:55] +/* mfmaIndex:23 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[56:63] +/* mfmaIndex:24 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[64:71] +/* mfmaIndex:25 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[72:79] +/* mfmaIndex:26 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[80:87] +/* mfmaIndex:27 */ +s_waitcnt vmcnt(7) // lgkmcnt=-1 vmcnt=7wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:2560 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 2560 +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[88:95] +/* mfmaIndex:28 */ +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+2], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_2_0 +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[96:103] +/* mfmaIndex:29 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[104:111] +/* mfmaIndex:30 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[112:119] +/* mfmaIndex:31 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=2 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:32 */ +s_waitcnt lgkmcnt(3) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=4 newLW=4 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[0:7] +/* mfmaIndex:33 */ +s_waitcnt vmcnt(7) // lgkmcnt=-1 vmcnt=7wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:3840 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 3840 +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[8:15] +/* mfmaIndex:34 */ +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+3], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // G -> Reg 0_0_3_0 +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[16:23] +/* mfmaIndex:35 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[24:31] +/* mfmaIndex:36 */ +s_setprio 0 // store optimization +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[32:39] +/* mfmaIndex:37 */ +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[40:47] +/* mfmaIndex:38 */ +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[48:55] +/* mfmaIndex:39 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[56:63] +/* mfmaIndex:40 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[64:71] +/* mfmaIndex:41 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[72:79] +/* mfmaIndex:42 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[80:87] +/* mfmaIndex:43 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[88:95] +/* mfmaIndex:44 */ +s_setprio 3 // store optimization +s_waitcnt lgkmcnt(0) // lgkmcnt=0 vmcnt=-13wait for local write +s_barrier // +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[96:103] +/* mfmaIndex:45 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[104:111] +/* mfmaIndex:46 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[112:119] +/* mfmaIndex:47 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=1 numReadsIterA=3 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + + +/* iter 3 (swap and reset local write pointers iteration) */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:48 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +s_waitcnt lgkmcnt(4) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=4 newLW=4 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[0:7] +/* mfmaIndex:49 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[8:15] +/* mfmaIndex:50 */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[16:23] +/* mfmaIndex:51 */ +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[24:31] +/* mfmaIndex:52 */ +buffer_load_dwordx4 v[vgprValuA_X0_I1+0:vgprValuA_X0_I1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_0_0 +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[32:39] +/* mfmaIndex:53 */ +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[40:47] +/* mfmaIndex:54 */ +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[48:55] +/* mfmaIndex:55 */ +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[56:63] +/* mfmaIndex:56 */ +buffer_load_dwordx4 v[vgprValuA_X0_I1+4:vgprValuA_X0_I1+4+3], v[vgprGlobalReadOffsetA+1], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[64:71] +/* mfmaIndex:57 */ +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[72:79] +/* mfmaIndex:58 */ +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[80:87] +/* mfmaIndex:59 */ +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[88:95] +/* mfmaIndex:60 */ +buffer_load_dwordx4 v[vgprValuA_X0_I1+8:vgprValuA_X0_I1+8+3], v[vgprGlobalReadOffsetA+2], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_2_0 +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[96:103] +/* mfmaIndex:61 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[104:111] +/* mfmaIndex:62 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[112:119] +/* mfmaIndex:63 */ +s_sub_u32 s[sgprLoopCounterL], s[sgprLoopCounterL], 1 // dec counterL +s_cmp_eq_u32 s[sgprLoopCounterL], 0x2 // counterL==2 +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[120:127] +buffer_load_dwordx4 v[vgprValuA_X0_I1+12:vgprValuA_X0_I1+12+3], v[vgprGlobalReadOffsetA+3], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // G -> Reg 0_0_3_0 +/* numPrefetchIter=1 */ +/* dataAtIterA=2 numReadsIterA=3 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=8 */ + + + + +/******************************************/ +/* Unrolled Loop - End */ +/******************************************/ + + +/* closeLoop loopL finalLoop=1 tailLoop=0 */ +s_cbranch_scc0 LoopBeginL_1 // restart LoopL +LoopEndL_oddexit_3: // unroll loop odditer exit +LoopEndL_2: + + +/* Before NLL: Check VGPR.checkin for INT8 LW */ + + + + +/******************************************/ +/* NoGlobalLoadLoop - Begin */ +/******************************************/ + +s_setprio 0 // store optimization + +/* iter 0 */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) vmcnt(8) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[0:7] +/* mfmaIndex:1 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* global read inc A loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +s_cselect_b32 s68, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +s_cselect_b32 s69, s[sgprWrapUA+1], 0 // incUpper <- ? +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[8:15] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s68 // limit -= inc) +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[16:23] +/* mfmaIndex:3 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0] // Move shadow to real if we are within 2^32 +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[24:31] +/* mfmaIndex:4 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* global read inc B loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +s_cselect_b32 s68, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +s_cselect_b32 s69, s[sgprWrapUB+1], 0 // incUpper <- ? +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[32:39] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s68 // limit -= inc) +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[40:47] +/* mfmaIndex:6 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0] // Move shadow to real if we are within 2^32 +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[48:55] +/* mfmaIndex:7 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[56:63] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[64:71] +/* mfmaIndex:9 */ +/* localReadsVacancy: letencyLeft 1 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[72:79] +/* mfmaIndex:10 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[80:87] +/* mfmaIndex:11 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[88:95] +/* mfmaIndex:12 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[96:103] +/* mfmaIndex:13 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[104:111] +/* mfmaIndex:14 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[112:119] +/* mfmaIndex:15 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=8 */ + + +/* iter 1 */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:16 */ +/* localReadsVacancy: letencyLeft 5 */ +s_waitcnt lgkmcnt(0) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=1, new=1 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[0:7] +/* mfmaIndex:17 */ +/* localReadsVacancy: letencyLeft 5 */ +/* 1 LDS buffer: read-sync-write */ +s_waitcnt lgkmcnt(0) // +s_barrier // +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[8:15] +/* mfmaIndex:18 */ +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[16:23] +/* mfmaIndex:19 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[24:31] +/* mfmaIndex:20 */ +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[32:39] +/* mfmaIndex:21 */ +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[40:47] +/* mfmaIndex:22 */ +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[48:55] +/* mfmaIndex:23 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[56:63] +/* mfmaIndex:24 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[64:71] +/* mfmaIndex:25 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[72:79] +/* mfmaIndex:26 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[80:87] +/* mfmaIndex:27 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[88:95] +/* mfmaIndex:28 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[96:103] +/* mfmaIndex:29 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[104:111] +/* mfmaIndex:30 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X1_I0+0+0+0:vgprValuA_X1_I0+0+0+0+1], v[112:119] +/* mfmaIndex:31 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X1_I0+2+0+0:vgprValuA_X1_I0+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=2 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:32 */ +s_waitcnt lgkmcnt(0) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=4 newLW=4 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[0:7] +/* mfmaIndex:33 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[8:15] +/* mfmaIndex:34 */ +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[16:23] +/* mfmaIndex:35 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[24:31] +/* mfmaIndex:36 */ +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[32:39] +/* mfmaIndex:37 */ +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[40:47] +/* mfmaIndex:38 */ +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[48:55] +/* mfmaIndex:39 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[56:63] +/* mfmaIndex:40 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[64:71] +/* mfmaIndex:41 */ +s_waitcnt vmcnt(7) // lgkmcnt=-1 vmcnt=3wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[72:79] +/* mfmaIndex:42 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[80:87] +/* mfmaIndex:43 */ +s_waitcnt vmcnt(6) // lgkmcnt=-1 vmcnt=2wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:1280 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 1280 +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[88:95] +/* mfmaIndex:44 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[96:103] +/* mfmaIndex:45 */ +s_waitcnt vmcnt(5) // lgkmcnt=-1 vmcnt=1wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:2560 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 2560 +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[104:111] +/* mfmaIndex:46 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], v[112:119] +/* mfmaIndex:47 */ +s_waitcnt vmcnt(4) // lgkmcnt=-1 vmcnt=0wait for global read before writing to local +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:3840 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 3840 + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+2+0+0:vgprValuA_X2_I0+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=1 numReadsIterA=3 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + + +/* iter 3 (swap and reset local write pointers iteration) */ + +/* grEndMfmaIndex:6, lwStartMfmaIndex:18, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:48 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +s_waitcnt lgkmcnt(4) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=4 newLW=4 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[0:7] +/* mfmaIndex:49 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[8:15] +/* mfmaIndex:50 */ +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[16:23] +/* mfmaIndex:51 */ +s_waitcnt lgkmcnt(0) // lgkmcnt=0 vmcnt=-13wait for local write +// Skip force waitcnt0 +s_barrier // +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[24:31] +/* mfmaIndex:52 */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[32:39] +/* mfmaIndex:53 */ +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[40:47] +/* mfmaIndex:54 */ +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[48:55] +/* mfmaIndex:55 */ +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[56:63] +/* mfmaIndex:56 */ +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[64:71] +/* mfmaIndex:57 */ +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[72:79] +/* mfmaIndex:58 */ +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[80:87] +/* mfmaIndex:59 */ +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[88:95] +/* mfmaIndex:60 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[96:103] +/* mfmaIndex:61 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[104:111] +/* mfmaIndex:62 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X3_I0+0+0+0:vgprValuA_X3_I0+0+0+0+1], v[112:119] +/* mfmaIndex:63 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X3_I0+2+0+0:vgprValuA_X3_I0+2+0+0+1], v[120:127] +/* numPrefetchIter=1 */ +/* dataAtIterA=2 numReadsIterA=3 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=8 */ + +label_0014: + + +/******************************************/ +/* Opt. NoLoadLoop Without PAP - Begin */ +/******************************************/ + +s_mov_b32 s68, s[sgprBeta+0] // tmp = Beta[0] +s_or_b32 s68, s[sgprBeta+1], s68 // tmp |= Beta[1] +s_cmpk_eq_u32 s68, 0x0 // Beta == 0 +s_cbranch_scc0 OptNLL_End_16 // Branch if Beta is not zero + +s_mov_b32 s68, 0 // Low part of double 1.0 +s_mov_b32 s69, 0x3ff00000 // High part of double 1.0 +s_cmp_eq_u64 s[sgprAlpha:sgprAlpha+1], s[68:69] // Alpha == 1.0 ? +s_cbranch_scc0 OptNLL_End_16 // branch if alpha != 1 + +s_and_b32 s68, 127, s[sgprSizeI] // s68 = s[sgprSizeI] % 128 +s_add_u32 s69, -0x1, s[sgprNumWorkGroups0] // +s_cmp_ge_u32 s[sgprWorkGroup0], s69 // wg0 >= nwg0-1 ? +s_cselect_b32 s68, s68, 0 // set rMT0 +s_cmpk_gt_u32 s68, 0x0 // rMT0 > 0 +s_cbranch_scc1 OptNLL_End_16 // jump if edges required +s_and_b32 s68, 127, s[sgprSizeJ] // s68 = s[sgprSizeJ] % 128 +s_add_u32 s69, -0x1, s[sgprNumWorkGroups1] // +s_cmp_ge_u32 s[sgprWorkGroup1], s69 // wg1 >= nwg1-1 +s_cselect_b32 s68, s68, 0 // set rMT1 +s_cmpk_gt_u32 s68, 0x0 // rMT1 > 0 +s_cbranch_scc1 OptNLL_End_16 // jump if edges required + +s_and_b32 s69, 15, s[sgprSizesSum+0] // s69 = s[sgprSizesSum+0] % 16 +s_cmp_eq_u32 s69, 0x0 // numIterL == 0 +s_cbranch_scc0 OptNLL_End_16 // skip if tail loop required + + + +/* iter 0 (last unrolled loop) */ + +/* grEndMfmaIndex:0, lwStartMfmaIndex:48, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) vmcnt(3) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[0:7] +/* mfmaIndex:1 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[8:15] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[16:23] +/* mfmaIndex:3 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[24:31] +/* mfmaIndex:4 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[32:39] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[40:47] +/* mfmaIndex:6 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[48:55] +/* mfmaIndex:7 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[56:63] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[64:71] +/* mfmaIndex:9 */ +/* localReadsVacancy: letencyLeft 1 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[72:79] +/* mfmaIndex:10 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[80:87] +/* mfmaIndex:11 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[88:95] +/* mfmaIndex:12 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[96:103] +/* mfmaIndex:13 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[104:111] +/* mfmaIndex:14 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[112:119] +/* mfmaIndex:15 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=8 */ + + +/* iter 1 (last unrolled loop) */ + +/* grEndMfmaIndex:0, lwStartMfmaIndex:48, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:16 */ +/* localReadsVacancy: letencyLeft 5 */ +s_waitcnt lgkmcnt(0) vmcnt(2) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=1, new=1 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[0:7] +/* mfmaIndex:17 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[8:15] +/* mfmaIndex:18 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[16:23] +/* mfmaIndex:19 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[24:31] +/* mfmaIndex:20 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[32:39] +/* mfmaIndex:21 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[40:47] +/* mfmaIndex:22 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[48:55] +/* mfmaIndex:23 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[56:63] +/* mfmaIndex:24 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[64:71] +/* mfmaIndex:25 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[72:79] +/* mfmaIndex:26 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[80:87] +/* mfmaIndex:27 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[88:95] +/* mfmaIndex:28 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[96:103] +/* mfmaIndex:29 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[104:111] +/* mfmaIndex:30 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[112:119] +/* mfmaIndex:31 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=2 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + + +/* iter 2 (last unrolled loop) */ + +/* grEndMfmaIndex:0, lwStartMfmaIndex:48, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:32 */ +/* localReadsVacancy: letencyLeft 5 */ +s_waitcnt lgkmcnt(0) vmcnt(1) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[0:7] +/* mfmaIndex:33 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[8:15] +/* mfmaIndex:34 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[16:23] +/* mfmaIndex:35 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[24:31] +/* mfmaIndex:36 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[32:39] +/* mfmaIndex:37 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[40:47] +/* mfmaIndex:38 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[48:55] +/* mfmaIndex:39 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[56:63] +/* mfmaIndex:40 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[64:71] +/* mfmaIndex:41 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[72:79] +/* mfmaIndex:42 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[80:87] +/* mfmaIndex:43 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[88:95] +/* mfmaIndex:44 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[96:103] +/* mfmaIndex:45 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[104:111] +/* mfmaIndex:46 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[112:119] +/* mfmaIndex:47 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=1 numReadsIterA=3 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + + +/* iter 3 (last unrolled loop) */ + +/* grEndMfmaIndex:0, lwStartMfmaIndex:48, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:48 */ +s_waitcnt lgkmcnt(0) vmcnt(0) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[0:7] +/* mfmaIndex:49 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[8:15] +/* mfmaIndex:50 */ +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[16:23] +/* mfmaIndex:51 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[24:31] +/* mfmaIndex:52 */ +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[32:39] +/* mfmaIndex:53 */ +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[40:47] +/* mfmaIndex:54 */ +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[48:55] +/* mfmaIndex:55 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[56:63] +/* mfmaIndex:56 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[64:71] +/* mfmaIndex:57 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[72:79] +/* mfmaIndex:58 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[80:87] +/* mfmaIndex:59 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[88:95] +/* mfmaIndex:60 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[96:103] +/* mfmaIndex:61 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[104:111] +/* mfmaIndex:62 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[112:119] +/* mfmaIndex:63 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=2 numReadsIterA=3 skipReadsIterA=0 readsPerIterA=1 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + +/* Stores for OptNLL */ +Summation_End_OptNLL_17: +s_setprio 0 // optimization store +/* endSummation: add vgpr [128...132) to pool (vgprValuA_X0_I0) */ +/* endSummation: add vgpr [144...160) to pool (vgprValuB_X0_I0) */ +/* endSummation: add vgpr [208...252) to pool */ +.set NumFullBlocks, UNDEF +.set WgmRemainder1, UNDEF +.set MagicNumberWgmRemainder1, UNDEF +.set ShadowLimitA, UNDEF +.set ShadowLimitB, UNDEF +.set WrapUA, UNDEF +.set WrapUB, UNDEF +.set GlobalReadIncsA, UNDEF +.set GlobalReadIncsB, UNDEF + +/* Mapping of Acc register -> C Vgpr register */ + +/* Multiply MI out register with Alpha -> C Vgpr register */ +/* computeStoreVgprs */ +v_lshrrev_b32 v144, 6, v[vgprSerial] // v144 = v[vgprSerial] / 64 +v_lshrrev_b32 v145, 2, v144 // v145 = v144 / 4 +v_mul_lo_u32 v145, 0x10, v145 // wave coordination offset 1 +v_and_b32 v129, 63, v[vgprSerial] // v129 = v[vgprSerial] % 64 +v_lshrrev_b32 v129, 4, v129 // v129 = v129 / 16 + // thread0 * continuous_output (multiplier is 1, do nothing) +v_add_u32 v129, v145, v129 // coordination 1 = wave_id1 + tid1 +v_mul_lo_u32 v130, v129, s[sgprStrideC1J] // offset 1 +v_mul_lo_u32 v131, v129, s[sgprStrideD1J] // offset 1 +v_and_b32 v128, 3, v144 // v128 = v144 % 4 +v_mul_lo_u32 v128, 0x10, v128 // wave coordination offset 0 +v_and_b32 v145, 15, v[vgprSerial] // v145 = v[vgprSerial] % 16 +_v_add_lshl_u32 v128, v145, v128, 1 // coordination 0 = wave_id0 + tid0 +s_mul_i32 s55, 128, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v128, s55, v128 // coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0 +s_mul_i32 s55, 128, s[sgprWorkGroup1] // wgp1 * MT1 +v_add_u32 v129, s55, v129 // coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1 +GW_B0_E0_20: + +/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=1 */ +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +_v_add_lshl_u32 v146, v131, v128, 0x3 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=128, coord0Vgpr=128 +v_mov_b32 v[vgprValuC+148], v[vgprValuC+0] // copy MI out reg to vreg[0] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+1] // copy MI out reg to vreg[1] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+8] // copy MI out reg to vreg[2] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+9] // copy MI out reg to vreg[3] + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+2] // copy MI out reg to vreg[4] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+3] // copy MI out reg to vreg[5] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+10] // copy MI out reg to vreg[6] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+11] // copy MI out reg to vreg[7] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+4] // copy MI out reg to vreg[8] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+5] // copy MI out reg to vreg[9] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+12] // copy MI out reg to vreg[10] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+13] // copy MI out reg to vreg[11] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+6] // copy MI out reg to vreg[12] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+7] // copy MI out reg to vreg[13] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+14] // copy MI out reg to vreg[14] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+15] // copy MI out reg to vreg[15] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #4 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+16] // copy MI out reg to vreg[16] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+17] // copy MI out reg to vreg[17] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+24] // copy MI out reg to vreg[18] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+25] // copy MI out reg to vreg[19] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #5 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+18] // copy MI out reg to vreg[20] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+19] // copy MI out reg to vreg[21] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+26] // copy MI out reg to vreg[22] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+27] // copy MI out reg to vreg[23] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #6 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+20] // copy MI out reg to vreg[24] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+21] // copy MI out reg to vreg[25] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+28] // copy MI out reg to vreg[26] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+29] // copy MI out reg to vreg[27] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #7 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+22] // copy MI out reg to vreg[28] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+23] // copy MI out reg to vreg[29] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+30] // copy MI out reg to vreg[30] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+31] // copy MI out reg to vreg[31] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #8 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+32] // copy MI out reg to vreg[32] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+33] // copy MI out reg to vreg[33] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+40] // copy MI out reg to vreg[34] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+41] // copy MI out reg to vreg[35] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #9 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+34] // copy MI out reg to vreg[36] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+35] // copy MI out reg to vreg[37] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+42] // copy MI out reg to vreg[38] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+43] // copy MI out reg to vreg[39] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #10 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+36] // copy MI out reg to vreg[40] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+37] // copy MI out reg to vreg[41] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+44] // copy MI out reg to vreg[42] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+45] // copy MI out reg to vreg[43] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #11 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+38] // copy MI out reg to vreg[44] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+39] // copy MI out reg to vreg[45] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+46] // copy MI out reg to vreg[46] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+47] // copy MI out reg to vreg[47] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #12 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+48] // copy MI out reg to vreg[48] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+49] // copy MI out reg to vreg[49] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+56] // copy MI out reg to vreg[50] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+57] // copy MI out reg to vreg[51] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #13 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+50] // copy MI out reg to vreg[52] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+51] // copy MI out reg to vreg[53] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+58] // copy MI out reg to vreg[54] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+59] // copy MI out reg to vreg[55] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #14 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+52] // copy MI out reg to vreg[56] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+53] // copy MI out reg to vreg[57] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+60] // copy MI out reg to vreg[58] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+61] // copy MI out reg to vreg[59] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #15 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+54] // copy MI out reg to vreg[60] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+55] // copy MI out reg to vreg[61] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+62] // copy MI out reg to vreg[62] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+63] // copy MI out reg to vreg[63] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #16 (d1,d0,vc1,vc0) = */ +/* (16,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(16,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+64] // copy MI out reg to vreg[64] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+65] // copy MI out reg to vreg[65] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+72] // copy MI out reg to vreg[66] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+73] // copy MI out reg to vreg[67] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #17 (d1,d0,vc1,vc0) = */ +/* (17,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(17,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+66] // copy MI out reg to vreg[68] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+67] // copy MI out reg to vreg[69] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+74] // copy MI out reg to vreg[70] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+75] // copy MI out reg to vreg[71] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #18 (d1,d0,vc1,vc0) = */ +/* (18,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(18,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+68] // copy MI out reg to vreg[72] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+69] // copy MI out reg to vreg[73] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+76] // copy MI out reg to vreg[74] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+77] // copy MI out reg to vreg[75] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #19 (d1,d0,vc1,vc0) = */ +/* (19,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(19,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+70] // copy MI out reg to vreg[76] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+71] // copy MI out reg to vreg[77] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+78] // copy MI out reg to vreg[78] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+79] // copy MI out reg to vreg[79] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #20 (d1,d0,vc1,vc0) = */ +/* (20,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(20,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+80] // copy MI out reg to vreg[80] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+81] // copy MI out reg to vreg[81] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+88] // copy MI out reg to vreg[82] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+89] // copy MI out reg to vreg[83] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #21 (d1,d0,vc1,vc0) = */ +/* (21,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(21,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+82] // copy MI out reg to vreg[84] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+83] // copy MI out reg to vreg[85] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+90] // copy MI out reg to vreg[86] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+91] // copy MI out reg to vreg[87] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #22 (d1,d0,vc1,vc0) = */ +/* (22,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(22,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+84] // copy MI out reg to vreg[88] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+85] // copy MI out reg to vreg[89] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+92] // copy MI out reg to vreg[90] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+93] // copy MI out reg to vreg[91] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #23 (d1,d0,vc1,vc0) = */ +/* (23,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(23,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+86] // copy MI out reg to vreg[92] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+87] // copy MI out reg to vreg[93] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+94] // copy MI out reg to vreg[94] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+95] // copy MI out reg to vreg[95] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #24 (d1,d0,vc1,vc0) = */ +/* (24,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(24,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+96] // copy MI out reg to vreg[96] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+97] // copy MI out reg to vreg[97] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+104] // copy MI out reg to vreg[98] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+105] // copy MI out reg to vreg[99] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #25 (d1,d0,vc1,vc0) = */ +/* (25,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(25,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+98] // copy MI out reg to vreg[100] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+99] // copy MI out reg to vreg[101] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+106] // copy MI out reg to vreg[102] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+107] // copy MI out reg to vreg[103] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #26 (d1,d0,vc1,vc0) = */ +/* (26,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(26,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+100] // copy MI out reg to vreg[104] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+101] // copy MI out reg to vreg[105] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+108] // copy MI out reg to vreg[106] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+109] // copy MI out reg to vreg[107] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #27 (d1,d0,vc1,vc0) = */ +/* (27,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(27,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+102] // copy MI out reg to vreg[108] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+103] // copy MI out reg to vreg[109] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+110] // copy MI out reg to vreg[110] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+111] // copy MI out reg to vreg[111] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #28 (d1,d0,vc1,vc0) = */ +/* (28,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(28,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+112] // copy MI out reg to vreg[112] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+113] // copy MI out reg to vreg[113] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+120] // copy MI out reg to vreg[114] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+121] // copy MI out reg to vreg[115] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #29 (d1,d0,vc1,vc0) = */ +/* (29,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(29,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+114] // copy MI out reg to vreg[116] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+115] // copy MI out reg to vreg[117] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+122] // copy MI out reg to vreg[118] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+123] // copy MI out reg to vreg[119] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #30 (d1,d0,vc1,vc0) = */ +/* (30,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(30,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+116] // copy MI out reg to vreg[120] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+117] // copy MI out reg to vreg[121] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+124] // copy MI out reg to vreg[122] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+125] // copy MI out reg to vreg[123] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #31 (d1,d0,vc1,vc0) = */ +/* (31,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(31,0,0,0) */ +v_mov_b32 v[vgprValuC+148], v[vgprValuC+118] // copy MI out reg to vreg[124] +v_mov_b32 v[vgprValuC+149], v[vgprValuC+119] // copy MI out reg to vreg[125] +v_mov_b32 v[vgprValuC+150], v[vgprValuC+126] // copy MI out reg to vreg[126] +v_mov_b32 v[vgprValuC+151], v[vgprValuC+127] // copy MI out reg to vreg[127] + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[148:151], v146, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_22 // jump to end +label_GW_End_22: + +s_endpgm // Kernel End +OptNLL_End_16: + + +/******************************************/ +/* Ord. NoLoadLoop - Begin */ +/******************************************/ + + + + +/* iter 0 (last unrolled loop) */ + +/* grEndMfmaIndex:0, lwStartMfmaIndex:48, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) vmcnt(3) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[0:7] +/* mfmaIndex:1 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[8:15] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[16:23] +/* mfmaIndex:3 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[24:31] +/* mfmaIndex:4 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[32:39] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[40:47] +/* mfmaIndex:6 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[48:55] +/* mfmaIndex:7 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[56:63] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=8 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[64:71] +/* mfmaIndex:9 */ +/* localReadsVacancy: letencyLeft 1 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[72:79] +/* mfmaIndex:10 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[80:87] +/* mfmaIndex:11 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[88:95] +/* mfmaIndex:12 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[96:103] +/* mfmaIndex:13 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[104:111] +/* mfmaIndex:14 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I1+0+0+0:vgprValuA_X0_I1+0+0+0+1], v[112:119] +/* mfmaIndex:15 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I1+2+0+0:vgprValuA_X0_I1+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=8 */ + + +/* iter 1 (last unrolled loop) */ + +/* grEndMfmaIndex:0, lwStartMfmaIndex:48, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:16 */ +/* localReadsVacancy: letencyLeft 5 */ +s_waitcnt lgkmcnt(0) vmcnt(2) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=1, new=1 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[0:7] +/* mfmaIndex:17 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[8:15] +/* mfmaIndex:18 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[16:23] +/* mfmaIndex:19 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[24:31] +/* mfmaIndex:20 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[32:39] +/* mfmaIndex:21 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[40:47] +/* mfmaIndex:22 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[48:55] +/* mfmaIndex:23 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[56:63] +/* mfmaIndex:24 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[64:71] +/* mfmaIndex:25 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[72:79] +/* mfmaIndex:26 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[80:87] +/* mfmaIndex:27 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[88:95] +/* mfmaIndex:28 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[96:103] +/* mfmaIndex:29 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[104:111] +/* mfmaIndex:30 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X1_I1+0+0+0:vgprValuA_X1_I1+0+0+0+1], v[112:119] +/* mfmaIndex:31 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X1_I1+2+0+0:vgprValuA_X1_I1+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=2 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + + +/* iter 2 (last unrolled loop) */ + +/* grEndMfmaIndex:0, lwStartMfmaIndex:48, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:32 */ +/* localReadsVacancy: letencyLeft 5 */ +s_waitcnt lgkmcnt(0) vmcnt(1) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[0:7] +/* mfmaIndex:33 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[8:15] +/* mfmaIndex:34 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[16:23] +/* mfmaIndex:35 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[24:31] +/* mfmaIndex:36 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[32:39] +/* mfmaIndex:37 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[40:47] +/* mfmaIndex:38 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[48:55] +/* mfmaIndex:39 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[56:63] +/* mfmaIndex:40 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[64:71] +/* mfmaIndex:41 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[72:79] +/* mfmaIndex:42 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[80:87] +/* mfmaIndex:43 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[88:95] +/* mfmaIndex:44 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[96:103] +/* mfmaIndex:45 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[104:111] +/* mfmaIndex:46 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I1+0+0+0:vgprValuA_X2_I1+0+0+0+1], v[112:119] +/* mfmaIndex:47 */ +/* localReadsVacancy: letencyLeft 5 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I1+2+0+0:vgprValuA_X2_I1+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=1 numReadsIterA=3 skipReadsIterA=1 readsPerIterA=1 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + + +/* iter 3 (last unrolled loop) */ + +/* grEndMfmaIndex:0, lwStartMfmaIndex:48, lwEndMfmaIndex:48 */ +/* numMfmaForLR:13, barrierMfmaIndex:50 */ +/* mfmaIndex:48 */ +s_waitcnt lgkmcnt(0) vmcnt(0) // lgkmcnt=0 vmcnt=-1wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[0:7] +/* mfmaIndex:49 */ +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[8:15] +/* mfmaIndex:50 */ +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[16:23] +/* mfmaIndex:51 */ +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[24:31] +/* mfmaIndex:52 */ +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[32:39] +/* mfmaIndex:53 */ +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[40:47] +/* mfmaIndex:54 */ +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[48:55] +/* mfmaIndex:55 */ +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[56:63] +/* mfmaIndex:56 */ +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[64:71] +/* mfmaIndex:57 */ +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[72:79] +/* mfmaIndex:58 */ +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[80:87] +/* mfmaIndex:59 */ +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[88:95] +/* mfmaIndex:60 */ +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[96:103] +/* mfmaIndex:61 */ +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[104:111] +/* mfmaIndex:62 */ +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X3_I1+0+0+0:vgprValuA_X3_I1+0+0+0+1], v[112:119] +/* mfmaIndex:63 */ +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X3_I1+2+0+0:vgprValuA_X3_I1+2+0+0+1], v[120:127] +/* numPrefetchIter=0 */ +/* dataAtIterA=2 numReadsIterA=3 skipReadsIterA=0 readsPerIterA=1 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=8 */ + +PrefetchGlobalLastIterEnd_4: + + +/******************************************/ +/* Tail Loop */ +/******************************************/ + + +/* local write reset offsets a */ + + + +/* local write reset offsets b */ + + +/* tail loop: add vgpr [132...136) to pool (vgprValuA_X1_I0) */ +/* tail loop: add vgpr [136...140) to pool (vgprValuA_X2_I0) */ +/* tail loop: add vgpr [140...144) to pool (vgprValuA_X3_I0) */ +/* tail loop: add vgpr [160...176) to pool (vgprValuB_X1_I0) */ +/* tail loop: add vgpr [176...192) to pool (vgprValuB_X2_I0) */ +/* tail loop: add vgpr [192...208) to pool (vgprValuB_X3_I0) */ + +//numIterL = (((sizeL % LOCAL_DEPTHU) + LOCAL_SPLITU - 1) / LOCAL_SPLITU) +s_and_b32 s[sgprLoopCounterL], 15, s[sgprSizesSum+0] // s[sgprLoopCounterL] = s[sgprSizesSum+0] % 16 +s_cmp_eq_u32 s[sgprLoopCounterL], 0x0 // numIterL == 0 +s_mov_b32 s[sgprOrigLoopCounter], 0 // repurpose to count each localRead increment +s_cbranch_scc1 SkipTailLoopL_7 // skip to end of tail loop b/c numIter==0 + + +/* remove stagger offsets for tail loop */ + +s_sub_i32 s68, 3, s[sgprStaggerUIter] // +s_mul_hi_i32 s69, s68, s[sgprGlobalReadIncsA+0] // start offset S in bytes +s_mul_i32 s68, s68, s[sgprGlobalReadIncsA+0] // start offset S in bytes +s_sub_u32 s68, s68, s[sgprWrapUA] // S - WrapU +s_subb_u32 s69, s69, s[sgprWrapUA+1] // S - WrapU +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s68 // limit -= inc) +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0] // Move shadow to real if we are within 2^32 + +s_sub_i32 s68, 3, s[sgprStaggerUIter] // +s_mul_hi_i32 s69, s68, s[sgprGlobalReadIncsB+0] // start offset S in bytes +s_mul_i32 s68, s68, s[sgprGlobalReadIncsB+0] // start offset S in bytes +s_sub_u32 s68, s68, s[sgprWrapUB] // S - WrapU +s_subb_u32 s69, s69, s[sgprWrapUB+1] // S - WrapU +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s68 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s69 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s68 // limit -= inc) +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s69 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cmov_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0] // Move shadow to real if we are within 2^32 + + +/* Update M0 for DTLDS */ + + + +/* global read a */ + +/* g2l=0, load component 0 */ +buffer_load_dwordx2 v[vgprG2LA+0+0:vgprG2LA+0+0+1], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_dwordx2 v[vgprG2LA+0+2:vgprG2LA+0+2+1], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0, offen offset:8 // load one buffer value +/* g2l=4, load component 0 */ +buffer_load_dwordx2 v[vgprG2LA+4+0:vgprG2LA+4+0+1], v[vgprGlobalReadOffsetA+1], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // load one buffer value +/* g2l=4, load component 1 */ +buffer_load_dwordx2 v[vgprG2LA+4+2:vgprG2LA+4+2+1], v[vgprGlobalReadOffsetA+1], s[sgprSrdA:sgprSrdA+3], 0, offen offset:8 // load one buffer value +/* g2l=8, load component 0 */ +buffer_load_dwordx2 v[vgprG2LA+8+0:vgprG2LA+8+0+1], v[vgprGlobalReadOffsetA+2], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // load one buffer value +/* g2l=8, load component 1 */ +buffer_load_dwordx2 v[vgprG2LA+8+2:vgprG2LA+8+2+1], v[vgprGlobalReadOffsetA+2], s[sgprSrdA:sgprSrdA+3], 0, offen offset:8 // load one buffer value +/* g2l=12, load component 0 */ +buffer_load_dwordx2 v[vgprG2LA+12+0:vgprG2LA+12+0+1], v[vgprGlobalReadOffsetA+3], s[sgprSrdA:sgprSrdA+3], 0, offen offset:0 // load one buffer value +/* g2l=12, load component 1 */ +buffer_load_dwordx2 v[vgprG2LA+12+2:vgprG2LA+12+2+1], v[vgprGlobalReadOffsetA+3], s[sgprSrdA:sgprSrdA+3], 0, offen offset:8 // load one buffer value + + +/* Update M0 for DTLDS */ + + + +/* global read b */ + +/* g2l=0, load component 0 */ +buffer_load_dwordx2 v[vgprG2LB+0+0:vgprG2LB+0+0+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_dwordx2 v[vgprG2LB+0+2:vgprG2LB+0+2+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0, offen offset:8 // load one buffer value +/* g2l=4, load component 0 */ +buffer_load_dwordx2 v[vgprG2LB+4+0:vgprG2LB+4+0+1], v[vgprGlobalReadOffsetB+1], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // load one buffer value +/* g2l=4, load component 1 */ +buffer_load_dwordx2 v[vgprG2LB+4+2:vgprG2LB+4+2+1], v[vgprGlobalReadOffsetB+1], s[sgprSrdB:sgprSrdB+3], 0, offen offset:8 // load one buffer value +/* g2l=8, load component 0 */ +buffer_load_dwordx2 v[vgprG2LB+8+0:vgprG2LB+8+0+1], v[vgprGlobalReadOffsetB+2], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // load one buffer value +/* g2l=8, load component 1 */ +buffer_load_dwordx2 v[vgprG2LB+8+2:vgprG2LB+8+2+1], v[vgprGlobalReadOffsetB+2], s[sgprSrdB:sgprSrdB+3], 0, offen offset:8 // load one buffer value +/* g2l=12, load component 0 */ +buffer_load_dwordx2 v[vgprG2LB+12+0:vgprG2LB+12+0+1], v[vgprGlobalReadOffsetB+3], s[sgprSrdB:sgprSrdB+3], 0, offen offset:0 // load one buffer value +/* g2l=12, load component 1 */ +buffer_load_dwordx2 v[vgprG2LB+12+2:vgprG2LB+12+2+1], v[vgprGlobalReadOffsetB+3], s[sgprSrdB:sgprSrdB+3], 0, offen offset:8 // load one buffer value + +s_waitcnt vmcnt(0) // lgkmcnt=-1 vmcnt=02wait for global read + +// Skip force waitcnt0 +s_barrier // + + + + +/* local write a */ + +ds_write_b128 v[vgprLocalWriteAddrA], v[vgprG2LA+0:vgprG2LA+0+3] offset:0 // lwoA_0_0_0_0 = (0*LSCA) + (0*LSPA)(*MT0I+PAD) = 0 +ds_write_b128 v[vgprLocalWriteAddrA], v[vgprG2LA+4:vgprG2LA+4+3] offset:4096 // lwoA_0_0_1_0 = (0*LSCA) + (1*LSPA)(*MT0I+PAD) = 4096 +ds_write_b128 v[vgprLocalWriteAddrA], v[vgprG2LA+8:vgprG2LA+8+3] offset:8192 // lwoA_0_0_2_0 = (0*LSCA) + (2*LSPA)(*MT0I+PAD) = 8192 +ds_write_b128 v[vgprLocalWriteAddrA], v[vgprG2LA+12:vgprG2LA+12+3] offset:12288 // lwoA_0_0_3_0 = (0*LSCA) + (3*LSPA)(*MT0I+PAD) = 12288 + + +/* local write b */ + +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:1280 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 1280 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:2560 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 2560 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:3840 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 3840 + + +/* Recalc local read offsets */ + +/*lr0I*/ +v_and_b32 v134, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v133, 15, v134 // 1. N offset: nIdx = wtid % MI_N(16) + // 1. N offset: nOffset = nIdx * nStride(1) (multiplier is 1, do nothing) +v_lshrrev_b32 v132, 4, v134 // 2. block offset: bnIdx = wtid / dividedForBlkId(16) +v_and_b32 v132, 0, v132 // 2. block offset: bnIdx = bnIdx % num1DBlocks(1) +v_lshlrev_b32 v132, 0x4, v132 // 2. block offset: bnOffset = bnIdx * strideBlock(16) +_v_add_u32 v133, v132, v133 // 3. add N and block offset: bnOffset = block and N offset +v_lshlrev_b32 v133, 0x1, v133 // 3. apply VectorWidth: bnOffset = bnOffset * vw(2) +v_lshrrev_b32 v134, 4, v134 // 4. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v134, 0x7, v134 // 4. K offset: lrKOffset = kIdx * mStride(128) +_v_add_u32 v133, v134, v133 // 5. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v132, 6, v[vgprSerial] // 6. wave offset in N dimen: wtid = tid / dividedForWaveId(64) +v_and_b32 v132, 3, v132 // 6. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) +v_lshlrev_b32 v132, 0x5, v132 // 6. wave offset in M dimen: wOffset = wtid0 * W0Stride(32) +_v_add_u32 v133, v132, v133 // 7. final local read offset: flrOffset = lrOffset + WOffset +/*lr1J*/ +v_and_b32 v135, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v134, 15, v135 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v134, 0x4, v134 // 1. N offset: nOffset = nIdx * nStride(16) +v_lshrrev_b32 v132, 4, v135 // 2. block offset: bnIdx = wtid / dividedForBlkId(16) +v_and_b32 v132, 0, v132 // 2. block offset: bnIdx = bnIdx % num1DBlocks(1) +v_lshlrev_b32 v132, 0x8, v132 // 2. block offset: bnOffset = bnIdx * strideBlock(256) +_v_add_u32 v134, v132, v134 // 3. add N and block offset: bnOffset = block and N offset + // 3. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) +v_lshrrev_b32 v135, 4, v135 // 4. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) + // 4. K offset: lrKOffset = kIdx * mStride(1) (multiplier is 1, do nothing) +_v_add_u32 v134, v135, v134 // 5. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v132, 8, v[vgprSerial] // LSU offset: sgid = Serial / subGroup(256) +s_mov_b32 s68, 128 // LSU offset: stirde = MT0(128) + PAD0(0) +v_mul_lo_u32 v132, s68, v132 // LSU offset: lsuoffset = sgid*(MT0+PAD) +_v_add_lshl_u32 v[vgprLocalReadAddrA], v132, v133, 0x3 // Final Offset: offset = (lro0*VW+lsuoffset)*bpe +/* N/A */ +v_lshrrev_b32 v132, 8, v[vgprSerial] // LSU offset: sgid = Serial / subGroup(256) +s_mov_b32 s68, 128 // LSU offset: stirde = MT1(128) + PAD1(0) +v_mul_lo_u32 v132, s68, v132 // LSU offset: lsuoffset = sgid*(MT1+PAD) +_v_add_lshl_u32 v[vgprLocalReadAddrB], v132, v134, 0x3 // Final Offset: offset = (lro1*VW+lsuoffset)*bpe +v_lshrrev_b32 v133, 7, v[vgprLocalReadAddrB] // Final Offset: padding 4 per block 128 +v_lshlrev_b32 v133, 0x5, v133 // Final Offset: padding 4 per block 128 +_v_add_u32 v[vgprLocalReadAddrB], v133, v[vgprLocalReadAddrB] // Final Offset: add padding 4 per block 128 +_v_add_co_u32 v[vgprLocalReadAddrB+0], vcc, 0x4000, v[vgprLocalReadAddrB+0] // += LdsOffsetB (lower) + +s_waitcnt lgkmcnt(0) // lgkmcnt=0 vmcnt=-15wait for local write + +// Skip force waitcnt0 +s_barrier // + + +/* local read reset offsets a */ + + + +/* local read reset offsets b */ + + + +/* local read init pointers a */ + + +/* localReadInitPointers */ + + +/* local read init pointers b */ + + +/* localReadInitPointers */ + + +/* tail loop: macs */ + +TailLoopBeginL_5: + + +/* local read a */ + +ds_read_b128 v[vgprValuA_X0_I0+0:vgprValuA_X0_I0+0+3], v[vgprLocalReadAddrA] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=128 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 + + +/* local read b */ + +ds_read_b64 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+1], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+2:vgprValuB_X0_I0+2+1], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+1], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+6:vgprValuB_X0_I0+6+1], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+1], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+10:vgprValuB_X0_I0+10+1], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+1], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+14:vgprValuB_X0_I0+14+1], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 + + +/* local read inc a */ + +s_mov_b32 s68, 0x1000 // inc +_v_add_co_u32 v[vgprLocalReadAddrA], vcc, s68, v[vgprLocalReadAddrA] // lrA += 4096 (LSU*(MT+PAD)*bpe) + + +/* local read inc b */ + +s_mov_b32 s68, 0x20 // inc +_v_add_co_u32 v[vgprLocalReadAddrB], vcc, s68, v[vgprLocalReadAddrB] // lrB += 32 (LSU*(MT+PAD)*bpe) + +s_waitcnt lgkmcnt(0) // lgkmcnt=0 vmcnt=-14wait for local read + + +v_and_b32 v132, 63, v[vgprSerial] // v132 = v[vgprSerial] % 64 +v_lshrrev_b32 v132, 4, v132 // v132 = v132 / 16 + // v132 = v132 * 1 (multiplier is 1, do nothing) +v_cmp_ge_i32 s[68:69], v132, s[sgprLoopCounterL] // check K index >= Size L +v_cndmask_b32 v[vgprValuA_X0_I0+0+0], v[vgprValuA_X0_I0+0+0], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+2+0], v[vgprValuA_X0_I0+2+0], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+0+0], v[vgprValuB_X0_I0+0+0], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+2+0], v[vgprValuB_X0_I0+2+0], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+4+0], v[vgprValuB_X0_I0+4+0], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+6+0], v[vgprValuB_X0_I0+6+0], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+8+0], v[vgprValuB_X0_I0+8+0], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+10+0], v[vgprValuB_X0_I0+10+0], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+12+0], v[vgprValuB_X0_I0+12+0], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+14+0], v[vgprValuB_X0_I0+14+0], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+0+1], v[vgprValuA_X0_I0+0+1], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+2+1], v[vgprValuA_X0_I0+2+1], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+0+1], v[vgprValuB_X0_I0+0+1], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+2+1], v[vgprValuB_X0_I0+2+1], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+4+1], v[vgprValuB_X0_I0+4+1], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+6+1], v[vgprValuB_X0_I0+6+1], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+8+1], v[vgprValuB_X0_I0+8+1], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+10+1], v[vgprValuB_X0_I0+10+1], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+12+1], v[vgprValuB_X0_I0+12+1], 0x0, s[68:69] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+14+1], v[vgprValuB_X0_I0+14+1], 0x0, s[68:69] // set 0 if K_idx >= sizeL +s_nop 1 +v_mfma_f64_16x16x4f64 v[0:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[0:7] +v_mfma_f64_16x16x4f64 v[8:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[8:15] +v_mfma_f64_16x16x4f64 v[16:23], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[16:23] +v_mfma_f64_16x16x4f64 v[24:31], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[24:31] +v_mfma_f64_16x16x4f64 v[32:39], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[32:39] +v_mfma_f64_16x16x4f64 v[40:47], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[40:47] +v_mfma_f64_16x16x4f64 v[48:55], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[48:55] +v_mfma_f64_16x16x4f64 v[56:63], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[56:63] +v_mfma_f64_16x16x4f64 v[64:71], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[64:71] +v_mfma_f64_16x16x4f64 v[72:79], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[72:79] +v_mfma_f64_16x16x4f64 v[80:87], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[80:87] +v_mfma_f64_16x16x4f64 v[88:95], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[88:95] +v_mfma_f64_16x16x4f64 v[96:103], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[96:103] +v_mfma_f64_16x16x4f64 v[104:111], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[104:111] +v_mfma_f64_16x16x4f64 v[112:119], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], v[112:119] +v_mfma_f64_16x16x4f64 v[120:127], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], v[120:127] + + +/* closeLoop loopL finalLoop=1 tailLoop=1 */ +s_sub_i32 s[sgprLoopCounterL], s[sgprLoopCounterL], 0x4 // dec counterL (tailLoop) +s_add_u32 s[sgprOrigLoopCounter], s[sgprOrigLoopCounter], 0x4 // inc counterL +s_cmp_le_i32 s[sgprLoopCounterL], 0x0 // counterL<=0 +s_cbranch_scc0 TailLoopBeginL_5 // restart LoopL +TailLoopEndL_6: + +SkipTailLoopL_7: + +Summation_End_25: +s_setprio 0 // optimization store +/* endSummation: add vgpr [128...132) to pool (vgprValuA_X0_I0) */ +/* endSummation: add vgpr [144...160) to pool (vgprValuB_X0_I0) */ +/* endSummation: add vgpr [208...252) to pool */ +.set NumFullBlocks, UNDEF +.set WgmRemainder1, UNDEF +.set MagicNumberWgmRemainder1, UNDEF +.set ShadowLimitA, UNDEF +.set ShadowLimitB, UNDEF +.set WrapUA, UNDEF +.set WrapUB, UNDEF +.set GlobalReadIncsA, UNDEF +.set GlobalReadIncsB, UNDEF + +/* Mapping of Acc register -> C Vgpr register */ + +/* Multiply MI out register with Alpha -> C Vgpr register */ + +// TODO in Generator +// skip shift vector if M % 2 == 0 +s_and_b32 s55, 0x1, s[sgprSizeI] +s_cbranch_scc0 label_0029 // done shifting + +/* shift vector components d0 */ + +v_mov_b32 v131, s[sgprWorkGroup0] // +v_mul_i32_i24 v131, -0x80, v131 // wg*MT +_v_add_co_u32 v131, vcc, s[sgprSizesFree+0], v131 // wgMT = Size - wg*MT +v_mov_b32 v132, 0x80 // MT +v_cmp_lt_u32 s[56:57], v131, v132 // wgMT < MT +v_cndmask_b32 v131, v132, v131, s[56:57] // wgMT = (wgMT < MT) ? wgMT : MT +v_lshrrev_b32 v133, 6, v[vgprSerial] // v133 = v[vgprSerial] / 64 +v_and_b32 v133, 3, v133 // v133 = v133 % 4 +v_lshrrev_b32 v134, 5, v131 // v134 = v131 / 32 +v_and_b32 v134, 3, v134 // v134 = v134 % 4 +v_cmp_eq_u32 s[56:57], v134, v133 // wave_id == block_belong_to_wave? +v_cndmask_b32 v131, v132, v131, s[56:57] // wgMT = (wgMT < MT) ? wgMT : MT + +/* mbReg: which mb block need to shift, mb(matrixInstCoal(16) * VectorWidth(2)) */ +v_lshrrev_b32 v132, 5, v131 // v132 = v131 / 32 +v_lshlrev_b32 v134, 0x0, v133 // v134 = v133 * 1 +_v_sub_u32 v132, v132, v134 // + +/* gbReg: glvw block id */ +v_lshrrev_b32 v134, 1, v131 // v134 = v131 / 2 + +/* tgbReg: glvw block id */ +v_lshrrev_b32 v135, 0, v[vgprSerial] // v135 = v[vgprSerial] / 1 +v_and_b32 v135, 15, v135 // v135 = v135 % 16 +v_lshlrev_b32 v135, 0x1, v135 // v135 = v135 * 2 +v_lshrrev_b32 v135, 1, v135 // v135 = v135 / 2 +v_lshlrev_b32 v133, 0x4, v133 // v133 = v133 * 16 +_v_add_co_u32 v135, vcc, v133, v135 // tgbReg = (tid_coal * continOut) / GLVW +_v_sub_u32 v134, v134, v135 // + +/* vwReg: glvw in which vw block? */ +v_and_b32 v133, 1, v131 // permute register between threads +v_lshrrev_b32 v133, 1, v133 // permute register between threads + +/* rReg : reminder of M_size % GlobalLoadVectorWidth */ +v_and_b32 v135, 1, v131 // v135 = v131 % 2 +v_cmp_eq_u32 vcc, v135, 0x1 // wgMT%VW == 1 +s_cbranch_vccnz label_0026 // branch to shift d0 r=1 +s_branch label_0029 // no shifting + +/******************************************/ +/* shift d0 r=1 */ +/******************************************/ +label_0026: +v_cmp_eq_u32 vcc, v132, 0x0 // +s_cbranch_vccnz label_0027 // branch to shift d0 r1 mb0 + +/******************************************/ +/* shift d0 r=1 mb=0 */ +/******************************************/ +label_0027: // r1 mb0 +v_cmp_eq_u32 vcc, v133, 0x0 // +s_cbranch_vccnz label_0028 // branch to shift d0 r1 mb0 vw0 + +/******************************************/ +/* shift d0 r=1 mb=0 vw0 */ +/******************************************/ +label_0028: // r1 mb0 vw0 +s_mov_b32 s56, 0 // +v_cmpx_eq_u32 s[56:57], v134, s56 // is thread in edge glvw region +v_and_b32 v128, 63, v[vgprSerial] // permute register between threads +v_lshlrev_b32 v128, 2, v128 // permute register between threads +v_mov_b32 v135, v8 // glvw 1 mb 0 tt1 0 r 0 +v_mov_b32 v0, v135 // +v_mov_b32 v135, v9 // glvw 1 mb 0 tt1 0 r 1 +v_mov_b32 v1, v135 // +v_mov_b32 v135, v10 // glvw 1 mb 0 tt1 1 r 0 +v_mov_b32 v2, v135 // +v_mov_b32 v135, v11 // glvw 1 mb 0 tt1 1 r 1 +v_mov_b32 v3, v135 // +v_mov_b32 v135, v12 // glvw 1 mb 0 tt1 2 r 0 +v_mov_b32 v4, v135 // +v_mov_b32 v135, v13 // glvw 1 mb 0 tt1 2 r 1 +v_mov_b32 v5, v135 // +v_mov_b32 v135, v14 // glvw 1 mb 0 tt1 3 r 0 +v_mov_b32 v6, v135 // +v_mov_b32 v135, v15 // glvw 1 mb 0 tt1 3 r 1 +v_mov_b32 v7, v135 // +v_mov_b32 v135, v24 // glvw 1 mb 0 tt1 4 r 0 +v_mov_b32 v16, v135 // +v_mov_b32 v135, v25 // glvw 1 mb 0 tt1 4 r 1 +v_mov_b32 v17, v135 // +v_mov_b32 v135, v26 // glvw 1 mb 0 tt1 5 r 0 +v_mov_b32 v18, v135 // +v_mov_b32 v135, v27 // glvw 1 mb 0 tt1 5 r 1 +v_mov_b32 v19, v135 // +v_mov_b32 v135, v28 // glvw 1 mb 0 tt1 6 r 0 +v_mov_b32 v20, v135 // +v_mov_b32 v135, v29 // glvw 1 mb 0 tt1 6 r 1 +v_mov_b32 v21, v135 // +v_mov_b32 v135, v30 // glvw 1 mb 0 tt1 7 r 0 +v_mov_b32 v22, v135 // +v_mov_b32 v135, v31 // glvw 1 mb 0 tt1 7 r 1 +v_mov_b32 v23, v135 // +v_mov_b32 v135, v40 // glvw 1 mb 0 tt1 8 r 0 +v_mov_b32 v32, v135 // +v_mov_b32 v135, v41 // glvw 1 mb 0 tt1 8 r 1 +v_mov_b32 v33, v135 // +v_mov_b32 v135, v42 // glvw 1 mb 0 tt1 9 r 0 +v_mov_b32 v34, v135 // +v_mov_b32 v135, v43 // glvw 1 mb 0 tt1 9 r 1 +v_mov_b32 v35, v135 // +v_mov_b32 v135, v44 // glvw 1 mb 0 tt1 10 r 0 +v_mov_b32 v36, v135 // +v_mov_b32 v135, v45 // glvw 1 mb 0 tt1 10 r 1 +v_mov_b32 v37, v135 // +v_mov_b32 v135, v46 // glvw 1 mb 0 tt1 11 r 0 +v_mov_b32 v38, v135 // +v_mov_b32 v135, v47 // glvw 1 mb 0 tt1 11 r 1 +v_mov_b32 v39, v135 // +v_mov_b32 v135, v56 // glvw 1 mb 0 tt1 12 r 0 +v_mov_b32 v48, v135 // +v_mov_b32 v135, v57 // glvw 1 mb 0 tt1 12 r 1 +v_mov_b32 v49, v135 // +v_mov_b32 v135, v58 // glvw 1 mb 0 tt1 13 r 0 +v_mov_b32 v50, v135 // +v_mov_b32 v135, v59 // glvw 1 mb 0 tt1 13 r 1 +v_mov_b32 v51, v135 // +v_mov_b32 v135, v60 // glvw 1 mb 0 tt1 14 r 0 +v_mov_b32 v52, v135 // +v_mov_b32 v135, v61 // glvw 1 mb 0 tt1 14 r 1 +v_mov_b32 v53, v135 // +v_mov_b32 v135, v62 // glvw 1 mb 0 tt1 15 r 0 +v_mov_b32 v54, v135 // +v_mov_b32 v135, v63 // glvw 1 mb 0 tt1 15 r 1 +v_mov_b32 v55, v135 // +v_mov_b32 v135, v72 // glvw 1 mb 0 tt1 16 r 0 +v_mov_b32 v64, v135 // +v_mov_b32 v135, v73 // glvw 1 mb 0 tt1 16 r 1 +v_mov_b32 v65, v135 // +v_mov_b32 v135, v74 // glvw 1 mb 0 tt1 17 r 0 +v_mov_b32 v66, v135 // +v_mov_b32 v135, v75 // glvw 1 mb 0 tt1 17 r 1 +v_mov_b32 v67, v135 // +v_mov_b32 v135, v76 // glvw 1 mb 0 tt1 18 r 0 +v_mov_b32 v68, v135 // +v_mov_b32 v135, v77 // glvw 1 mb 0 tt1 18 r 1 +v_mov_b32 v69, v135 // +v_mov_b32 v135, v78 // glvw 1 mb 0 tt1 19 r 0 +v_mov_b32 v70, v135 // +v_mov_b32 v135, v79 // glvw 1 mb 0 tt1 19 r 1 +v_mov_b32 v71, v135 // +v_mov_b32 v135, v88 // glvw 1 mb 0 tt1 20 r 0 +v_mov_b32 v80, v135 // +v_mov_b32 v135, v89 // glvw 1 mb 0 tt1 20 r 1 +v_mov_b32 v81, v135 // +v_mov_b32 v135, v90 // glvw 1 mb 0 tt1 21 r 0 +v_mov_b32 v82, v135 // +v_mov_b32 v135, v91 // glvw 1 mb 0 tt1 21 r 1 +v_mov_b32 v83, v135 // +v_mov_b32 v135, v92 // glvw 1 mb 0 tt1 22 r 0 +v_mov_b32 v84, v135 // +v_mov_b32 v135, v93 // glvw 1 mb 0 tt1 22 r 1 +v_mov_b32 v85, v135 // +v_mov_b32 v135, v94 // glvw 1 mb 0 tt1 23 r 0 +v_mov_b32 v86, v135 // +v_mov_b32 v135, v95 // glvw 1 mb 0 tt1 23 r 1 +v_mov_b32 v87, v135 // +v_mov_b32 v135, v104 // glvw 1 mb 0 tt1 24 r 0 +v_mov_b32 v96, v135 // +v_mov_b32 v135, v105 // glvw 1 mb 0 tt1 24 r 1 +v_mov_b32 v97, v135 // +v_mov_b32 v135, v106 // glvw 1 mb 0 tt1 25 r 0 +v_mov_b32 v98, v135 // +v_mov_b32 v135, v107 // glvw 1 mb 0 tt1 25 r 1 +v_mov_b32 v99, v135 // +v_mov_b32 v135, v108 // glvw 1 mb 0 tt1 26 r 0 +v_mov_b32 v100, v135 // +v_mov_b32 v135, v109 // glvw 1 mb 0 tt1 26 r 1 +v_mov_b32 v101, v135 // +v_mov_b32 v135, v110 // glvw 1 mb 0 tt1 27 r 0 +v_mov_b32 v102, v135 // +v_mov_b32 v135, v111 // glvw 1 mb 0 tt1 27 r 1 +v_mov_b32 v103, v135 // +v_mov_b32 v135, v120 // glvw 1 mb 0 tt1 28 r 0 +v_mov_b32 v112, v135 // +v_mov_b32 v135, v121 // glvw 1 mb 0 tt1 28 r 1 +v_mov_b32 v113, v135 // +v_mov_b32 v135, v122 // glvw 1 mb 0 tt1 29 r 0 +v_mov_b32 v114, v135 // +v_mov_b32 v135, v123 // glvw 1 mb 0 tt1 29 r 1 +v_mov_b32 v115, v135 // +v_mov_b32 v135, v124 // glvw 1 mb 0 tt1 30 r 0 +v_mov_b32 v116, v135 // +v_mov_b32 v135, v125 // glvw 1 mb 0 tt1 30 r 1 +v_mov_b32 v117, v135 // +v_mov_b32 v135, v126 // glvw 1 mb 0 tt1 31 r 0 +v_mov_b32 v118, v135 // +v_mov_b32 v135, v127 // glvw 1 mb 0 tt1 31 r 1 +v_mov_b32 v119, v135 // +s_mov_b64 s[56:57], 0xFFFFFFFFFFFFFFFF // to restore all threads active +s_or_saveexec_b64 vcc, s[56:57] // all threads active +s_branch label_0029 // done shifting + +label_0029: // end shift0 + + + +/* not-LocalSplitU: global write indices */ + +/* computeStoreVgprs */ +v_lshrrev_b32 v132, 6, v[vgprSerial] // v132 = v[vgprSerial] / 64 +v_lshrrev_b32 v133, 2, v132 // v133 = v132 / 4 +v_mul_lo_u32 v133, 0x10, v133 // wave coordination offset 1 +v_and_b32 v129, 63, v[vgprSerial] // v129 = v[vgprSerial] % 64 +v_lshrrev_b32 v129, 4, v129 // v129 = v129 / 16 + // thread0 * continuous_output (multiplier is 1, do nothing) +v_add_u32 v129, v133, v129 // coordination 1 = wave_id1 + tid1 +v_mul_lo_u32 v130, v129, s[sgprStrideC1J] // offset 1 +v_mul_lo_u32 v131, v129, s[sgprStrideD1J] // offset 1 +v_and_b32 v128, 3, v132 // v128 = v132 % 4 +v_mul_lo_u32 v128, 0x10, v128 // wave coordination offset 0 +v_and_b32 v133, 15, v[vgprSerial] // v133 = v[vgprSerial] % 16 +_v_add_lshl_u32 v128, v133, v128, 1 // coordination 0 = wave_id0 + tid0 +s_mul_i32 s55, 128, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v128, s55, v128 // coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0 +s_mul_i32 s55, 128, s[sgprWorkGroup1] // wgp1 * MT1 +v_add_u32 v129, s55, v129 // coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1 + + +/* not-LocalSplitU: global write */ + +s_mov_b32 s56, s[sgprBeta+0] // tmp = Beta[0] +s_or_b32 s56, s[sgprBeta+1], s56 // tmp |= Beta[1] +s_cmpk_eq_u32 s56, 0x0 // Beta == 0 +s_cbranch_scc0 GW_Beta_46 // Branch if Beta is not zero + +s_and_b32 s56, 127, s[sgprSizeI] // s56 = s[sgprSizeI] % 128 +s_add_u32 s57, -0x1, s[sgprNumWorkGroups0] // +s_cmp_ge_u32 s[sgprWorkGroup0], s57 // wg0 >= nwg0-1 ? +s_cselect_b32 s56, s56, 0 // set rMT0 +s_cmpk_gt_u32 s56, 0x0 // rMT0 > 0 +s_cbranch_scc1 GW_B0_E1_37 // jump if edges required +s_and_b32 s56, 127, s[sgprSizeJ] // s56 = s[sgprSizeJ] % 128 +s_add_u32 s57, -0x1, s[sgprNumWorkGroups1] // +s_cmp_ge_u32 s[sgprWorkGroup1], s57 // wg1 >= nwg1-1 +s_cselect_b32 s56, s56, 0 // set rMT1 +s_cmpk_gt_u32 s56, 0x0 // rMT1 > 0 +s_cbranch_scc1 GW_B0_E1_37 // jump if edges required +GW_B0_E0_34: + +/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=1 */ +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +_v_add_lshl_u32 v134, v131, v128, 0x3 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=128, coord0Vgpr=128 + +/* rC *= alpha batchEements=[(0, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+0:vgprValuC+0+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+8:vgprValuC+8+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ + +/* rC *= alpha batchEements=[(1, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+2:vgprValuC+2+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+10:vgprValuC+10+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ + +/* rC *= alpha batchEements=[(2, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+4:vgprValuC+4+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+12:vgprValuC+12+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ + +/* rC *= alpha batchEements=[(3, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+6:vgprValuC+6+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+14:vgprValuC+14+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #4 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ + +/* rC *= alpha batchEements=[(4, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+16:vgprValuC+16+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+24:vgprValuC+24+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #5 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ + +/* rC *= alpha batchEements=[(5, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+18:vgprValuC+18+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+26:vgprValuC+26+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #6 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ + +/* rC *= alpha batchEements=[(6, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+20:vgprValuC+20+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+28:vgprValuC+28+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #7 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ + +/* rC *= alpha batchEements=[(7, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+22:vgprValuC+22+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+30:vgprValuC+30+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #8 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ + +/* rC *= alpha batchEements=[(8, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+32:vgprValuC+32+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+40:vgprValuC+40+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #9 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ + +/* rC *= alpha batchEements=[(9, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+34:vgprValuC+34+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+42:vgprValuC+42+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #10 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ + +/* rC *= alpha batchEements=[(10, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+36:vgprValuC+36+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+44:vgprValuC+44+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #11 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ + +/* rC *= alpha batchEements=[(11, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+38:vgprValuC+38+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+46:vgprValuC+46+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #12 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ + +/* rC *= alpha batchEements=[(12, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+48:vgprValuC+48+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+56:vgprValuC+56+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #13 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ + +/* rC *= alpha batchEements=[(13, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+50:vgprValuC+50+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+58:vgprValuC+58+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #14 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ + +/* rC *= alpha batchEements=[(14, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+52:vgprValuC+52+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+60:vgprValuC+60+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #15 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ + +/* rC *= alpha batchEements=[(15, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+54:vgprValuC+54+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+62:vgprValuC+62+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #16 (d1,d0,vc1,vc0) = */ +/* (16,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(16,0,0,0) */ + +/* rC *= alpha batchEements=[(16, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+64:vgprValuC+64+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+72:vgprValuC+72+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #17 (d1,d0,vc1,vc0) = */ +/* (17,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(17,0,0,0) */ + +/* rC *= alpha batchEements=[(17, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+66:vgprValuC+66+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+74:vgprValuC+74+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #18 (d1,d0,vc1,vc0) = */ +/* (18,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(18,0,0,0) */ + +/* rC *= alpha batchEements=[(18, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+68:vgprValuC+68+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+76:vgprValuC+76+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #19 (d1,d0,vc1,vc0) = */ +/* (19,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(19,0,0,0) */ + +/* rC *= alpha batchEements=[(19, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+70:vgprValuC+70+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+78:vgprValuC+78+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #20 (d1,d0,vc1,vc0) = */ +/* (20,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(20,0,0,0) */ + +/* rC *= alpha batchEements=[(20, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+80:vgprValuC+80+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+88:vgprValuC+88+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #21 (d1,d0,vc1,vc0) = */ +/* (21,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(21,0,0,0) */ + +/* rC *= alpha batchEements=[(21, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+82:vgprValuC+82+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+90:vgprValuC+90+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #22 (d1,d0,vc1,vc0) = */ +/* (22,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(22,0,0,0) */ + +/* rC *= alpha batchEements=[(22, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+84:vgprValuC+84+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+92:vgprValuC+92+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #23 (d1,d0,vc1,vc0) = */ +/* (23,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(23,0,0,0) */ + +/* rC *= alpha batchEements=[(23, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+86:vgprValuC+86+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+94:vgprValuC+94+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #24 (d1,d0,vc1,vc0) = */ +/* (24,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(24,0,0,0) */ + +/* rC *= alpha batchEements=[(24, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+96:vgprValuC+96+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+104:vgprValuC+104+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #25 (d1,d0,vc1,vc0) = */ +/* (25,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(25,0,0,0) */ + +/* rC *= alpha batchEements=[(25, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+98:vgprValuC+98+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+106:vgprValuC+106+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #26 (d1,d0,vc1,vc0) = */ +/* (26,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(26,0,0,0) */ + +/* rC *= alpha batchEements=[(26, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+100:vgprValuC+100+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+108:vgprValuC+108+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #27 (d1,d0,vc1,vc0) = */ +/* (27,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(27,0,0,0) */ + +/* rC *= alpha batchEements=[(27, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+102:vgprValuC+102+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+110:vgprValuC+110+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #28 (d1,d0,vc1,vc0) = */ +/* (28,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(28,0,0,0) */ + +/* rC *= alpha batchEements=[(28, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+112:vgprValuC+112+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+120:vgprValuC+120+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #29 (d1,d0,vc1,vc0) = */ +/* (29,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(29,0,0,0) */ + +/* rC *= alpha batchEements=[(29, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+114:vgprValuC+114+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+122:vgprValuC+122+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #30 (d1,d0,vc1,vc0) = */ +/* (30,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(30,0,0,0) */ + +/* rC *= alpha batchEements=[(30, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+116:vgprValuC+116+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+124:vgprValuC+124+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Batch #31 (d1,d0,vc1,vc0) = */ +/* (31,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(31,0,0,0) */ + +/* rC *= alpha batchEements=[(31, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+118:vgprValuC+118+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+126:vgprValuC+126+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[136:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_45 // jump to end +GW_B0_E1_37: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=1 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(0, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+0:vgprValuC+0+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (0,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(0, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+8:vgprValuC+8+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(1, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+2:vgprValuC+2+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (1,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(1,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(1, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+10:vgprValuC+10+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(2, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+4:vgprValuC+4+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #5 (d1,d0,vc1,vc0) = */ +/* (2,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(2,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(2, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+12:vgprValuC+12+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #6 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(3, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+6:vgprValuC+6+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #7 (d1,d0,vc1,vc0) = */ +/* (3,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(3,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(3, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+14:vgprValuC+14+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #8 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(4, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+16:vgprValuC+16+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #9 (d1,d0,vc1,vc0) = */ +/* (4,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(4, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+24:vgprValuC+24+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #10 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(5, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+18:vgprValuC+18+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #11 (d1,d0,vc1,vc0) = */ +/* (5,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(5,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(5, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+26:vgprValuC+26+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #12 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(6, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+20:vgprValuC+20+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #13 (d1,d0,vc1,vc0) = */ +/* (6,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(6,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(6, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+28:vgprValuC+28+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #14 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(7, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+22:vgprValuC+22+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #15 (d1,d0,vc1,vc0) = */ +/* (7,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(7,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(7, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+30:vgprValuC+30+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #16 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(8, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+32:vgprValuC+32+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #17 (d1,d0,vc1,vc0) = */ +/* (8,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(8, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+40:vgprValuC+40+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #18 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(9, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+34:vgprValuC+34+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #19 (d1,d0,vc1,vc0) = */ +/* (9,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(9,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(9, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+42:vgprValuC+42+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #20 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(10, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+36:vgprValuC+36+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #21 (d1,d0,vc1,vc0) = */ +/* (10,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(10,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(10, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+44:vgprValuC+44+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #22 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(11, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+38:vgprValuC+38+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #23 (d1,d0,vc1,vc0) = */ +/* (11,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(11,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(11, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+46:vgprValuC+46+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #24 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(12, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+48:vgprValuC+48+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #25 (d1,d0,vc1,vc0) = */ +/* (12,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(12, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+56:vgprValuC+56+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #26 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(13, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+50:vgprValuC+50+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #27 (d1,d0,vc1,vc0) = */ +/* (13,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(13,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(13, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+58:vgprValuC+58+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #28 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(14, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+52:vgprValuC+52+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #29 (d1,d0,vc1,vc0) = */ +/* (14,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(14,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(14, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+60:vgprValuC+60+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #30 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(15, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+54:vgprValuC+54+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #31 (d1,d0,vc1,vc0) = */ +/* (15,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(15,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(15, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+62:vgprValuC+62+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #32 (d1,d0,vc1,vc0) = */ +/* (16,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(16,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(16, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+64:vgprValuC+64+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #33 (d1,d0,vc1,vc0) = */ +/* (16,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(16,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(16, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+72:vgprValuC+72+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #34 (d1,d0,vc1,vc0) = */ +/* (17,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(17,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(17, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+66:vgprValuC+66+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #35 (d1,d0,vc1,vc0) = */ +/* (17,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(17,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(17, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+74:vgprValuC+74+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #36 (d1,d0,vc1,vc0) = */ +/* (18,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(18,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(18, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+68:vgprValuC+68+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #37 (d1,d0,vc1,vc0) = */ +/* (18,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(18,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(18, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+76:vgprValuC+76+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #38 (d1,d0,vc1,vc0) = */ +/* (19,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(19,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(19, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+70:vgprValuC+70+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #39 (d1,d0,vc1,vc0) = */ +/* (19,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(19,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(19, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+78:vgprValuC+78+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #40 (d1,d0,vc1,vc0) = */ +/* (20,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(20,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(20, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+80:vgprValuC+80+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #41 (d1,d0,vc1,vc0) = */ +/* (20,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(20,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(20, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+88:vgprValuC+88+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #42 (d1,d0,vc1,vc0) = */ +/* (21,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(21,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(21, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+82:vgprValuC+82+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #43 (d1,d0,vc1,vc0) = */ +/* (21,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(21,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(21, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+90:vgprValuC+90+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #44 (d1,d0,vc1,vc0) = */ +/* (22,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(22,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(22, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+84:vgprValuC+84+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #45 (d1,d0,vc1,vc0) = */ +/* (22,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(22,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(22, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+92:vgprValuC+92+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #46 (d1,d0,vc1,vc0) = */ +/* (23,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(23,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(23, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+86:vgprValuC+86+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #47 (d1,d0,vc1,vc0) = */ +/* (23,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(23,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(23, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+94:vgprValuC+94+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #48 (d1,d0,vc1,vc0) = */ +/* (24,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(24,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(24, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+96:vgprValuC+96+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #49 (d1,d0,vc1,vc0) = */ +/* (24,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(24,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(24, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+104:vgprValuC+104+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #50 (d1,d0,vc1,vc0) = */ +/* (25,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(25,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(25, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+98:vgprValuC+98+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #51 (d1,d0,vc1,vc0) = */ +/* (25,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(25,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(25, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+106:vgprValuC+106+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #52 (d1,d0,vc1,vc0) = */ +/* (26,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(26,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(26, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+100:vgprValuC+100+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #53 (d1,d0,vc1,vc0) = */ +/* (26,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(26,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(26, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+108:vgprValuC+108+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #54 (d1,d0,vc1,vc0) = */ +/* (27,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(27,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(27, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+102:vgprValuC+102+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #55 (d1,d0,vc1,vc0) = */ +/* (27,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(27,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(27, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+110:vgprValuC+110+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #56 (d1,d0,vc1,vc0) = */ +/* (28,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(28,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(28, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+112:vgprValuC+112+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #57 (d1,d0,vc1,vc0) = */ +/* (28,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(28,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(28, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+120:vgprValuC+120+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #58 (d1,d0,vc1,vc0) = */ +/* (29,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(29,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(29, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+114:vgprValuC+114+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #59 (d1,d0,vc1,vc0) = */ +/* (29,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(29,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(29, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+122:vgprValuC+122+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #60 (d1,d0,vc1,vc0) = */ +/* (30,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(30,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(30, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+116:vgprValuC+116+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #61 (d1,d0,vc1,vc0) = */ +/* (30,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(30,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(30, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+124:vgprValuC+124+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #62 (d1,d0,vc1,vc0) = */ +/* (31,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(31,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(31, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+118:vgprValuC+118+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Edge Batch #63 (d1,d0,vc1,vc0) = */ +/* (31,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(31,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset + +/* rC *= alpha batchEements=[(31, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+136:vgprValuC+136+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+126:vgprValuC+126+1] // Multiply MI out reg with alpha + +/* apply mask, calc new C and issue writes */ +buffer_store_dwordx2 v[136:137], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_45 // jump to end +GW_Beta_46: +s_and_b32 s56, 127, s[sgprSizeI] // s56 = s[sgprSizeI] % 128 +s_add_u32 s57, -0x1, s[sgprNumWorkGroups0] // +s_cmp_ge_u32 s[sgprWorkGroup0], s57 // wg0 >= nwg0-1 ? +s_cselect_b32 s56, s56, 0 // set rMT0 +s_cmpk_gt_u32 s56, 0x0 // rMT0 > 0 +s_cbranch_scc1 GW_B1_E1_44 // jump if edges required +s_and_b32 s56, 127, s[sgprSizeJ] // s56 = s[sgprSizeJ] % 128 +s_add_u32 s57, -0x1, s[sgprNumWorkGroups1] // +s_cmp_ge_u32 s[sgprWorkGroup1], s57 // wg1 >= nwg1-1 +s_cselect_b32 s56, s56, 0 // set rMT1 +s_cmpk_gt_u32 s56, 0x0 // rMT1 > 0 +s_cbranch_scc1 GW_B1_E1_44 // jump if edges required +GW_B1_E0_41: + +/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=1 */ +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(0, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+0:vgprValuC+0+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+8:vgprValuC+8+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +_v_add_lshl_u32 v135, v130, v128, 0x3 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=128, coord0Vgpr=128 +_v_add_lshl_u32 v134, v131, v128, 0x3 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=128, coord0Vgpr=128 +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #1 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(1, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+2:vgprValuC+2+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+10:vgprValuC+10+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #2 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(2, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+4:vgprValuC+4+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+12:vgprValuC+12+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #3 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(3, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+6:vgprValuC+6+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+14:vgprValuC+14+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #4 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(4, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+16:vgprValuC+16+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+24:vgprValuC+24+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #5 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(5, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+18:vgprValuC+18+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+26:vgprValuC+26+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #6 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(6, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+20:vgprValuC+20+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+28:vgprValuC+28+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #7 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(7, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+22:vgprValuC+22+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+30:vgprValuC+30+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #8 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(8, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+32:vgprValuC+32+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+40:vgprValuC+40+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #9 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(9, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+34:vgprValuC+34+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+42:vgprValuC+42+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #10 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(10, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+36:vgprValuC+36+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+44:vgprValuC+44+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #11 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(11, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+38:vgprValuC+38+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+46:vgprValuC+46+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #12 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(12, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+48:vgprValuC+48+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+56:vgprValuC+56+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #13 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(13, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+50:vgprValuC+50+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+58:vgprValuC+58+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #14 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(14, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+52:vgprValuC+52+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+60:vgprValuC+60+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #15 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(15, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+54:vgprValuC+54+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+62:vgprValuC+62+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #16 (d1,d0,vc1,vc0) = */ +/* (16,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(16, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+64:vgprValuC+64+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+72:vgprValuC+72+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(16,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #17 (d1,d0,vc1,vc0) = */ +/* (17,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(17, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+66:vgprValuC+66+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+74:vgprValuC+74+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(17,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #18 (d1,d0,vc1,vc0) = */ +/* (18,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(18, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+68:vgprValuC+68+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+76:vgprValuC+76+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(18,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #19 (d1,d0,vc1,vc0) = */ +/* (19,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(19, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+70:vgprValuC+70+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+78:vgprValuC+78+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(19,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #20 (d1,d0,vc1,vc0) = */ +/* (20,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(20, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+80:vgprValuC+80+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+88:vgprValuC+88+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(20,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #21 (d1,d0,vc1,vc0) = */ +/* (21,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(21, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+82:vgprValuC+82+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+90:vgprValuC+90+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(21,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #22 (d1,d0,vc1,vc0) = */ +/* (22,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(22, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+84:vgprValuC+84+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+92:vgprValuC+92+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(22,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #23 (d1,d0,vc1,vc0) = */ +/* (23,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(23, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+86:vgprValuC+86+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+94:vgprValuC+94+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(23,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #24 (d1,d0,vc1,vc0) = */ +/* (24,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(24, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+96:vgprValuC+96+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+104:vgprValuC+104+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(24,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #25 (d1,d0,vc1,vc0) = */ +/* (25,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(25, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+98:vgprValuC+98+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+106:vgprValuC+106+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(25,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #26 (d1,d0,vc1,vc0) = */ +/* (26,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(26, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+100:vgprValuC+100+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+108:vgprValuC+108+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(26,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #27 (d1,d0,vc1,vc0) = */ +/* (27,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(27, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+102:vgprValuC+102+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+110:vgprValuC+110+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(27,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #28 (d1,d0,vc1,vc0) = */ +/* (28,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(28, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+112:vgprValuC+112+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+120:vgprValuC+120+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(28,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #29 (d1,d0,vc1,vc0) = */ +/* (29,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(29, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+114:vgprValuC+114+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+122:vgprValuC+122+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(29,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #30 (d1,d0,vc1,vc0) = */ +/* (30,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(30, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+116:vgprValuC+116+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+124:vgprValuC+124+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(30,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Batch #31 (d1,d0,vc1,vc0) = */ +/* (31,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(31, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+118:vgprValuC+118+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+126:vgprValuC+126+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(31,0,0,0) */ +s_mul_i32 s56, s[sgprStrideC1J], 32 // scale StrideC *= numRows(4) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +s_mul_i32 s56, s[sgprStrideD1J], 32 // scale StrideD *= numRows(4) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s56 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_45 // jump to end +GW_B1_E1_44: + +// TODO in Generator +// wider store if M % 2 == 0 +s_and_b32 s55, 0x1, s[sgprSizeI] +s_cbranch_scc0 GW_B1_E1_VW2 // done shifting + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=1 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(0, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+0:vgprValuC+0+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (0,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(0, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+8:vgprValuC+8+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(0,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(1, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+2:vgprValuC+2+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (1,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(1, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+10:vgprValuC+10+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(1,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(2, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+4:vgprValuC+4+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #5 (d1,d0,vc1,vc0) = */ +/* (2,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(2, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+12:vgprValuC+12+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(2,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #6 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(3, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+6:vgprValuC+6+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #7 (d1,d0,vc1,vc0) = */ +/* (3,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(3, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+14:vgprValuC+14+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(3,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #8 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(4, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+16:vgprValuC+16+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #9 (d1,d0,vc1,vc0) = */ +/* (4,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(4, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+24:vgprValuC+24+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(4,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #10 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(5, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+18:vgprValuC+18+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #11 (d1,d0,vc1,vc0) = */ +/* (5,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(5, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+26:vgprValuC+26+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(5,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #12 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(6, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+20:vgprValuC+20+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #13 (d1,d0,vc1,vc0) = */ +/* (6,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(6, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+28:vgprValuC+28+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(6,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #14 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(7, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+22:vgprValuC+22+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #15 (d1,d0,vc1,vc0) = */ +/* (7,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(7, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+30:vgprValuC+30+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(7,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #16 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(8, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+32:vgprValuC+32+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #17 (d1,d0,vc1,vc0) = */ +/* (8,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(8, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+40:vgprValuC+40+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(8,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #18 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(9, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+34:vgprValuC+34+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #19 (d1,d0,vc1,vc0) = */ +/* (9,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(9, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+42:vgprValuC+42+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(9,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #20 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(10, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+36:vgprValuC+36+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #21 (d1,d0,vc1,vc0) = */ +/* (10,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(10, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+44:vgprValuC+44+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(10,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #22 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(11, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+38:vgprValuC+38+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #23 (d1,d0,vc1,vc0) = */ +/* (11,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(11, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+46:vgprValuC+46+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(11,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #24 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(12, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+48:vgprValuC+48+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #25 (d1,d0,vc1,vc0) = */ +/* (12,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(12, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+56:vgprValuC+56+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(12,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #26 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(13, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+50:vgprValuC+50+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #27 (d1,d0,vc1,vc0) = */ +/* (13,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(13, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+58:vgprValuC+58+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(13,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #28 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(14, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+52:vgprValuC+52+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #29 (d1,d0,vc1,vc0) = */ +/* (14,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(14, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+60:vgprValuC+60+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(14,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #30 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(15, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+54:vgprValuC+54+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #31 (d1,d0,vc1,vc0) = */ +/* (15,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(15, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+62:vgprValuC+62+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(15,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #32 (d1,d0,vc1,vc0) = */ +/* (16,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(16, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+64:vgprValuC+64+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(16,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #33 (d1,d0,vc1,vc0) = */ +/* (16,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(16, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+72:vgprValuC+72+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(16,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #34 (d1,d0,vc1,vc0) = */ +/* (17,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(17, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+66:vgprValuC+66+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(17,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #35 (d1,d0,vc1,vc0) = */ +/* (17,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(17, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+74:vgprValuC+74+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(17,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #36 (d1,d0,vc1,vc0) = */ +/* (18,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(18, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+68:vgprValuC+68+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(18,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #37 (d1,d0,vc1,vc0) = */ +/* (18,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(18, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+76:vgprValuC+76+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(18,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #38 (d1,d0,vc1,vc0) = */ +/* (19,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(19, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+70:vgprValuC+70+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(19,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #39 (d1,d0,vc1,vc0) = */ +/* (19,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(19, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+78:vgprValuC+78+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(19,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #40 (d1,d0,vc1,vc0) = */ +/* (20,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(20, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+80:vgprValuC+80+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(20,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #41 (d1,d0,vc1,vc0) = */ +/* (20,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(20, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+88:vgprValuC+88+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(20,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #42 (d1,d0,vc1,vc0) = */ +/* (21,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(21, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+82:vgprValuC+82+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(21,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #43 (d1,d0,vc1,vc0) = */ +/* (21,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(21, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+90:vgprValuC+90+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(21,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #44 (d1,d0,vc1,vc0) = */ +/* (22,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(22, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+84:vgprValuC+84+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(22,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #45 (d1,d0,vc1,vc0) = */ +/* (22,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(22, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+92:vgprValuC+92+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(22,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #46 (d1,d0,vc1,vc0) = */ +/* (23,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(23, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+86:vgprValuC+86+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(23,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #47 (d1,d0,vc1,vc0) = */ +/* (23,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(23, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+94:vgprValuC+94+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(23,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #48 (d1,d0,vc1,vc0) = */ +/* (24,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(24, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+96:vgprValuC+96+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(24,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #49 (d1,d0,vc1,vc0) = */ +/* (24,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(24, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+104:vgprValuC+104+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(24,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #50 (d1,d0,vc1,vc0) = */ +/* (25,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(25, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+98:vgprValuC+98+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(25,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #51 (d1,d0,vc1,vc0) = */ +/* (25,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(25, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+106:vgprValuC+106+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(25,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #52 (d1,d0,vc1,vc0) = */ +/* (26,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(26, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+100:vgprValuC+100+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(26,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #53 (d1,d0,vc1,vc0) = */ +/* (26,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(26, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+108:vgprValuC+108+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(26,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #54 (d1,d0,vc1,vc0) = */ +/* (27,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(27, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+102:vgprValuC+102+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(27,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #55 (d1,d0,vc1,vc0) = */ +/* (27,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(27, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+110:vgprValuC+110+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(27,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #56 (d1,d0,vc1,vc0) = */ +/* (28,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(28, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+112:vgprValuC+112+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(28,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #57 (d1,d0,vc1,vc0) = */ +/* (28,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(28, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+120:vgprValuC+120+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(28,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #58 (d1,d0,vc1,vc0) = */ +/* (29,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(29, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+114:vgprValuC+114+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(29,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #59 (d1,d0,vc1,vc0) = */ +/* (29,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(29, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+122:vgprValuC+122+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(29,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #60 (d1,d0,vc1,vc0) = */ +/* (30,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(30, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+116:vgprValuC+116+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(30,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #61 (d1,d0,vc1,vc0) = */ +/* (30,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(30, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+124:vgprValuC+124+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(30,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #62 (d1,d0,vc1,vc0) = */ +/* (31,0,0,0:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(31, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+118:vgprValuC+118+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(31,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #63 (d1,d0,vc1,vc0) = */ +/* (31,0,0,1:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(31, 0, 0, 1)] */ +v_mul_f64 v[vgprValuC+138:vgprValuC+138+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+126:vgprValuC+126+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(31,0,0,1) */ +_v_add_co_u32 v132, vcc, v128, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[56:57], v132, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v132, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx2 v[136:137], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+138:vgprValuC+138+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+138:vgprValuC+138+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx2 v[138:139], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_45 // jump to end + +GW_B1_E1_VW2: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=1 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(0, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+0:vgprValuC+0+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+8:vgprValuC+8+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(1, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+2:vgprValuC+2+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+10:vgprValuC+10+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(2, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+4:vgprValuC+4+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+12:vgprValuC+12+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(3, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+6:vgprValuC+6+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+14:vgprValuC+14+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(4, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+16:vgprValuC+16+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+24:vgprValuC+24+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #5 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(5, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+18:vgprValuC+18+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+26:vgprValuC+26+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #6 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(6, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+20:vgprValuC+20+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+28:vgprValuC+28+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #7 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(7, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+22:vgprValuC+22+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+30:vgprValuC+30+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #8 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(8, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+32:vgprValuC+32+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+40:vgprValuC+40+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #9 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(9, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+34:vgprValuC+34+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+42:vgprValuC+42+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #10 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(10, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+36:vgprValuC+36+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+44:vgprValuC+44+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #11 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(11, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+38:vgprValuC+38+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+46:vgprValuC+46+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #12 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(12, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+48:vgprValuC+48+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+56:vgprValuC+56+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #13 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(13, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+50:vgprValuC+50+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+58:vgprValuC+58+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #14 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(14, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+52:vgprValuC+52+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+60:vgprValuC+60+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #15 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(15, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+54:vgprValuC+54+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+62:vgprValuC+62+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #16 (d1,d0,vc1,vc0) = */ +/* (16,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(16, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+64:vgprValuC+64+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+72:vgprValuC+72+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(16,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #17 (d1,d0,vc1,vc0) = */ +/* (17,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(17, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+66:vgprValuC+66+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+74:vgprValuC+74+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(17,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #18 (d1,d0,vc1,vc0) = */ +/* (18,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(18, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+68:vgprValuC+68+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+76:vgprValuC+76+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(18,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #19 (d1,d0,vc1,vc0) = */ +/* (19,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(19, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+70:vgprValuC+70+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+78:vgprValuC+78+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(19,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #20 (d1,d0,vc1,vc0) = */ +/* (20,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(20, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+80:vgprValuC+80+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+88:vgprValuC+88+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(20,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #21 (d1,d0,vc1,vc0) = */ +/* (21,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(21, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+82:vgprValuC+82+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+90:vgprValuC+90+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(21,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #22 (d1,d0,vc1,vc0) = */ +/* (22,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(22, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+84:vgprValuC+84+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+92:vgprValuC+92+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(22,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #23 (d1,d0,vc1,vc0) = */ +/* (23,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(23, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+86:vgprValuC+86+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+94:vgprValuC+94+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(23,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #24 (d1,d0,vc1,vc0) = */ +/* (24,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(24, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+96:vgprValuC+96+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+104:vgprValuC+104+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(24,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #25 (d1,d0,vc1,vc0) = */ +/* (25,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(25, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+98:vgprValuC+98+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+106:vgprValuC+106+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(25,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #26 (d1,d0,vc1,vc0) = */ +/* (26,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(26, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+100:vgprValuC+100+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+108:vgprValuC+108+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(26,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #27 (d1,d0,vc1,vc0) = */ +/* (27,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(27, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+102:vgprValuC+102+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+110:vgprValuC+110+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(27,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #28 (d1,d0,vc1,vc0) = */ +/* (28,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(28, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+112:vgprValuC+112+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+120:vgprValuC+120+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(28,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #29 (d1,d0,vc1,vc0) = */ +/* (29,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(29, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+114:vgprValuC+114+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+122:vgprValuC+122+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(29,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #30 (d1,d0,vc1,vc0) = */ +/* (30,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(30, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+116:vgprValuC+116+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+124:vgprValuC+124+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(30,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 */ +s_sleep 5 // optimization: sync and wait +s_barrier + +/******************************************/ +/* Global Write Beta Edge Batch #31 (d1,d0,vc1,vc0) = */ +/* (31,0,0,0:vw2) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ + +/* rC *= alpha batchEements=[(31, 0, 0, 0)] */ +v_mul_f64 v[vgprValuC+140:vgprValuC+140+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+118:vgprValuC+118+1] // Multiply MI out reg with alpha +v_mul_f64 v[vgprValuC+142:vgprValuC+142+1], s[sgprAlpha:sgprAlpha+1], v[vgprValuC+126:vgprValuC+126+1] // Multiply MI out reg with alpha +/* (d1,vc1,d0,vc0)=(31,0,0,0) */ +_v_add_co_u32 v129, vcc, v129, 4 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s56, s[sgprStrideC1J], 4 // scale stride +_v_add_u32 v130, v130, s56 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s56, s[sgprStrideD1J], 4 // scale stride +_v_add_u32 v131, v131, s56 // Move coutRowPtr to next row +v_cmp_lt_u32 s[56:57], v128, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[60:61], v129, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[60:61], s[56:57], s[60:61] // in0 && in1 +_v_add_lshl_u32 v135, v130, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v135, -1, v135, s[60:61] // LDC clip if OOB. offset +_v_add_lshl_u32 v134, v131, v128, 0x3 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v134, -1, v134, s[60:61] // LDD clip if OOB. offset +buffer_load_dwordx4 v[136:139], v135, s[sgprSrdC:sgprSrdC+3], 0, offen offset:0, glc slc // load C for beta calc +s_sleep 5 // optimization: sync and wait +s_barrier +s_waitcnt vmcnt(0) // wait C + +/* apply mask, calc new C and issue writes */ +v_fma_f64 v[vgprValuC+140:vgprValuC+140+1], v[136:137], s[sgprBeta:sgprBeta+1], v[vgprValuC+140:vgprValuC+140+1] // finalSum = sum*alpha + C*beta +v_fma_f64 v[vgprValuC+142:vgprValuC+142+1], v[138:139], s[sgprBeta:sgprBeta+1], v[vgprValuC+142:vgprValuC+142+1] // finalSum = sum*alpha + C*beta +buffer_store_dwordx4 v[140:143], v134, s[sgprSrdD:sgprSrdD+3], 0, offen, offset:0, glc slc // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_45 // jump to end +label_GW_End_45: + +label_0047: /// KernelEnd +s_endpgm // Kernel End + + diff --git a/tensilelite/Tensile/CustomKernels/MDA2_Kernel1.s b/tensilelite/Tensile/CustomKernels/MDA2_Kernel1.s new file mode 100644 index 0000000000..ae482b76de --- /dev/null +++ b/tensilelite/Tensile/CustomKernels/MDA2_Kernel1.s @@ -0,0 +1,1629 @@ +/***********************************************************************************/ + /* */ + /* Copyright 2020-2021 Advanced Micro Devices, Inc. */ + /* */ + /* Permission is hereby granted, free of charge, to any person obtaining a copy */ + /* of this software and associated documentation files (the "Software"), to deal */ + /* in the Software without restriction, including without limitation the rights */ + /* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell */ + /* copies of the Software, and to permit persons to whom the Software is */ + /* furnished to do so, subject to the following conditions: */ + /* */ + /* The above copyright notice and this permission notice shall be included in */ + /* all copies or substantial portions of the Software. */ + /* */ + /* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR */ + /* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, */ + /* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE */ + /* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER */ + /* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, */ + /* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE */ + /* SOFTWARE. */ + /* */ + /**********************************************************************************/ + +/******************************************/ +/* Function Prefix */ +/******************************************/ + + + +/******************************************/ +/* Begin Kernel */ +/******************************************/ + +.amdgcn_target "amdgcn-amd-amdhsa--gfx908" +.text +.protected Translated1 +.globl Translated1 +.p2align 8 +.type Translated1,@function +.section .rodata,#alloc +.p2align 6 +.amdhsa_kernel Translated1 + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_next_free_vgpr 108 // vgprs + .amdhsa_next_free_sgpr 98 // sgprs + .amdhsa_group_segment_fixed_size 60000 // lds bytes + .amdhsa_private_segment_fixed_size 0 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_system_sgpr_workgroup_id_y 1 + .amdhsa_system_sgpr_workgroup_id_z 1 + .amdhsa_system_vgpr_workitem_id 0 +.end_amdhsa_kernel +.text + +/******************************************/ +/* Optimizations and Config: */ +/******************************************/ +/* ThreadTile= 4 x 4 */ +/* SubGroup= 16 x 32 */ +/* VectorWidth=4 */ +/* GlobalLoadVectorWidthA=4, GlobalLoadVectorWidthB=4 */ +/* DirectToLdsA=False */ +/* DirectToLdsB=False */ +/* UseSgprForGRO=1 */ +.amdgpu_metadata +--- +custom.config: + ProblemType: + OperationType: GEMM + DataType: s + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + 1LDSBuffer: 0 + BufferLoad: True + DepthU: 32 + DirectToLds: False + ExpandPointerSwap: True + FractionalLoad: 0 + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + InnerUnroll: 1 + KernelLanguage: A + LdsBlockSizePerPad: 0 + LdsPadA: 0 + LdsPadB: 0 + LocalReadVectorWidth: 4 + MagicDivAlg: 2 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + OptNoLoadLoop: 1 + PrefetchGlobalRead: 1 + PrefetchLocalRead: 1 + ScheduleIterAlg: 1 + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + StoreRemapVectorWidth: 0 + StoreVectorWidth: 4 + SuppressNoLoadLoop: False + ThreadTile: [ 4, 4 ] + TransposeLDS: 0 + UseSgprForGRO: 1 + VectorStore: 1 + VectorWidth: 4 + WaveSeparateGlobalReadA: 0 + WaveSeparateGlobalReadB: 0 + WorkGroup: [ 16, 32, 1 ] + WorkGroupMapping: 8 + ISA: [9, 0, 8] + AssertFree0ElementMultiple: 8 + AssertFree1ElementMultiple: 1 + AssertSummationElementMultiple: 8 + +amdhsa.version: + - 1 + - 0 +amdhsa.kernels: + - .name: Translated1 + .symbol: 'Translated1.kd' + .language: OpenCL C + .language_version: + - 2 + - 0 + .args: + - .name: sizeC + .size: 8 + .offset: 0 + .value_kind: by_value + .value_type: u64 + - .name: sizeA + .size: 8 + .offset: 8 + .value_kind: by_value + .value_type: u64 + - .name: sizeB + .size: 8 + .offset: 16 + .value_kind: by_value + .value_type: u64 + - .name: D + .size: 8 + .offset: 24 + .value_kind: global_buffer + .value_type: f32 + .address_space: generic + - .name: C + .size: 8 + .offset: 32 + .value_kind: global_buffer + .value_type: f32 + .address_space: generic + - .name: A + .size: 8 + .offset: 40 + .value_kind: global_buffer + .value_type: f32 + .address_space: generic + - .name: B + .size: 8 + .offset: 48 + .value_kind: global_buffer + .value_type: f32 + .address_space: generic + - .name: alpha + .size: 4 + .offset: 56 + .value_kind: by_value + .value_type: f32 + - .name: beta + .size: 4 + .offset: 60 + .value_kind: by_value + .value_type: f32 + - .name: strideD0 + .size: 4 + .offset: 64 + .value_kind: by_value + .value_type: u32 + - .name: strideD1 + .size: 4 + .offset: 68 + .value_kind: by_value + .value_type: u32 + - .name: strideC0 + .size: 4 + .offset: 72 + .value_kind: by_value + .value_type: u32 + - .name: strideC1 + .size: 4 + .offset: 76 + .value_kind: by_value + .value_type: u32 + - .name: strideA0 + .size: 4 + .offset: 80 + .value_kind: by_value + .value_type: u32 + - .name: strideA1 + .size: 4 + .offset: 84 + .value_kind: by_value + .value_type: u32 + - .name: strideB0 + .size: 4 + .offset: 88 + .value_kind: by_value + .value_type: u32 + - .name: strideB1 + .size: 4 + .offset: 92 + .value_kind: by_value + .value_type: u32 + - .name: SizesFree0 + .size: 4 + .offset: 96 + .value_kind: by_value + .value_type: u32 + - .name: SizesFree1 + .size: 4 + .offset: 100 + .value_kind: by_value + .value_type: u32 + - .name: SizesFree2 + .size: 4 + .offset: 104 + .value_kind: by_value + .value_type: u32 + - .name: SizesSum0 + .size: 4 + .offset: 108 + .value_kind: by_value + .value_type: u32 + - .name: OrigStaggerUIter + .size: 4 + .offset: 112 + .value_kind: by_value + .value_type: i32 + - .name: NumWorkGroups0 + .size: 4 + .offset: 116 + .value_kind: by_value + .value_type: u32 + - .name: NumWorkGroups1 + .size: 4 + .offset: 120 + .value_kind: by_value + .value_type: u32 + - .name: MagicNumberProblemNumGroupTiles0 + .size: 4 + .offset: 124 + .value_kind: by_value + .value_type: u32 + - .name: GridNumWorkGroups0 + .size: 4 + .offset: 128 + .value_kind: by_value + .value_type: u32 + - .name: NumFullBlocks + .size: 4 + .offset: 132 + .value_kind: by_value + .value_type: u32 + - .name: WgmRemainder1 + .size: 4 + .offset: 136 + .value_kind: by_value + .value_type: u32 + - .name: MagicNumberWgmRemainder1 + .size: 4 + .offset: 140 + .value_kind: by_value + .value_type: u32 + - .name: padding + .size: 4 + .offset: 144 + .value_kind: by_value + .value_type: u32 + .group_segment_fixed_size: 60000 + .kernarg_segment_align: 8 + .kernarg_segment_size: 152 + .max_flat_workgroup_size: 512 + .private_segment_fixed_size: 0 + .sgpr_count: 98 + .sgpr_spill_count: 0 + .vgpr_count: 108 + .vgpr_spill_count: 0 + .wavefront_size: 64 +... +.end_amdgpu_metadata +Translated1: + +/******************************************/ +/* Asm syntax workarounds */ +/******************************************/ +.macro _v_add_co_u32 dst, cc, src0, src1, dpp= + v_add_co_u32 \dst, \cc, \src0, \src1 \dpp +.endm + +.macro _v_add_u32 dst, src0, src1, dpp= + v_add_u32 \dst, \src0, \src1 \dpp +.endm + +.macro _v_sub_co_u32 dst, cc, src0, src1, dpp= + v_sub_co_u32 \dst, \cc, \src0, \src1 \dpp +.endm + +.macro _v_sub_u32 dst, src0, src1, dpp= + v_sub_u32 \dst, \src0, \src1 \dpp +.endm + +.macro _v_addc_co_u32 dst, ccOut, src0, ccIn, src1, dpp= + v_addc_co_u32 \dst, \ccOut, \src0, \ccIn, \src1 \dpp +.endm + +.macro _v_add_lshl_u32 dst, src0, src1, shiftCnt + v_add_lshl_u32 \dst, \src0, \src1, \shiftCnt +.endm + +.macro _v_lshl_add_u32 dst, src0, src1, shiftCnt + v_lshl_add_u32 \dst, \src0, \src1, \shiftCnt +.endm + +/******************************************/ +/* Magic div and mod functions */ +/******************************************/ +.macro V_MAGIC_DIV dstIdx, dividend, magicNumber, magicShift + v_mul_hi_u32 v[\dstIdx+1], \dividend, \magicNumber + v_mul_lo_u32 v[\dstIdx+0], \dividend, \magicNumber + v_lshrrev_b64 v[\dstIdx:\dstIdx+1], \magicShift, v[\dstIdx:\dstIdx+1] +.endm + +/******************************************/ +/* VGPR Assignments */ +/******************************************/ +.set vgprValuC, 0 +/* ValuA/B Xn=PLR buffer idx, In=InnerUnroll idx */ +.set vgprValuA_X0_I0, 32 +.set vgprValuA_X1_I0, 36 +.set vgprG2LA, 40 +.set vgprValuB_X0_I0, 48 +.set vgprValuB_X1_I0, 56 +.set vgprG2LB, 64 +.set vgprLocalWriteAddrA, 80 +.set vgprLocalWriteAddrB, 81 +.set vgprGlobalReadOffsetA, 82 +.set vgprGlobalReadOffsetB, 83 +.set vgprLocalReadAddrA, 84 +.set vgprLocalReadAddrB, 85 +.set vgprSerial, 86 +/* Num VGPR=87 */ + +/******************************************/ +/* SGPR Assignments */ +/******************************************/ +.set sgprKernArgAddress, 0 +.set sgprWorkGroup0, 2 +.set sgprWorkGroup1, 3 +.set sgprWorkGroup2, 4 +.set sgprNumWorkGroups0, 5 +.set sgprNumWorkGroups1, 6 +.set sgprSrdA, 8 +.set sgprSrdB, 12 +.set sgprSrdD, 16 +.set sgprSrdC, 20 +.set sgprTensor2dSizeC, 24 +.set sgprTensor2dSizeA, 26 +.set sgprTensor2dSizeB, 28 +.set sgprSaveExecMask, 30 +.set sgprAddressD, 32 +.set sgprAddressC, 34 +.set sgprStridesD, 36 +.set sgprStridesC, 38 +.set sgprAlpha, 40 +.set sgprBeta, 41 +.set sgprSizesFree, 42 +.set sgprSizesSum, 45 +.set sgprLoopCounters, 46 +.set sgprOrigLoopCounter, 47 +.set sgprStridesA, 48 +.set sgprStridesB, 50 +.set sgprAddressA, 52 +.set sgprAddressB, 54 +.set sgprShadowLimitA, 56 +.set sgprShadowLimitB, 58 +.set sgprOrigStaggerUIter, 60 +.set sgprStaggerUIter, 61 +.set sgprWrapUA, 62 +.set sgprWrapUB, 64 +.set sgprNumFullBlocks, 66 +.set sgprWgmRemainder1, 67 +.set sgprMagicNumberWgmRemainder1, 68 +.set sgprGlobalReadIncsA, 69 +.set sgprGlobalReadIncsB, 70 +.set sgprScalarGlobalReadOffsetA, 71 +.set sgprScalarGlobalReadOffsetB, 72 +/* max SGPR=98 */ + +/* Size Assignments */ +.set sgprSizeD0I, sgprSizesFree+0 +.set sgprSizeD1J, sgprSizesFree+1 +.set sgprSizeDK, sgprSizesFree+2 +.set sgprSizeC0I, sgprSizesFree+0 +.set sgprSizeC1J, sgprSizesFree+1 +.set sgprSizeCK, sgprSizesFree+2 +.set sgprSizeAL, sgprSizesSum+0 +.set sgprSizeA0I, sgprSizesFree+0 +.set sgprSizeAK, sgprSizesFree+2 +.set sgprSizeBL, sgprSizesSum+0 +.set sgprSizeB1J, sgprSizesFree+1 +.set sgprSizeBK, sgprSizesFree+2 + +/* Stride Assignments */ +.set constStrideD0I, 1 +.set sgprStrideD1J, sgprStridesD+0 +.set sgprStrideDK, sgprStridesD+1 +.set constStrideC0I, 1 +.set sgprStrideC1J, sgprStridesC+0 +.set sgprStrideCK, sgprStridesC+1 +.set constStrideAL, 1 +.set sgprStrideA0I, sgprStridesA+0 +.set sgprStrideAK, sgprStridesA+1 +.set constStrideBL, 1 +.set sgprStrideB1J, sgprStridesB+0 +.set sgprStrideBK, sgprStridesB+1 + +.set DepthU, 32 +/* Number of elements to shift-left SRD */ +.set SrdShiftLeftA, 4 +.set SrdShiftLeftB, 4 +/* 2GB limit - set offsets to -1 to exceed this and clamp */ +.set BufferLimit, 0x80000000 +/* Bits 127:96 of SRD. Set DataFormat = 32 bit */ +.set Srd127_96, 0x0020000 +.set BufferOOB, 0x80000000 + + + + + +.long 0xC00A0D00, 0x00000028 +.long 0xC00A0C00, 0x00000050 +.long 0xC00A0600, 0x00000008 +.long 0xC0020B40, 0x0000006C +.long 0xBEFC00FF, 0x00006000 +.long 0x7EC80300 +.long 0x26CA00BF +.long 0x2004C886 +.long 0xB8D0F804 +.long 0xD1130004, 0x0000A0B0 +.long 0x20CC0884 +.long 0x7EA40566 +.long 0xD1130067, 0x0000A08F +.long 0x7EA20567 +.long 0xBF068151 +.long 0xBF840212 +.long 0xBF8CC07F +.long 0xBE880034 +.long 0xBE890035 +.long 0xBE8B00FF, 0x00020000 +.long 0x80B85418 +.long 0x80B95518 +.long 0x8EB88238 +.long 0x80388438 +.long 0x82398039 +.long 0xBF068039 +.long 0x850AFF38, 0x80000000 +.long 0xBE8A00FF, 0x80000000 +.long 0x9254C030 +.long 0x92545402 +.long 0x8E558452 +.long 0x92533055 +.long 0x92553104 +.long 0x81545354 +.long 0x80545554 +.long 0x2000CA85 +.long 0xD2850004, 0x00020030 +.long 0x2602CA9F +.long 0x32A40304 +.long 0x68A4A454 +.long 0x24A4A482 +.long 0x8E478330 +.long 0x80C7FF47, 0x00000108 +.long 0x68A6A447 +.long 0x68A8A647 +.long 0x68AAA847 +.long 0x68ACAA47 +.long 0x68AEAC47 +.long 0x68B0AE47 +.long 0x68B2B047 +.long 0xBECC00FF, 0x00000840 +.long 0x924C4C52 +.long 0xBE8C0036 +.long 0xBE8D0037 +.long 0xBE8F00FF, 0x00020000 +.long 0x80BA541A +.long 0x80BB551A +.long 0x8EBA823A +.long 0x803A843A +.long 0x823B803B +.long 0xBF06803B +.long 0x850EFF3A, 0x80000000 +.long 0xBE8E00FF, 0x80000000 +.long 0x9254FF32, 0x00000080 +.long 0x92545403 +.long 0x925532A0 +.long 0x92555552 +.long 0x81545554 +.long 0x92553304 +.long 0x80545554 +.long 0x2004CA85 +.long 0x2606CA9F +.long 0xD2850004, 0x00020432 +.long 0x32400704 +.long 0x68404054 +.long 0x24404082 +.long 0x8E4A8332 +.long 0x80CAFF4A, 0x00000108 +.long 0x6842404A +.long 0x6844424A +.long 0x6846444A +.long 0x6848464A +.long 0x684A484A +.long 0x684C4A4A +.long 0x684E4C4A +.long 0x68504E4A +.long 0x6852504A +.long 0x6854524A +.long 0x6856544A +.long 0x6858564A +.long 0x685A584A +.long 0x685C5A4A +.long 0x685E5C4A +.long 0xBECE00FF, 0x00001080 +.long 0x924E4E52 +.long 0x814EFF4E, 0x00004200 +.long 0xBF8A0000 +.long 0xBEFC004C +.long 0x814DFF4C, 0x00002100 +.long 0xE0511000, 0x80023052 +.long 0xE0511108, 0x80023153 +.long 0xE0511210, 0x80023254 +.long 0xE0511318, 0x80023355 +.long 0xE0511420, 0x80023456 +.long 0xE0511528, 0x80023557 +.long 0xE0511630, 0x80023658 +.long 0xE0511738, 0x80023759 +.long 0xBEFC004E +.long 0x814FFF4E, 0x00004200 +.long 0xE0511000, 0x80033020 +.long 0xE0511108, 0x80033121 +.long 0xE0511210, 0x80033222 +.long 0xE0511318, 0x80033323 +.long 0xE0511420, 0x80033424 +.long 0xE0511528, 0x80033525 +.long 0xE0511630, 0x80033626 +.long 0xE0511738, 0x80033727 +.long 0xE0511840, 0x80033828 +.long 0xE0511948, 0x80033929 +.long 0xE0511A50, 0x80033A2A +.long 0xE0511B58, 0x80033B2B +.long 0xE0511C60, 0x80033C2C +.long 0xE0511D68, 0x80033D2D +.long 0xE0511E70, 0x80033E2E +.long 0xE0511F78, 0x80033F2F +.long 0xBEFC004D +.long 0x68A4A4FF, 0x00000080 +.long 0x68A6A6FF, 0x00000080 +.long 0x68A8A8FF, 0x00000080 +.long 0x68AAAAFF, 0x00000080 +.long 0x68ACACFF, 0x00000080 +.long 0x68AEAEFF, 0x00000080 +.long 0x68B0B0FF, 0x00000080 +.long 0x68B2B2FF, 0x00000080 +.long 0x684040FF, 0x00000080 +.long 0x684242FF, 0x00000080 +.long 0x684444FF, 0x00000080 +.long 0x684646FF, 0x00000080 +.long 0x684848FF, 0x00000080 +.long 0x684A4AFF, 0x00000080 +.long 0x684C4CFF, 0x00000080 +.long 0x684E4EFF, 0x00000080 +.long 0x685050FF, 0x00000080 +.long 0x685252FF, 0x00000080 +.long 0x685454FF, 0x00000080 +.long 0x685656FF, 0x00000080 +.long 0x685858FF, 0x00000080 +.long 0x685A5AFF, 0x00000080 +.long 0x685C5CFF, 0x00000080 +.long 0x685E5EFF, 0x00000080 +.long 0xE0511000, 0x80023052 +.long 0xE0511108, 0x80023153 +.long 0xE0511210, 0x80023254 +.long 0xE0511318, 0x80023355 +.long 0xE0511420, 0x80023456 +.long 0xE0511528, 0x80023557 +.long 0xE0511630, 0x80023658 +.long 0xE0511738, 0x80023759 +.long 0xBEFC004F +.long 0xBF800000 +.long 0xE0511000, 0x80033020 +.long 0xE0511108, 0x80033121 +.long 0xE0511210, 0x80033222 +.long 0xE0511318, 0x80033323 +.long 0xE0511420, 0x80033424 +.long 0xE0511528, 0x80033525 +.long 0xE0511630, 0x80033626 +.long 0xE0511738, 0x80033727 +.long 0xE0511840, 0x80033828 +.long 0xE0511948, 0x80033929 +.long 0xE0511A50, 0x80033A2A +.long 0xE0511B58, 0x80033B2B +.long 0xE0511C60, 0x80033C2C +.long 0xE0511D68, 0x80033D2D +.long 0xE0511E70, 0x80033E2E +.long 0xE0511F78, 0x80033F2F +.long 0x68A4A4FF, 0x00000080 +.long 0x68A6A6FF, 0x00000080 +.long 0x68A8A8FF, 0x00000080 +.long 0x68AAAAFF, 0x00000080 +.long 0x68ACACFF, 0x00000080 +.long 0x68AEAEFF, 0x00000080 +.long 0x68B0B0FF, 0x00000080 +.long 0x68B2B2FF, 0x00000080 +.long 0x684040FF, 0x00000080 +.long 0x684242FF, 0x00000080 +.long 0x684444FF, 0x00000080 +.long 0x684646FF, 0x00000080 +.long 0x684848FF, 0x00000080 +.long 0x684A4AFF, 0x00000080 +.long 0x684C4CFF, 0x00000080 +.long 0x684E4EFF, 0x00000080 +.long 0x685050FF, 0x00000080 +.long 0x685252FF, 0x00000080 +.long 0x685454FF, 0x00000080 +.long 0x685656FF, 0x00000080 +.long 0x685858FF, 0x00000080 +.long 0x685A5AFF, 0x00000080 +.long 0x685C5CFF, 0x00000080 +.long 0x685E5EFF, 0x00000080 +.long 0xBEFC004C +.long 0xBF8C8F78 +.long 0xBF8A0000 +.long 0xBF8C4F78 +.long 0xBF8A0000 +.long 0x8F2E852D +.long 0x80AE2E80 +.long 0xBF06802E +.long 0xBF8500DC +.long 0xBF8A0000 +.long 0xE0511000, 0x80023052 +.long 0xE0511108, 0x80023153 +.long 0xE0511210, 0x80023254 +.long 0xE0511318, 0x80023355 +.long 0xE0511420, 0x80023456 +.long 0xE0511528, 0x80023557 +.long 0xE0511630, 0x80023658 +.long 0xE0511738, 0x80023759 +.long 0xBEFC004E +.long 0xBF800000 +.long 0xE0511000, 0x80033020 +.long 0xE0511108, 0x80033121 +.long 0xE0511210, 0x80033222 +.long 0xE0511318, 0x80033323 +.long 0xE0511420, 0x80033424 +.long 0xE0511528, 0x80033525 +.long 0xE0511630, 0x80033626 +.long 0xE0511738, 0x80033727 +.long 0xE0511840, 0x80033828 +.long 0xE0511948, 0x80033929 +.long 0xE0511A50, 0x80033A2A +.long 0xE0511B58, 0x80033B2B +.long 0xE0511C60, 0x80033C2C +.long 0xE0511D68, 0x80033D2D +.long 0xE0511E70, 0x80033E2E +.long 0xE0511F78, 0x80033F2F +.long 0xBF8C8F78 +.long 0xBF8F0001 +.long 0xBF8A0000 +.long 0x68A4A4FF, 0x00000080 +.long 0x68A6A6FF, 0x00000080 +.long 0x68A8A8FF, 0x00000080 +.long 0x68AAAAFF, 0x00000080 +.long 0x68ACACFF, 0x00000080 +.long 0x68AEAEFF, 0x00000080 +.long 0x68B0B0FF, 0x00000080 +.long 0x68B2B2FF, 0x00000080 +.long 0x684040FF, 0x00000080 +.long 0x684242FF, 0x00000080 +.long 0x684444FF, 0x00000080 +.long 0x684646FF, 0x00000080 +.long 0x684848FF, 0x00000080 +.long 0x684A4AFF, 0x00000080 +.long 0x684C4CFF, 0x00000080 +.long 0x684E4EFF, 0x00000080 +.long 0xBF8F0000 +.long 0xBF8C4F78 +.long 0xBF8F0001 +.long 0xBF8A0000 +.long 0x685050FF, 0x00000080 +.long 0x685252FF, 0x00000080 +.long 0x685454FF, 0x00000080 +.long 0x685656FF, 0x00000080 +.long 0x685858FF, 0x00000080 +.long 0x685A5AFF, 0x00000080 +.long 0x685C5CFF, 0x00000080 +.long 0x685E5EFF, 0x00000080 +.long 0xBF8F0000 +.long 0xBEFC004D +.long 0x802E812E +.long 0xBF8A0000 +.long 0xE0511000, 0x80023052 +.long 0xE0511108, 0x80023153 +.long 0xE0511210, 0x80023254 +.long 0xE0511318, 0x80023355 +.long 0xE0511420, 0x80023456 +.long 0xE0511528, 0x80023557 +.long 0xE0511630, 0x80023658 +.long 0xE0511738, 0x80023759 +.long 0xBEFC004F +.long 0xBF800000 +.long 0xE0511000, 0x80033020 +.long 0xE0511108, 0x80033121 +.long 0xE0511210, 0x80033222 +.long 0xE0511318, 0x80033323 +.long 0xE0511420, 0x80033424 +.long 0xE0511528, 0x80033525 +.long 0xE0511630, 0x80033626 +.long 0xE0511738, 0x80033727 +.long 0xE0511840, 0x80033828 +.long 0xE0511948, 0x80033929 +.long 0xE0511A50, 0x80033A2A +.long 0xE0511B58, 0x80033B2B +.long 0xE0511C60, 0x80033C2C +.long 0xE0511D68, 0x80033D2D +.long 0xE0511E70, 0x80033E2E +.long 0xE0511F78, 0x80033F2F +.long 0xBF8C8F78 +.long 0xBF8A0000 +.long 0xBF8F0001 +.long 0x68A4A4FF, 0x00000080 +.long 0x68A6A6FF, 0x00000080 +.long 0x68A8A8FF, 0x00000080 +.long 0x68AAAAFF, 0x00000080 +.long 0x68ACACFF, 0x00000080 +.long 0x68AEAEFF, 0x00000080 +.long 0x68B0B0FF, 0x00000080 +.long 0x68B2B2FF, 0x00000080 +.long 0x684040FF, 0x00000080 +.long 0x684242FF, 0x00000080 +.long 0x684444FF, 0x00000080 +.long 0x684646FF, 0x00000080 +.long 0x684848FF, 0x00000080 +.long 0x684A4AFF, 0x00000080 +.long 0x684C4CFF, 0x00000080 +.long 0x684E4EFF, 0x00000080 +.long 0xBF8F0000 +.long 0xBF8C4F78 +.long 0xBF8A0000 +.long 0xBF8F0001 +.long 0x685050FF, 0x00000080 +.long 0x685252FF, 0x00000080 +.long 0x685454FF, 0x00000080 +.long 0x685656FF, 0x00000080 +.long 0x685858FF, 0x00000080 +.long 0x685A5AFF, 0x00000080 +.long 0x685C5CFF, 0x00000080 +.long 0x685E5EFF, 0x00000080 +.long 0xBF8F0000 +.long 0xBEFC004C +.long 0x802E812E +.long 0xBF00C22E +.long 0xBF84FF24 +.long 0xBF8C4F70 +.long 0xBF8A0000 +.long 0xBF8C0F70 +.long 0xBF8A0000 +.long 0xBF810000 +.long 0xD3D94000, 0x18000080 +.long 0xD3D94001, 0x18000080 +.long 0xD3D94002, 0x18000080 +.long 0xD3D94003, 0x18000080 +.long 0xD3D94004, 0x18000080 +.long 0xD3D94005, 0x18000080 +.long 0xD3D94006, 0x18000080 +.long 0xD3D94007, 0x18000080 +.long 0xD3D94008, 0x18000080 +.long 0xD3D94009, 0x18000080 +.long 0xD3D9400A, 0x18000080 +.long 0xD3D9400B, 0x18000080 +.long 0xD3D9400C, 0x18000080 +.long 0xD3D9400D, 0x18000080 +.long 0xD3D9400E, 0x18000080 +.long 0xD3D9400F, 0x18000080 +.long 0xD3D94010, 0x18000080 +.long 0xD3D94011, 0x18000080 +.long 0xD3D94012, 0x18000080 +.long 0xD3D94013, 0x18000080 +.long 0xD3D94014, 0x18000080 +.long 0xD3D94015, 0x18000080 +.long 0xD3D94016, 0x18000080 +.long 0xD3D94017, 0x18000080 +.long 0xD3D94018, 0x18000080 +.long 0xD3D94019, 0x18000080 +.long 0xD3D9401A, 0x18000080 +.long 0xD3D9401B, 0x18000080 +.long 0xD3D9401C, 0x18000080 +.long 0xD3D9401D, 0x18000080 +.long 0xD3D9401E, 0x18000080 +.long 0xD3D9401F, 0x18000080 +.long 0xC0060700, 0x00000000 +.long 0xC00A0A00, 0x00000038 +.long 0xC00A0900, 0x00000040 +.long 0xC00A0800, 0x00000018 +.long 0xD1130001, 0x00013F65 +.long 0xD2850060, 0x000202A0 +.long 0x20020281 +.long 0xD2850001, 0x00020282 +.long 0x68C0C101 +.long 0x2002CA85 +.long 0x68C0C101 +.long 0x24C0C082 +.long 0x68C0C080 +.long 0x68C2C0FF, 0x00002100 +.long 0xBF8A0000 +.long 0xD1130001, 0x00013F65 +.long 0xD2850062, 0x000202A0 +.long 0x20020281 +.long 0xD2850001, 0x00020282 +.long 0x68C4C501 +.long 0x2002CA85 +.long 0x68C4C501 +.long 0x24C4C482 +.long 0x9254FF52, 0x00001080 +.long 0x68C4C454 +.long 0x68C4C4FF, 0x00004200 +.long 0x68C6C4FF, 0x00004200 +.long 0xBF8CC07F +.long 0xBE900022 +.long 0xBE910023 +.long 0xBE9200FF, 0x80000000 +.long 0xBE9300FF, 0x00020000 +.long 0xBE940020 +.long 0xBE950021 +.long 0xBE9600FF, 0x80000000 +.long 0xBE9700FF, 0x00020000 +.long 0x925603FF, 0x00000080 +.long 0x96552656 +.long 0x92542656 +.long 0x8ED48254 +.long 0x80105410 +.long 0x82115511 +.long 0x80145414 +.long 0x82155515 +.long 0x96552704 +.long 0x92542704 +.long 0x8ED48254 +.long 0x80105410 +.long 0x82115511 +.long 0x80145414 +.long 0x82155515 +.long 0x24C8CC86 +.long 0x68C8C965 +.long 0xD2850004, 0x0002CCA0 +.long 0xD2850003, 0x00004D04 +.long 0x2608C89F +.long 0xD2850005, 0x00004D04 +.long 0x2608C8BF +.long 0x200C0885 +.long 0x240C0C82 +.long 0x68D60B03 +.long 0x925402C0 +.long 0x32D40C54 +.long 0xD1FE0068, 0x020AD76A +.long 0xBF8A0000 +.long 0xD86C0000, 0x20000060 +.long 0xD86C1080, 0x21000060 +.long 0xD86C0008, 0x22000060 +.long 0xD86C1088, 0x23000060 +.long 0xD86C0010, 0x24000060 +.long 0xD86C1090, 0x25000060 +.long 0xD86C0018, 0x26000060 +.long 0xD86C1098, 0x27000060 +.long 0xD86C0020, 0x28000060 +.long 0xD86C10A0, 0x29000060 +.long 0xD86C0028, 0x2A000060 +.long 0xD86C10A8, 0x2B000060 +.long 0xD86C0030, 0x2C000060 +.long 0xD86C10B0, 0x2D000060 +.long 0xD86C0038, 0x2E000060 +.long 0xD86C10B8, 0x2F000060 +.long 0xD86C0040, 0x30000060 +.long 0xD86C10C0, 0x31000060 +.long 0xD86C0048, 0x32000060 +.long 0xD86C10C8, 0x33000060 +.long 0xD86C0050, 0x34000060 +.long 0xD86C10D0, 0x35000060 +.long 0xD86C0058, 0x36000060 +.long 0xD86C10D8, 0x37000060 +.long 0xD86C0060, 0x38000060 +.long 0xD86C10E0, 0x39000060 +.long 0xD86C0068, 0x3A000060 +.long 0xD86C10E8, 0x3B000060 +.long 0xD86C0070, 0x3C000060 +.long 0xD86C10F0, 0x3D000060 +.long 0xD86C0078, 0x3E000060 +.long 0xD86C10F8, 0x3F000060 +.long 0xBF8A0000 +.long 0xD86C0000, 0x00000062 +.long 0xD86C0008, 0x01000062 +.long 0xD86C0010, 0x02000062 +.long 0xD86C0018, 0x03000062 +.long 0xD86C0020, 0x04000062 +.long 0xD86C0028, 0x05000062 +.long 0x8F2E852D +.long 0x80AE2E80 +.long 0xBF06802E +.long 0xBF85053B +.long 0xBF8CC47F +.long 0xD3C40000, 0x04020120 +.long 0xD86C0030, 0x06000062 +.long 0xD86C0038, 0x07000062 +.long 0xD3C40010, 0x04420121 +.long 0xD86C0040, 0x08000062 +.long 0xD86C0048, 0x09000062 +.long 0xD3C40000, 0x04020322 +.long 0xD86C0050, 0x0A000062 +.long 0xD86C0058, 0x0B000062 +.long 0xD86C0060, 0x0C000062 +.long 0xD3C40010, 0x04420323 +.long 0xD86C0068, 0x0D000062 +.long 0xD86C0070, 0x0E000062 +.long 0xD86C0078, 0x0F000062 +.long 0xBF8A0000 +.long 0xBF8CCD7F +.long 0xD3C40000, 0x04020524 +.long 0xD3C40010, 0x04420525 +.long 0xBF8CCC7F +.long 0xD3C40000, 0x04020726 +.long 0xD3C40010, 0x04420727 +.long 0xBF8CCB7F +.long 0xD3C40000, 0x04020928 +.long 0xD3C40010, 0x04420929 +.long 0xBF8CCA7F +.long 0xD3C40000, 0x04020B2A +.long 0xD3C40010, 0x04420B2B +.long 0xBF8CC97F +.long 0xD3C40000, 0x04020D2C +.long 0xD3C40010, 0x04420D2D +.long 0xBF8CC87F +.long 0xD3C40000, 0x04020F2E +.long 0xD3C40010, 0x04420F2F +.long 0xBF8CC07F +.long 0xD3C40000, 0x04021130 +.long 0xD3C40010, 0x04421131 +.long 0xD3C40000, 0x04021332 +.long 0xD3C40010, 0x04421333 +.long 0xBF8F0000 +.long 0xD3C40000, 0x04021534 +.long 0xD3C40010, 0x04421535 +.long 0xBF8A0000 +.long 0xD3C40000, 0x04021736 +.long 0xD86C0000, 0x40000061 +.long 0xD86C1080, 0x41000061 +.long 0xD86C0008, 0x42000061 +.long 0xD86C1088, 0x43000061 +.long 0xD3C40010, 0x04421737 +.long 0xD86C0010, 0x44000061 +.long 0xD86C1090, 0x45000061 +.long 0xD86C0018, 0x46000061 +.long 0xD86C1098, 0x47000061 +.long 0xD3C40000, 0x04021938 +.long 0xD86C0020, 0x48000061 +.long 0xD86C10A0, 0x49000061 +.long 0xD86C0028, 0x4A000061 +.long 0xD86C10A8, 0x4B000061 +.long 0xD3C40010, 0x04421939 +.long 0xD86C0030, 0x4C000061 +.long 0xD86C10B0, 0x4D000061 +.long 0xD86C0038, 0x4E000061 +.long 0xD86C10B8, 0x4F000061 +.long 0xD3C40000, 0x04021B3A +.long 0xD86C0040, 0x50000061 +.long 0xD86C10C0, 0x51000061 +.long 0xD86C0048, 0x52000061 +.long 0xD86C10C8, 0x53000061 +.long 0xD3C40010, 0x04421B3B +.long 0xD86C0050, 0x54000061 +.long 0xD86C10D0, 0x55000061 +.long 0xD86C0058, 0x56000061 +.long 0xD86C10D8, 0x57000061 +.long 0xD3C40000, 0x04021D3C +.long 0xD86C0060, 0x58000061 +.long 0xD86C10E0, 0x59000061 +.long 0xD86C0068, 0x5A000061 +.long 0xD86C10E8, 0x5B000061 +.long 0xD3C40010, 0x04421D3D +.long 0xD86C0070, 0x5C000061 +.long 0xD86C10F0, 0x5D000061 +.long 0xD86C0078, 0x5E000061 +.long 0xD86C10F8, 0x5F000061 +.long 0xBF8A0000 +.long 0xD86C0000, 0x10000063 +.long 0xD86C0008, 0x11000063 +.long 0xD3C40000, 0x04021F3E +.long 0xD86C0010, 0x12000063 +.long 0xD86C0018, 0x13000063 +.long 0xD86C0020, 0x14000063 +.long 0xD86C0028, 0x15000063 +.long 0xD86C0030, 0x16000063 +.long 0xD3C40010, 0x04421F3F +.long 0xD86C0038, 0x17000063 +.long 0xD86C0040, 0x18000063 +.long 0xD86C0048, 0x19000063 +.long 0xD86C0050, 0x1A000063 +.long 0xD86C0058, 0x1B000063 +.long 0xBF8F0001 +.long 0x802E812E +.long 0xBF8CCA7F +.long 0xD3C40000, 0x04022140 +.long 0xD86C0060, 0x1C000063 +.long 0xD86C0068, 0x1D000063 +.long 0xD3C40010, 0x04422141 +.long 0xD86C0070, 0x1E000063 +.long 0xD86C0078, 0x1F000063 +.long 0xD3C40000, 0x04022342 +.long 0xD3C40010, 0x04422343 +.long 0xBF8A0000 +.long 0xBF8CCD7F +.long 0xD3C40000, 0x04022544 +.long 0xD3C40010, 0x04422545 +.long 0xBF8CCC7F +.long 0xD3C40000, 0x04022746 +.long 0xD3C40010, 0x04422747 +.long 0xBF8CCB7F +.long 0xD3C40000, 0x04022948 +.long 0xD3C40010, 0x04422949 +.long 0xBF8CCA7F +.long 0xD3C40000, 0x04022B4A +.long 0xD3C40010, 0x04422B4B +.long 0xBF8CC97F +.long 0xD3C40000, 0x04022D4C +.long 0xD3C40010, 0x04422D4D +.long 0xBF8CC87F +.long 0xD3C40000, 0x04022F4E +.long 0xD3C40010, 0x04422F4F +.long 0xBF8CC07F +.long 0xD3C40000, 0x04023150 +.long 0xD3C40010, 0x04423151 +.long 0xD3C40000, 0x04023352 +.long 0xD3C40010, 0x04423353 +.long 0xBF8F0000 +.long 0xD3C40000, 0x04023554 +.long 0xD3C40010, 0x04423555 +.long 0xBF8A0000 +.long 0xD86C0000, 0x20000060 +.long 0xD86C1080, 0x21000060 +.long 0xD86C0008, 0x22000060 +.long 0xD86C1088, 0x23000060 +.long 0xD3C40000, 0x04023756 +.long 0xD86C0010, 0x24000060 +.long 0xD86C1090, 0x25000060 +.long 0xD86C0018, 0x26000060 +.long 0xD86C1098, 0x27000060 +.long 0xD3C40010, 0x04423757 +.long 0xD86C0020, 0x28000060 +.long 0xD86C10A0, 0x29000060 +.long 0xD86C0028, 0x2A000060 +.long 0xD86C10A8, 0x2B000060 +.long 0xD3C40000, 0x04023958 +.long 0xD86C0030, 0x2C000060 +.long 0xD86C10B0, 0x2D000060 +.long 0xD86C0038, 0x2E000060 +.long 0xD86C10B8, 0x2F000060 +.long 0xD3C40010, 0x04423959 +.long 0xD86C0040, 0x30000060 +.long 0xD86C10C0, 0x31000060 +.long 0xD86C0048, 0x32000060 +.long 0xD86C10C8, 0x33000060 +.long 0xD3C40000, 0x04023B5A +.long 0xD86C0050, 0x34000060 +.long 0xD86C10D0, 0x35000060 +.long 0xD86C0058, 0x36000060 +.long 0xD86C10D8, 0x37000060 +.long 0xD3C40010, 0x04423B5B +.long 0xD86C0060, 0x38000060 +.long 0xD86C10E0, 0x39000060 +.long 0xD86C0068, 0x3A000060 +.long 0xD86C10E8, 0x3B000060 +.long 0xD3C40000, 0x04023D5C +.long 0xD86C0070, 0x3C000060 +.long 0xD86C10F0, 0x3D000060 +.long 0xD86C0078, 0x3E000060 +.long 0xD86C10F8, 0x3F000060 +.long 0xD3C40010, 0x04423D5D +.long 0xBF8A0000 +.long 0xD86C0000, 0x00000062 +.long 0xD86C0008, 0x01000062 +.long 0xD86C0010, 0x02000062 +.long 0xD86C0018, 0x03000062 +.long 0xD3C40000, 0x04023F5E +.long 0xD86C0020, 0x04000062 +.long 0xD86C0028, 0x05000062 +.long 0xD86C0030, 0x06000062 +.long 0xD86C0038, 0x07000062 +.long 0xD3C40010, 0x04423F5F +.long 0xD86C0040, 0x08000062 +.long 0xD86C0048, 0x09000062 +.long 0xD86C0050, 0x0A000062 +.long 0xD86C0058, 0x0B000062 +.long 0xBF8F0001 +.long 0x802E812E +.long 0xBF8CCA7F +.long 0xD3C40000, 0x04020120 +.long 0xD86C0060, 0x0C000062 +.long 0xD86C0068, 0x0D000062 +.long 0xD3C40010, 0x04420121 +.long 0xD86C0070, 0x0E000062 +.long 0xD86C0078, 0x0F000062 +.long 0xD3C40000, 0x04020322 +.long 0xD3C40010, 0x04420323 +.long 0xBF8A0000 +.long 0xBF8CC87F +.long 0xD3C40000, 0x04020524 +.long 0xD3C40010, 0x04420525 +.long 0xD3C40000, 0x04020726 +.long 0xD3C40010, 0x04420727 +.long 0xD3C40000, 0x04020928 +.long 0xD3C40010, 0x04420929 +.long 0xD3C40000, 0x04020B2A +.long 0xD3C40010, 0x04420B2B +.long 0xD3C40000, 0x04020D2C +.long 0xD3C40010, 0x04420D2D +.long 0xD3C40000, 0x04020F2E +.long 0xD3C40010, 0x04420F2F +.long 0xBF8CC07F +.long 0xD3C40000, 0x04021130 +.long 0xD3C40010, 0x04421131 +.long 0xD3C40000, 0x04021332 +.long 0xD3C40010, 0x04421333 +.long 0xBF8F0000 +.long 0xD3C40000, 0x04021534 +.long 0xD3C40010, 0x04421535 +.long 0xBF8A0000 +.long 0xD86C0000, 0x40000061 +.long 0xD86C1080, 0x41000061 +.long 0xD86C0008, 0x42000061 +.long 0xD86C1088, 0x43000061 +.long 0xD3C40000, 0x04021736 +.long 0xD86C0010, 0x44000061 +.long 0xD86C1090, 0x45000061 +.long 0xD86C0018, 0x46000061 +.long 0xD86C1098, 0x47000061 +.long 0xD3C40010, 0x04421737 +.long 0xD86C0020, 0x48000061 +.long 0xD86C10A0, 0x49000061 +.long 0xD86C0028, 0x4A000061 +.long 0xD86C10A8, 0x4B000061 +.long 0xD3C40000, 0x04021938 +.long 0xD86C0030, 0x4C000061 +.long 0xD86C10B0, 0x4D000061 +.long 0xD86C0038, 0x4E000061 +.long 0xD86C10B8, 0x4F000061 +.long 0xD3C40010, 0x04421939 +.long 0xD86C0040, 0x50000061 +.long 0xD86C10C0, 0x51000061 +.long 0xD86C0048, 0x52000061 +.long 0xD86C10C8, 0x53000061 +.long 0xD3C40000, 0x04021B3A +.long 0xD86C0050, 0x54000061 +.long 0xD86C10D0, 0x55000061 +.long 0xD86C0058, 0x56000061 +.long 0xD86C10D8, 0x57000061 +.long 0xD3C40010, 0x04421B3B +.long 0xD86C0060, 0x58000061 +.long 0xD86C10E0, 0x59000061 +.long 0xD86C0068, 0x5A000061 +.long 0xD86C10E8, 0x5B000061 +.long 0xD3C40000, 0x04021D3C +.long 0xD86C0070, 0x5C000061 +.long 0xD86C10F0, 0x5D000061 +.long 0xD86C0078, 0x5E000061 +.long 0xD86C10F8, 0x5F000061 +.long 0xD3C40010, 0x04421D3D +.long 0xBF8A0000 +.long 0xD86C0000, 0x10000063 +.long 0xD86C0008, 0x11000063 +.long 0xD3C40000, 0x04021F3E +.long 0xD86C0010, 0x12000063 +.long 0xD86C0018, 0x13000063 +.long 0xD86C0020, 0x14000063 +.long 0xD86C0028, 0x15000063 +.long 0xD86C0030, 0x16000063 +.long 0xD3C40010, 0x04421F3F +.long 0xD86C0038, 0x17000063 +.long 0xD86C0040, 0x18000063 +.long 0xD86C0048, 0x19000063 +.long 0xD86C0050, 0x1A000063 +.long 0xD86C0058, 0x1B000063 +.long 0xBF8F0001 +.long 0x802E812E +.long 0xBF8CCA7F +.long 0xD3C40000, 0x04022140 +.long 0xD86C0060, 0x1C000063 +.long 0xD86C0068, 0x1D000063 +.long 0xD3C40010, 0x04422141 +.long 0xD86C0070, 0x1E000063 +.long 0xD86C0078, 0x1F000063 +.long 0xD3C40000, 0x04022342 +.long 0xD3C40010, 0x04422343 +.long 0xBF8A0000 +.long 0xBF8CC87F +.long 0xD3C40000, 0x04022544 +.long 0xD3C40010, 0x04422545 +.long 0xD3C40000, 0x04022746 +.long 0xD3C40010, 0x04422747 +.long 0xD3C40000, 0x04022948 +.long 0xD3C40010, 0x04422949 +.long 0xD3C40000, 0x04022B4A +.long 0xD3C40010, 0x04422B4B +.long 0xD3C40000, 0x04022D4C +.long 0xD3C40010, 0x04422D4D +.long 0xD3C40000, 0x04022F4E +.long 0xD3C40010, 0x04422F4F +.long 0xBF8CC07F +.long 0xD3C40000, 0x04023150 +.long 0xD3C40010, 0x04423151 +.long 0xD3C40000, 0x04023352 +.long 0xD3C40010, 0x04423353 +.long 0xBF8F0000 +.long 0xD3C40000, 0x04023554 +.long 0xD3C40010, 0x04423555 +.long 0xBF8A0000 +.long 0xD86C0000, 0x20000060 +.long 0xD86C1080, 0x21000060 +.long 0xD86C0008, 0x22000060 +.long 0xD86C1088, 0x23000060 +.long 0xD3C40000, 0x04023756 +.long 0xD86C0010, 0x24000060 +.long 0xD86C1090, 0x25000060 +.long 0xD86C0018, 0x26000060 +.long 0xD86C1098, 0x27000060 +.long 0xD3C40010, 0x04423757 +.long 0xD86C0020, 0x28000060 +.long 0xD86C10A0, 0x29000060 +.long 0xD86C0028, 0x2A000060 +.long 0xD86C10A8, 0x2B000060 +.long 0xD3C40000, 0x04023958 +.long 0xD86C0030, 0x2C000060 +.long 0xD86C10B0, 0x2D000060 +.long 0xD86C0038, 0x2E000060 +.long 0xD86C10B8, 0x2F000060 +.long 0xD3C40010, 0x04423959 +.long 0xD86C0040, 0x30000060 +.long 0xD86C10C0, 0x31000060 +.long 0xD86C0048, 0x32000060 +.long 0xD86C10C8, 0x33000060 +.long 0xD3C40000, 0x04023B5A +.long 0xD86C0050, 0x34000060 +.long 0xD86C10D0, 0x35000060 +.long 0xD86C0058, 0x36000060 +.long 0xD86C10D8, 0x37000060 +.long 0xD3C40010, 0x04423B5B +.long 0xD86C0060, 0x38000060 +.long 0xD86C10E0, 0x39000060 +.long 0xD86C0068, 0x3A000060 +.long 0xD86C10E8, 0x3B000060 +.long 0xD3C40000, 0x04023D5C +.long 0xD86C0070, 0x3C000060 +.long 0xD86C10F0, 0x3D000060 +.long 0xD86C0078, 0x3E000060 +.long 0xD86C10F8, 0x3F000060 +.long 0xD3C40010, 0x04423D5D +.long 0xBF8A0000 +.long 0xD86C0000, 0x00000062 +.long 0xD86C0008, 0x01000062 +.long 0xD3C40000, 0x04023F5E +.long 0xD86C0010, 0x02000062 +.long 0xD86C0018, 0x03000062 +.long 0xD86C0020, 0x04000062 +.long 0xD86C0028, 0x05000062 +.long 0xD86C0030, 0x06000062 +.long 0xD3C40010, 0x04423F5F +.long 0xD86C0038, 0x07000062 +.long 0xD86C0040, 0x08000062 +.long 0xD86C0048, 0x09000062 +.long 0xD86C0050, 0x0A000062 +.long 0xD86C0058, 0x0B000062 +.long 0xBF8F0001 +.long 0x802E812E +.long 0xBF00C22E +.long 0xBF84FEAC +.long 0xBF8CCA7F +.long 0xD3C40000, 0x04020120 +.long 0xD3C40010, 0x04420121 +.long 0xD86C0060, 0x0C000062 +.long 0xD86C0068, 0x0D000062 +.long 0xD86C0070, 0x0E000062 +.long 0xD86C0078, 0x0F000062 +.long 0xD3C40000, 0x04020322 +.long 0xD3C40010, 0x04420323 +.long 0xBF8CCC7F +.long 0xD3C40000, 0x04020524 +.long 0xD3C40010, 0x04420525 +.long 0xD3C40000, 0x04020726 +.long 0xD3C40010, 0x04420727 +.long 0xBF8CCA7F +.long 0xD3C40000, 0x04020928 +.long 0xD3C40010, 0x04420929 +.long 0xD3C40000, 0x04020B2A +.long 0xD3C40010, 0x04420B2B +.long 0xBF8CC87F +.long 0xD3C40000, 0x04020D2C +.long 0xD3C40010, 0x04420D2D +.long 0xD3C40000, 0x04020F2E +.long 0xD3C40010, 0x04420F2F +.long 0xBF8CC07F +.long 0xD3C40000, 0x04021130 +.long 0xBF068029 +.long 0xBF850008 +.long 0xE05C1000, 0x80042068 +.long 0xE05C1020, 0x80042468 +.long 0xE05C1040, 0x80042868 +.long 0xE05C1060, 0x80042C68 +.long 0xD3C40010, 0x04421131 +.long 0xD3C40000, 0x04021332 +.long 0xD3C40010, 0x04421333 +.long 0xD3C40000, 0x04021534 +.long 0xD3C40010, 0x04421535 +.long 0xBF8A0000 +.long 0xD86C0000, 0x40000061 +.long 0xD86C1080, 0x41000061 +.long 0xD86C0008, 0x42000061 +.long 0xD86C1088, 0x43000061 +.long 0xD3C40000, 0x04021736 +.long 0xD86C0010, 0x44000061 +.long 0xD86C1090, 0x45000061 +.long 0xD86C0018, 0x46000061 +.long 0xD86C1098, 0x47000061 +.long 0xD3C40010, 0x04421737 +.long 0xD86C0020, 0x48000061 +.long 0xD86C10A0, 0x49000061 +.long 0xD86C0028, 0x4A000061 +.long 0xD86C10A8, 0x4B000061 +.long 0xD3C40000, 0x04021938 +.long 0xD86C0030, 0x4C000061 +.long 0xD86C10B0, 0x4D000061 +.long 0xD86C0038, 0x4E000061 +.long 0xD86C10B8, 0x4F000061 +.long 0xD3C40010, 0x04421939 +.long 0xD86C0040, 0x50000061 +.long 0xD86C10C0, 0x51000061 +.long 0xD86C0048, 0x52000061 +.long 0xD86C10C8, 0x53000061 +.long 0xD3C40000, 0x04021B3A +.long 0xD86C0050, 0x54000061 +.long 0xD86C10D0, 0x55000061 +.long 0xD86C0058, 0x56000061 +.long 0xD86C10D8, 0x57000061 +.long 0xD3C40010, 0x04421B3B +.long 0xD86C0060, 0x58000061 +.long 0xD86C10E0, 0x59000061 +.long 0xD86C0068, 0x5A000061 +.long 0xD86C10E8, 0x5B000061 +.long 0xD3C40000, 0x04021D3C +.long 0xD86C0070, 0x5C000061 +.long 0xD86C10F0, 0x5D000061 +.long 0xD86C0078, 0x5E000061 +.long 0xD86C10F8, 0x5F000061 +.long 0xD3C40010, 0x04421D3D +.long 0xBF8A0000 +.long 0xD86C0000, 0x10000063 +.long 0xD86C0008, 0x11000063 +.long 0xD3C40000, 0x04021F3E +.long 0xD86C0010, 0x12000063 +.long 0xD86C0018, 0x13000063 +.long 0xD86C0020, 0x14000063 +.long 0xD86C0028, 0x15000063 +.long 0xD86C0030, 0x16000063 +.long 0xD3C40010, 0x04421F3F +.long 0xD86C0038, 0x17000063 +.long 0xD86C0040, 0x18000063 +.long 0xD86C0048, 0x19000063 +.long 0xD86C0050, 0x1A000063 +.long 0xD86C0058, 0x1B000063 +.long 0xBF8CCA7F +.long 0xD3C40000, 0x04022140 +.long 0xBF068029 +.long 0xBF850008 +.long 0xE05C1080, 0x80043068 +.long 0xE05C10A0, 0x80043468 +.long 0xE05C10C0, 0x80043868 +.long 0xE05C10E0, 0x80043C68 +.long 0xD3C40010, 0x04422141 +.long 0xD86C0060, 0x1C000063 +.long 0xD86C0068, 0x1D000063 +.long 0xD86C0070, 0x1E000063 +.long 0xD86C0078, 0x1F000063 +.long 0xD3C40000, 0x04022342 +.long 0xD3C40010, 0x04422343 +.long 0xBF8CCC7F +.long 0xD3C40000, 0x04022544 +.long 0xD3C40010, 0x04422545 +.long 0xD3C40000, 0x04022746 +.long 0xD3C40010, 0x04422747 +.long 0xBF8CCA7F +.long 0xD3C40000, 0x04022948 +.long 0xD3C40010, 0x04422949 +.long 0xD3C40000, 0x04022B4A +.long 0xD3C40010, 0x04422B4B +.long 0xBF8CC87F +.long 0xD3C40000, 0x04022D4C +.long 0xD3C40010, 0x04422D4D +.long 0xD3C40000, 0x04022F4E +.long 0xD3C40010, 0x04422F4F +.long 0xBF8CC07F +.long 0xD3C40000, 0x04023150 +.long 0xD3C40000, 0x04023352 +.long 0xD3C40000, 0x04023554 +.long 0xD3C40000, 0x04023756 +.long 0xD3C40000, 0x04023958 +.long 0xD3C40000, 0x04023B5A +.long 0xD3C40000, 0x04023D5C +.long 0xD3C40000, 0x04023F5E +.long 0xBF068029 +.long 0xBF8400A3 +.long 0xD3C40010, 0x04423151 +.long 0xD3D84000, 0x18000100 +.long 0xD3D84001, 0x18000101 +.long 0xD3D84002, 0x18000102 +.long 0xD3D84003, 0x18000103 +.long 0xD3C40010, 0x04423353 +.long 0xD3D84004, 0x18000104 +.long 0xD3D84005, 0x18000105 +.long 0xD3D84006, 0x18000106 +.long 0xD3D84007, 0x18000107 +.long 0xD1050000, 0x00005100 +.long 0xD1050001, 0x00005101 +.long 0xD1050002, 0x00005102 +.long 0xD1050003, 0x00005103 +.long 0xE07C1000, 0x80050068 +.long 0xD3C40010, 0x04423555 +.long 0xD3D84008, 0x18000108 +.long 0xD3D84009, 0x18000109 +.long 0xD3D8400A, 0x1800010A +.long 0xD3D8400B, 0x1800010B +.long 0xD1050004, 0x00005104 +.long 0xD1050005, 0x00005105 +.long 0xD1050006, 0x00005106 +.long 0xD1050007, 0x00005107 +.long 0xE07C1020, 0x80050468 +.long 0xD3C40010, 0x04423757 +.long 0xD3D8400C, 0x1800010C +.long 0xD3D8400D, 0x1800010D +.long 0xD3D8400E, 0x1800010E +.long 0xD3D8400F, 0x1800010F +.long 0xD1050008, 0x00005108 +.long 0xD1050009, 0x00005109 +.long 0xD105000A, 0x0000510A +.long 0xD105000B, 0x0000510B +.long 0xE07C1040, 0x80050868 +.long 0xD3C40010, 0x04423959 +.long 0xD105000C, 0x0000510C +.long 0xD105000D, 0x0000510D +.long 0xD105000E, 0x0000510E +.long 0xD105000F, 0x0000510F +.long 0xE07C1060, 0x80050C68 +.long 0xD3C40010, 0x04423B5B +.long 0xD3C40010, 0x04423D5D +.long 0xD3C40010, 0x04423F5F +.long 0xBF800003 +.long 0xD3D84000, 0x18000110 +.long 0xD3D84001, 0x18000111 +.long 0xD3D84002, 0x18000112 +.long 0xD3D84003, 0x18000113 +.long 0xD3D84004, 0x18000114 +.long 0xD3D84005, 0x18000115 +.long 0xD3D84006, 0x18000116 +.long 0xD3D84007, 0x18000117 +.long 0xD1050000, 0x00005100 +.long 0xD1050001, 0x00005101 +.long 0xD1050002, 0x00005102 +.long 0xD1050003, 0x00005103 +.long 0xE07C1080, 0x80050068 +.long 0xD3D84008, 0x18000118 +.long 0xD3D84009, 0x18000119 +.long 0xD3D8400A, 0x1800011A +.long 0xD3D8400B, 0x1800011B +.long 0xD1050004, 0x00005104 +.long 0xD1050005, 0x00005105 +.long 0xD1050006, 0x00005106 +.long 0xD1050007, 0x00005107 +.long 0xE07C10A0, 0x80050468 +.long 0xD3D8400C, 0x1800011C +.long 0xD3D8400D, 0x1800011D +.long 0xD3D8400E, 0x1800011E +.long 0xD3D8400F, 0x1800011F +.long 0xD1050008, 0x00005108 +.long 0xD1050009, 0x00005109 +.long 0xD105000A, 0x0000510A +.long 0xD105000B, 0x0000510B +.long 0xE07C10C0, 0x80050868 +.long 0xD105000C, 0x0000510C +.long 0xD105000D, 0x0000510D +.long 0xD105000E, 0x0000510E +.long 0xD105000F, 0x0000510F +.long 0xE07C10E0, 0x80050C68 +.long 0xBF8C0000 +.long 0xBF810000 +.long 0xD3C40010, 0x04423151 +.long 0xD3C40010, 0x04423353 +.long 0xD3C40010, 0x04423555 +.long 0xD3C40010, 0x04423757 +.long 0xD3C40010, 0x04423959 +.long 0xD3C40010, 0x04423B5B +.long 0xD3C40010, 0x04423D5D +.long 0xD3C40010, 0x04423F5F +.long 0xD3D84000, 0x18000100 +.long 0xD3D84001, 0x18000101 +.long 0xD3D84002, 0x18000102 +.long 0xD3D84003, 0x18000103 +.long 0xD3D84004, 0x18000104 +.long 0xD3D84005, 0x18000105 +.long 0xD3D84006, 0x18000106 +.long 0xD3D84007, 0x18000107 +.long 0xD3D84008, 0x18000108 +.long 0xD3D84009, 0x18000109 +.long 0xD3D8400A, 0x1800010A +.long 0xD3D8400B, 0x1800010B +.long 0xD3D8400C, 0x1800010C +.long 0xD3D8400D, 0x1800010D +.long 0xD3D8400E, 0x1800010E +.long 0xD3D8400F, 0x1800010F +.long 0xBF8C0F74 +.long 0xD1050000, 0x00005100 +.long 0xD1050001, 0x00005101 +.long 0xD1050002, 0x00005102 +.long 0xD1050003, 0x00005103 +.long 0xD1160000, 0x00005320 +.long 0xD1160001, 0x00005321 +.long 0xD1160002, 0x00005322 +.long 0xD1160003, 0x00005323 +.long 0xE07C1000, 0x80050068 +.long 0xD1050004, 0x00005104 +.long 0xD1050005, 0x00005105 +.long 0xD1050006, 0x00005106 +.long 0xD1050007, 0x00005107 +.long 0xD1160004, 0x00005324 +.long 0xD1160005, 0x00005325 +.long 0xD1160006, 0x00005326 +.long 0xD1160007, 0x00005327 +.long 0xE07C1020, 0x80050468 +.long 0xD1050008, 0x00005108 +.long 0xD1050009, 0x00005109 +.long 0xD105000A, 0x0000510A +.long 0xD105000B, 0x0000510B +.long 0xD1160008, 0x00005328 +.long 0xD1160009, 0x00005329 +.long 0xD116000A, 0x0000532A +.long 0xD116000B, 0x0000532B +.long 0xE07C1040, 0x80050868 +.long 0xD105000C, 0x0000510C +.long 0xD105000D, 0x0000510D +.long 0xD105000E, 0x0000510E +.long 0xD105000F, 0x0000510F +.long 0xD116000C, 0x0000532C +.long 0xD116000D, 0x0000532D +.long 0xD116000E, 0x0000532E +.long 0xD116000F, 0x0000532F +.long 0xE07C1060, 0x80050C68 +.long 0xD3D84000, 0x18000110 +.long 0xD3D84001, 0x18000111 +.long 0xD3D84002, 0x18000112 +.long 0xD3D84003, 0x18000113 +.long 0xD3D84004, 0x18000114 +.long 0xD3D84005, 0x18000115 +.long 0xD3D84006, 0x18000116 +.long 0xD3D84007, 0x18000117 +.long 0xD3D84008, 0x18000118 +.long 0xD3D84009, 0x18000119 +.long 0xD3D8400A, 0x1800011A +.long 0xD3D8400B, 0x1800011B +.long 0xD3D8400C, 0x1800011C +.long 0xD3D8400D, 0x1800011D +.long 0xD3D8400E, 0x1800011E +.long 0xD3D8400F, 0x1800011F +.long 0xBF8C0F70 +.long 0xD1050000, 0x00005100 +.long 0xD1050001, 0x00005101 +.long 0xD1050002, 0x00005102 +.long 0xD1050003, 0x00005103 +.long 0xD1160000, 0x00005330 +.long 0xD1160001, 0x00005331 +.long 0xD1160002, 0x00005332 +.long 0xD1160003, 0x00005333 +.long 0xE07C1080, 0x80050068 +.long 0xD1050004, 0x00005104 +.long 0xD1050005, 0x00005105 +.long 0xD1050006, 0x00005106 +.long 0xD1050007, 0x00005107 +.long 0xD1160004, 0x00005334 +.long 0xD1160005, 0x00005335 +.long 0xD1160006, 0x00005336 +.long 0xD1160007, 0x00005337 +.long 0xE07C10A0, 0x80050468 +.long 0xD1050008, 0x00005108 +.long 0xD1050009, 0x00005109 +.long 0xD105000A, 0x0000510A +.long 0xD105000B, 0x0000510B +.long 0xD1160008, 0x00005338 +.long 0xD1160009, 0x00005339 +.long 0xD116000A, 0x0000533A +.long 0xD116000B, 0x0000533B +.long 0xE07C10C0, 0x80050868 +.long 0xD105000C, 0x0000510C +.long 0xD105000D, 0x0000510D +.long 0xD105000E, 0x0000510E +.long 0xD105000F, 0x0000510F +.long 0xD116000C, 0x0000533C +.long 0xD116000D, 0x0000533D +.long 0xD116000E, 0x0000533E +.long 0xD116000F, 0x0000533F +.long 0xE07C10E0, 0x80050C68 +.long 0xBF8C0000 +.long 0xBF810000 + diff --git a/tensilelite/Tensile/EmbeddedData.py b/tensilelite/Tensile/EmbeddedData.py new file mode 100644 index 0000000000..acb3cfc239 --- /dev/null +++ b/tensilelite/Tensile/EmbeddedData.py @@ -0,0 +1,209 @@ +################################################################################ +# Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from . import Common + +import itertools +import os + +class Namespace: + def __init__(self, parent, name=None): + self.parent = parent + self.name = name + + def __enter__(self): + if self.name is None: + self.parent.write("namespace {") + else: + self.parent.write("namespace {} {{".format(self.name)) + self.parent.indent() + return self + + def __exit__(self, *args, **kwargs): + self.parent.dedent() + if self.name is None: + self.parent.write("} // anonymous namespace") + else: + self.parent.write("}} // namespace {}".format(self.name)) + +class Indent: + def __init__(self, parent): + self.parent = parent + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + self.parent.dedent() + +class EmbeddedDataFile: + def __init__(self, filename, file=None, indent_spaces=4): + self.filename = filename + + self._indent_spaces = indent_spaces + self._indent_levels = [0] + self._open_blocks = [] + + self.file = file + if self.file is None: + self.file = open(filename, 'w') + + self.write_header() + + def __enter__(self): + self.file.__enter__() + return self + + def __exit__(self, *args, **kwargs): + while len(self._open_blocks) > 0: + b = self._open_blocks.pop() + b.__exit__(*args, **kwargs) + + self.write_footer() + + self.file.__exit__(*args, **kwargs) + + def namespace(self, name=None): + ns = Namespace(self, name).__enter__() + self._open_blocks.append(ns) + + def end_namespace(self, name=None): + ns = self._open_blocks.pop() + if not isinstance(ns, Namespace): + raise RuntimeError("Mismatched block types: expected Namespace, found {}".format(ns.__class__)) + if name != ns.name: + raise RuntimeError("Mismatched namespace open/close: expected {}, found {}".format(name, ns.name)) + + ns.__exit__(None, None, None) + + def write_header(self): + self.write(Common.CHeader) + + self.write(self.includes) + + self.namespace("Tensile") + + @property + def include_guard(self): + return "#pragma once" + + @property + def includes(self): + return """ + #include + + #include + #include + """ + + def get_lines(self, item): + if isinstance(item, str): + return item.split('\n') + + if hasattr(item, '__iter__'): + return item + + return str(item).split('\n') + + def format(self, item): + out_lines = [] + for line in self.get_lines(item): + out_lines.append(self.apply_indent(line)) + + return '\n'.join(out_lines) + '\n' + + @property + def indent_level(self): + if len(self._indent_levels) == 0: + return 0 + + return self._indent_levels[-1] + + def apply_indent(self, line=None): + if line is None: + return ' ' * self.indent_level + + line = line.strip() + + if line.startswith('#'): + return line + + return (' ' * self.indent_level) + line + + def indent(self, spaces=None): + if spaces is None: + spaces = self._indent_spaces + self._indent_levels.append(spaces + self.indent_level) + return Indent(self) + + def dedent(self): + self._indent_levels.pop() + + def write(self, *items): + for item in items: + self.file.write(self.format(item)) + + def comment(self, text): + self.write(['// ' + line for line in text.split('\n')]) + + def write_footer(self): + self.write('') + + def embed_data(self, assocType, data, nullTerminated=False, comment=None, key=None): + if nullTerminated: + empty = False + data = itertools.chain(data, [0]) + else: + empty = len(data) == 0 + data = iter(data) + + with Namespace(self): + if comment is not None: + self.comment(comment) + if empty: + if key is None: + self.write("EmbedData<{0}> TENSILE_EMBED_SYMBOL_NAME{{}};".format(assocType)) + else: + self.write('EmbedData<{0}> TENSILE_EMBED_SYMBOL_NAME("{1}", {{}});'.format(assocType, key)) + return + + hex_format = '{:#04x}' + + if key is None: + self.write("EmbedData<{0}> TENSILE_EMBED_SYMBOL_NAME({{".format(assocType)) + else: + self.write('EmbedData<{0}> TENSILE_EMBED_SYMBOL_NAME("{1}", {{'.format(assocType, key)) + with self.indent(): + line = hex_format.format(next(data)) + for byteIdx, byte in enumerate(data): + if byteIdx % 16 == 15: + self.write(line + ",") + line = hex_format.format(byte) + else: + line += ', ' + hex_format.format(byte) + + self.write(line + '});') + + def embed_file(self, assocType, filename, nullTerminated=False, key=None): + with open(filename, 'rb') as f: + byteArray = bytearray(f.read()) + self.embed_data(assocType, byteArray, nullTerminated, os.path.basename(filename), key) + diff --git a/tensilelite/Tensile/GenerateSummations.py b/tensilelite/Tensile/GenerateSummations.py new file mode 100644 index 0000000000..23a7ddf67b --- /dev/null +++ b/tensilelite/Tensile/GenerateSummations.py @@ -0,0 +1,194 @@ + +################################################################################ +# Copyright 2016-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +import os + +import pandas as pd +import numpy as np +import yaml +import subprocess +import glob + +from shutil import copyfile +from copy import deepcopy + +from . import LibraryIO + +from . import ClientWriter +from .TensileInstructions import getGfxName +from .Common import assignGlobalParameters, ensurePath, globalParameters, \ + gfxArch, printExit, getArchitectureName +from .SolutionStructs import ProblemSizes + + +def getArchitecture(isaName): + archid = getGfxName(isaName) + return getArchitectureName(archid) + +def isValidArch(archName, currentArch): + arch = gfxArch(archName) + return currentArch == arch + +############################################################################## +# createLibraryForBenchmark +############################################################################## +def createLibraryForBenchmark(logicPath, libraryPath, currentPath): + """ + takes the path of existing logic files as input and adds the summation + model for each of the solutions. This is used in the Tile Aware Metirc + Selection. + """ + + pythonExePath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin", "TensileCreateLibrary") + args = [pythonExePath, \ + "--merge-files", "--new-client-only", "--no-short-file-names", "--no-library-print-debug", \ + "--architecture=all", "--code-object-version=V3", "--cxx-compiler=hipcc", "--library-format=yaml", \ + logicPath, libraryPath, "HIP"] + + try: + subprocess.run(args, check=True, cwd=currentPath) + except (subprocess.CalledProcessError, OSError) as e: + printExit("ClientWriter Benchmark Process exited with error: {}".format(e)) + +def GenerateSummations(userArgs): + + inputLogicPath = userArgs[0] + outputPath = userArgs[1] + assignGlobalParameters({}) + + currentISA = globalParameters["CurrentISA"] + currentArchitecture = getArchitecture(currentISA) + + globPath = os.path.join(inputLogicPath, "{}*".format(currentArchitecture)) + logicFileNames = glob.glob(globPath) + + for logicFileName in logicFileNames: + + logicFileBaseName = os.path.basename(logicFileName) + logicFileStem, ext = os.path.splitext(logicFileBaseName) + if (ext != ".yaml"): + continue + + currentPath = ensurePath(os.path.join(outputPath, logicFileStem)) + + libPath = ensurePath(os.path.join(currentPath, "lib")) + finalPath = ensurePath(os.path.join(currentPath, "final")) + localLogicPath = ensurePath(os.path.join(currentPath, "logic")) + localLogicFilePath = os.path.join(localLogicPath, logicFileBaseName) + + # Here we read in two version of the logic the first one fills the solutions with + # defaults and modifies some of the parameters. The final logic file should be the + # same as the initial logic with the summation model added. To preseve the original + # logic we also read in the raw unaltered version of the logic and stage the content + # to write the final logic. + logic = LibraryIO.parseLibraryLogicFile(logicFileName) + rawLogic = LibraryIO.rawLibraryLogic(logicFileName) + + # If we cannot read the logic file then skip it + if rawLogic == None or logic == None: + printExit("Error reading the file: %s. skipping." % logicFileName) + + (versionStringR, scheduleNameR, architectureNameR, deviceNamesR, problemTypeStateR,\ + solutionStatesR, indexOrderR, exactLogicR, rangeLogicR, otherFieldsR) =\ + rawLogic + + (_, _, problemType, solutionsForSchedule, \ + _, _, _, _, _) = logic + + copyfile(logicFileName, localLogicFilePath) + createLibraryForBenchmark(localLogicPath, libPath, currentPath) + + exactList = [] + + solutionSummationSizes = [32,64,96,128,256,512,1024,2048,4096,8192,16384] + + for K in solutionSummationSizes: + e = {"Exact" : [8192, 4096, 1, K]} + exactList.append(e) + + libraryPath = libPath + clientBuildDir = os.path.join(outputPath, "client") + problemTypeObj = problemType.state + + problemSizes = ProblemSizes(problemTypeObj, exactList) + dataPath = ensurePath(os.path.join(outputPath, logicFileStem, "data")) + configFilePath = ensurePath(os.path.join(outputPath, logicFileStem, "configs")) + dataFilePath = os.path.join(dataPath, "benchmark.csv") + configFile = os.path.join(configFilePath, "ClientParameters.ini") + scriptPath = ensurePath(os.path.join(outputPath, logicFileStem, "script")) + + ClientWriter.CreateBenchmarkClientParametersForSizes(libraryPath, problemSizes, dataFilePath, configFile, problemTypeObj) + ClientWriter.runNewClient(scriptPath, configFile, clientBuildDir) + + tensileLibraryFile = os.path.join(libPath, "library", "TensileLibrary.yaml") + + stream = open(tensileLibraryFile, "r") + tensileLibrary = yaml.load(stream, yaml.SafeLoader) + stream.close() + + libSolutions = tensileLibrary["solutions"] + + libSolutionNames = [] + for s in libSolutions: + kernelName=s["name"] + libSolutionNames.append(kernelName) + + working_data=pd.read_csv(dataFilePath).rename(str.strip,axis='columns') + + index_keys = working_data.SizeL.unique() + solutionsDF = working_data.filter(like='Cij') + + perf_max = solutionsDF.max().max().item() + + solutionIndex = 0 + for s_stateR, kernelName in zip(solutionStatesR, libSolutionNames): + solutionIndex += 1 + perf_raw = working_data[kernelName] + perf = (1000*index_keys) / perf_raw + model = np.polyfit(x=index_keys, y=perf, deg=1) + slope = model[0].item() + intercept = model[1].item() + linearModel = {"slope": slope, "intercept": intercept, "max": perf_max} + s_stateR["LinearModel"] = deepcopy(linearModel) + + rawLogicData = [] + rawLogicData.append(deepcopy(versionStringR)) + rawLogicData.append(deepcopy(scheduleNameR)) + rawLogicData.append(deepcopy(architectureNameR)) + rawLogicData.append(deepcopy(deviceNamesR)) + rawLogicData.append(deepcopy(problemTypeStateR)) + rawLogicData.append(deepcopy(solutionStatesR)) + rawLogicData.append(deepcopy(indexOrderR)) + rawLogicData.append(deepcopy(exactLogicR)) + rawLogicData.append(deepcopy(rangeLogicR)) + for idx in range(0, len(otherFieldsR)): + rawLogicData.append(deepcopy(otherFieldsR[idx])) + + localFinalLogic = os.path.join(finalPath, logicFileBaseName) + + yamlWriter = LibraryIO.YAMLWriter() + yamlWriter.write(localFinalLogic, rawLogicData) + + outputFinal = ensurePath(os.path.join(outputPath, "final")) + finalLogic = os.path.join(outputFinal, logicFileBaseName) + + copyfile(localFinalLogic, finalLogic) diff --git a/tensilelite/Tensile/Hardware.py b/tensilelite/Tensile/Hardware.py new file mode 100644 index 0000000000..0c10b7fb00 --- /dev/null +++ b/tensilelite/Tensile/Hardware.py @@ -0,0 +1,79 @@ +################################################################################ +# Copyright 2019-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from . import Properties +from .TensileInstructions import getGfxName +import copy + +class HardwarePredicate(Properties.Predicate): + @classmethod + def FromISA(cls, isa): + gfxArch = getGfxName(isa) + return cls("AMDGPU", value=cls("Processor", value=gfxArch)) + + @classmethod + def FromHardware(cls, isa, cuCount=None): + gfxArch = getGfxName(isa) + if cuCount == None: + return cls("AMDGPU", value=cls("Processor", value=gfxArch)) + else: + return cls("AMDGPU", value=cls.And([cls("Processor", value=gfxArch), + cls("CUCount", value=cuCount)])) + + def __lt__(self, other): + # Use superclass logic for TruePreds + if other.tag == 'TruePred' or self.tag == 'TruePred': + return super().__lt__(other) + + # Compute unit counts are embedded as 'And' with + # 'Processor' and 'ComputeUnitCount' as children + if self.value.tag == 'And': + myAndPred = self.value + myProcPred = next(iter(x for x in myAndPred.value if x.tag == "Processor"), None) + myCUPred = next(iter(x for x in myAndPred.value if x.tag == "CUCount"), None) + myCUCount = myCUPred.value if myCUPred != None else 0 + else: + myProcPred = self.value + myCUCount = 0 + + if other.value.tag == 'And': + otherAndPred = other.value + otherProcPred = next(iter(x for x in otherAndPred.value if x.tag == "Processor"), None) + otherCUPred = next(iter(x for x in otherAndPred.value if x.tag == "CUCount"), None) + otherCUCount = otherCUPred.value if otherCUPred != None else 0 + else: + otherProcPred = other.value + otherCUCount = 0 + + # If CU properties are empty, then compare processor predicates + if myCUCount == otherCUCount == 0: + # Make sure that we have valid processor preds + assert myProcPred != None and otherProcPred != None, "Missing processor predicate" + assert myProcPred.tag == otherProcPred.tag == "Processor", "Invalid processor predicate" + + # Downgrade to base class so that we don't recurse + myProcPredCopy = copy.deepcopy(myProcPred) + otherProcPredCopy = copy.deepcopy(otherProcPred) + myProcPredCopy.__class__ = otherProcPredCopy.__class__ = Properties.Predicate + return myProcPredCopy < otherProcPredCopy + + # Higher priority given to higher CU count + return myCUCount > otherCUCount diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py new file mode 100644 index 0000000000..0305fa6ada --- /dev/null +++ b/tensilelite/Tensile/KernelWriter.py @@ -0,0 +1,3256 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from . import Common +from .TensileInstructions import Item, TensileInstructions, slash50, replaceHolder, \ + KernelBody, Module, StructuredModule, TextBlock, \ + TensileInstructionsPass +from .TensileInstructions.Instructions import * +from .TensilePass import TensilePass, TensilePassOptions +from .Common import globalParameters, CHeader, roundUp, Backup, print2, printExit +from .Component import Component +from .CustomKernels import isCustomKernelConfig +from .SolutionStructs import Solution + +import abc +import os +import subprocess +import copy +from typing import NamedTuple + +################################################################################ +# Kernel Writer +################################################################################ +class KernelWriter(metaclass=abc.ABCMeta): + #__metaclass__=abc.ABCMeta + + ############################################################################## + # Init + ############################################################################## + def __init__( self, kernelMinNaming, kernelSerialNaming ): + self.kernelMinNaming = kernelMinNaming + self.kernelSerialNaming = kernelSerialNaming + self.overflowedResources = 0 + + @property + def asmCaps(self): + """ + Assembler capabilities for the current ISA version. + """ + return globalParameters["AsmCaps"][self.version] + + @property + def archCaps(self): + """ + Architectural capabilities for the current ISA version. + """ + return globalParameters["ArchCaps"][self.version] + + @property + def asmBugs(self): + """ + Assembler bugs for the current ISA version. + """ + return globalParameters["AsmBugs"][self.version] + + @property + def globalParams(self): + """ + Global parameters for current configuration. + """ + return globalParameters + + ############################################################################## + # makeSchedule: Schedule work into interations. + + # Tensile uses a two-level scheduler. This the first-level, which + # schedules global reads, global incs, and local writes into iteration. + # Then makeSubIterSchedule schedules the instructions within the iteration. + # + # Inputs: + # localWriteEndIter: loop iteration where last writes should be inserted + # If scheduleLocalWrite=0, all writes will be be placed in this iteration. + # If scheduleLocalWrite=1, the scheduler will work backwards from this + # iteration. + # + # Outputs: + # self.unrollLoopHeaderCode: + # - Code module that should be added into the unroll loop header + # In unscheduled code this contains global loads and global address increment + # self.perIterGlobalReadCode[], self.perIterLocalWriteCode[] + # - List indexed by unroll iteration. + # Each entry in the list is a code module that should be added into that iteration. + # May be None, indicating no extra code for that iteration + # self.grEndMfmaIndex + # self.lwStartMfmaIndex + # self.lwEndMfmaIndex + # self.barrierMfmaIndex + # self.numMfmaForNextLoopLR + # This routine is responsible for setting the schedule including determining + # that all necessary dependency are met. The driver code in kernelBody + # blindly follows the plan set in unrollLoopHeaderCode and perIterCode + ############################################################################## + def makeSchedule(self, kernel, tensorParametersA, tensorParametersB, localWriteEndIter, uDu=0, skipGlobalReadInc=False, firstIter=False, lastLoop=False, lastLc=False): + + currentIsa = globalParameters["CurrentISA"] + maxVmcnt = globalParameters["AsmCaps"][currentIsa]["MaxVmcnt"] + + self.unrollLoopHeaderCode = Module() + # schedule of work for each local_read iteration: + self.perIterGlobalReadCode = [ Module() for i in range (kernel["LoopIters"]) ] + self.perIterLocalWriteCode = [ Module() for i in range (kernel["LoopIters"]) ] + if lastLc: + self.perIterLocalWriteCodeNGLL = [ Module() for i in range (kernel["LoopIters"]) ] + self.perIterLocalWriteCanSkip = [ 0 for i in range (kernel["LoopIters"]) ] + self.perIterGlobalReadCodeDTV = [ Module() for i in range (kernel["LoopIters"]) ] # global read for DirectToVgpr + assert([item.name for item in self.globalReadIncrements.itemList] == ['globalReadIncrementA', 'globalReadIncrementB']) + + globalReadIncACode = self.globalReadIncrements.findNamedItem("globalReadIncrementA") + globalReadIncBCode = self.globalReadIncrements.findNamedItem("globalReadIncrementB") + + if uDu < kernel["DepthULdsDivisor"] - 1 and kernel.enabledSplitLDS and kernel["PrefetchGlobalRead"] \ + or skipGlobalReadInc: + globalReadIncACode = Module() + globalReadIncBCode = Module() + + grBackup = None + if uDu != kernel["DepthULdsDivisor"] - 2 and kernel.enabledSplitLDS: + # hack RAII object for auto restore + # withhold issuing global read codes until in the 2nd last subloop, meaning we empty the code + # modules in other subloops. + grBackup = Backup(self, globalReadACode = self.globalReadACode, globalReadBCode = self.globalReadBCode) + self.globalReadACode = StructuredModule() # empty + self.globalReadBCode = StructuredModule() # empty + + siaComponent = Component.SIA.find(self) + siaComponent.schedIntoIteration(self, kernel, tensorParametersA, tensorParametersB, \ + localWriteEndIter, uDu, firstIter, lastLoop, lastLc, maxVmcnt, globalReadIncACode, \ + globalReadIncBCode) + + if grBackup is not None: + del grBackup + + ############################################################################## + # Schedule work into the each unroll loop iteration + # localReadCode is the local reads for this loop iteration + # (returned by localReadDo). The instructions in localReadCode + # will retain their relative order, but may be interleaved + # with instructions from otherCode. + + # globalReadCode is the 'other' buffer loads and addr increments + # localWriteCode is the 'other' local writes + # to schedule in with the ds reads. The instructions + # will retain their relative order, but may be interleaved + # with instructions from localReadCode. + + # pointerCode contains local pointer changes (if needed) + # waitCode contains s_waitcnt before macs. + # - Cannot be "" or None + # - may be empty Module if not waiting is desired (perhaps for debug) + # - may be multiple instructions (ConservativeWaitCnt) + # - typically is a single SWaitCnt. This routine will + # modify the lgkmcnt to account for any scheduling decisions. + # If this is not desired, add the waitCnt to pointerCode and + # set waitCode to an empty module + # macIterCode contains the mac iters. May be a macro call. + # + # returns: a Module with the combined, optimally scheduled + # localReadCode + otherCode + ############################################################################## + def makeSubIterSchedule(self, kernel, tPA, tPB, localReadCode, iteration, pointerLWCode, pointerLRCode, waitCode, macIterCode, \ + waitLWCode = Module(), syncCode = Module(), packCode = Module(), isDTVodd = False, NLLlast = False): + + iterCode = Module() + globalReadCode = copy.deepcopy(self.perIterGlobalReadCode[iteration]) + globalReadCodeDTV = self.perIterGlobalReadCodeDTV[iteration] + origLenGlobalReadCodeDTV = len(list(self.perIterGlobalReadCodeDTV[iteration].items())) + localWriteCode = self.perIterLocalWriteCode[iteration] + isBarrier = kernel["LoopIters"] - self.numItersPLR + hasLocalRead = localReadCode.countType(LocalReadInstruction) + # Default schedule is other, local reads, then local writes: + if self.scheduleIterAlg==0: + # simple schedule, just add the modules in-order + iterCode.add(globalReadCode) + iterCode.add(waitLWCode) + iterCode.add(syncCode) + iterCode.add(localReadCode) + iterCode.add(localWriteCode) + iterCode.add(pointerLWCode) + iterCode.add(pointerLRCode) + iterCode.add(waitCode) + iterCode.add(packCode) + iterCode.add(macIterCode) + elif self.scheduleIterAlg == 1: + iterCode.add(waitLWCode) + iterCode.add(syncCode) + #import pdb + #pdb.set_trace() + # simple algorithm - do half the reads first: + readsToSchedule = localReadCode.countType(LocalReadInstruction) / 2 + #localReadCode.prettyPrint() + readItems = localReadCode.flatitems() + while readItems: + item = readItems.pop(0) + #print "readsToSchedule=", readsToSchedule, "item=", item + iterCode.add(item) + readsThisItem = item.countType(LocalReadInstruction) + if readsThisItem: + assert readsThisItem==1, "Scheduler assumes 1 read per item" + readsToSchedule = readsToSchedule - 1 + if readsToSchedule == 0: + break + + iterCode.add(globalReadCode) + + # add rest of the reads here + for item in readItems: + iterCode.add(item) + + #move down write to be the last + iterCode.add(localWriteCode) + # tack on the pointer and mac code: + iterCode.add(pointerLWCode) + iterCode.add(pointerLRCode) + iterCode.add(waitCode) + iterCode.add(packCode) + iterCode.add(macIterCode) + elif self.scheduleIterAlg == 2: + # SIA2 use only 1 iteration and separate compute and fetch by raising compute priority + # 2 workgroup interleave, while WG0/WG1 doing compute, WG1/WG0 doing fetch + # EPS need to be 1, or valu instruction will break interleave + iterCode.add(globalReadCode) + iterCode.add(waitLWCode) + iterCode.add(syncCode) + iterCode.add(localReadCode) + iterCode.add(waitCode) + + # interleave pack code + # BF16 or FP16: each packCode is for one 32-bit reg, 1 packing inst: half-to-single x1 + # INT8 : each packCode is for one 32-bit regs, 3 packing inst: byte-to-half x2 + half-to-single x1 + instPerRegPack = 1 / kernel["ProblemType"]["DataType"].numRegisters() - 1 + instPerPack = int(kernel["MIInputPerThread"] * kernel["ProblemType"]["DataType"].numRegisters() * instPerRegPack) + packItems = [] + for iui in range(kernel["InnerUnroll"]): + packINtems = [ [] for j in range(max(self.numReadsIterCoalescedA,self.numReadsIterCoalescedB)) ] + packA = packCode.findNamedItem("packA_I%s"%(iui)) + packB = packCode.findNamedItem("packB_I%s"%(iui)) + # In case localReadDo not generate pack Module + # and findNamedItem will return None type + # TODO: let all type have pack Module + if not packA: + packA = Module() + packAItems = packA.flatitems() + if not packB: + packB = Module() + packBItems = packB.flatitems() + if packAItems: + for j in range(self.numReadsIterCoalescedA): + for n in range(instPerPack): + packINtems[j].append(packAItems.pop(0)) + if packBItems: + for j in range(self.numReadsIterCoalescedB): + for n in range(instPerPack): + packINtems[j].append(packBItems.pop(0)) + while packAItems: + for j in range(self.numReadsIterCoalescedA): + for n in range(instPerPack): + packINtems[j].append(packAItems.pop(0)) + while packBItems: + for j in range(self.numReadsIterCoalescedB): + for n in range(instPerPack): + packINtems[j].append(packBItems.pop(0)) + for j in range(max(self.numReadsIterCoalescedA,self.numReadsIterCoalescedB)): + packItems += packINtems.pop(0) + + macIterItem = macIterCode.flatitems() + # pop the first code which is s_nop 1 for packing + item = macIterItem.pop(0) + + numMfmaPerIter = self.numMfmaPerIter + curPackIdx = 0 + packAIdx = 0 + packBIdx = 0 + + for i in range(numMfmaPerIter): + if packItems: + # how many pack have to be done + # calculate the data index of this mfma used for A and B + # if i // kernel["MIWaveTile"][0]==0, mfma will use new A (need to take iu into account) + # if i % kernel["MIWaveTile"][0]==0, mfma will use new B + packAIdx += instPerPack if i//(kernel["MIWaveTileA"]+kernel["MIWaveTileA"]*kernel["MIWaveTileB"]*(i//(kernel["MIWaveTileA"]*kernel["MIWaveTileB"]))) == 0 else 0 + packBIdx += instPerPack if i % kernel["MIWaveTileA"] == 0 else 0 + # blockWidth < 1, means 0.5 or 0.25 (BF,H,Int8) + packAIdx = packAIdx if tPA["localReadInstruction"].blockWidth < 1 else 0 + packBIdx = packBIdx if tPB["localReadInstruction"].blockWidth < 1 else 0 + numPack = (packAIdx + packBIdx) + iterCode.addComment0("pack scheduling: packAIdx:%u, packBIdx:%u" %(packAIdx,packBIdx)) + # we put 2 pack in each mfma, "2" means A & B + if packItems: + for j in range(instPerPack): + iterCode.add(packItems.pop(0)) + curPackIdx += 1 + if packItems: + for j in range(instPerPack): + iterCode.add(packItems.pop(0)) + curPackIdx += 1 + # since packed register need to wait 2 quad cycle to finish packing + # we insert pack instruction if we can, or s_nop + while curPackIdx < numPack+2: + if packItems: + for j in range(instPerPack): + iterCode.add(packItems.pop(0)) + curPackIdx += 1 + else: + iterCode.add(SNop(waitState=0, comment="VALU packing writes to be consumed by matrix instruction")) + curPackIdx += 1 + if i == 0: + if not packItems: + tmpVgpr = self.vgprPool.checkOut(1) + iterCode.add(VMovB32(dst="v%u"%(tmpVgpr), src="0x0", comment="valu operation to have different priority")) + self.vgprPool.checkIn(tmpVgpr) + iterCode.add(SSetPrior(prior=3, comment="Raise priority while processing macs")) + item = macIterItem.pop(0) + iterCode.add(item) + + iterCode.add(SSetPrior(prior=1, comment="Raise priority while processing macs")) + if kernel["1LDSBuffer"]: + barrier = Module() + barrier.addComment0("1 LDS buffer: read-sync-write") + barrier.add(SWaitCnt(lgkmcnt=0, comment="")) + barrier.add(SBarrier()) + iterCode.add(barrier) + iterCode.add(localWriteCode) + iterCode.add(pointerLWCode) + iterCode.add(pointerLRCode) + iterCode.add(SSetPrior(prior=2, comment="Raise priority while processing macs")) + pass + elif self.scheduleIterAlg == 3: + iterCode.addComment0(" grEndMfmaIndex:%u, lwStartMfmaIndex:%u, lwEndMfmaIndex:%u " %(self.grEndMfmaIndex,self.lwStartMfmaIndex,self.lwEndMfmaIndex)) + iterCode.addComment0(" numMfmaForLR:%u, barrierMfmaIndex:%u " %(self.numMfmaForNextLoopLR,self.barrierMfmaIndex)) + ##### + # Prepare and Assign parameter + #### + if iteration == 0: + self.localReadsVacancy = [] + self.localReadsWait = [ [] for j in range(kernel["LoopIters"])] + self.localReadsWait[iteration] = waitCode + numMfmaPerIter = self.numMfmaPerIter + isBarrier = kernel["LoopIters"] - self.numItersPLR + writeItems = list(localWriteCode.items()) + macIterItems = macIterCode.flatitems() + skipLocalWriteWaitcnt = 0 + localReadsWaitcnt = 0 + curPackIdx = 0 + packAIdx = 0 + packBIdx = 0 + + ##### + # Prepare localReadCode + #### + localReadCodeAB = Module() + for iui in range(kernel["InnerUnroll"]): + localReadCodeA = localReadCode.findNamedItem("LocalReadDoA_I%s"%(iui)) + localReadCodeB = localReadCode.findNamedItem("LocalReadDoB_I%s"%(iui)) + # In case localReadDo not generate localReadCode Module + # and findNamedItem will return None type + # TODO: findNamedItem return Module() if not found + if not localReadCodeA: + localReadCodeA = Module() + if not localReadCodeB: + localReadCodeB = Module() + if localReadCodeA.items(): + localReadCodeAB.add(localReadCodeA.items().pop(0)) + if localReadCodeB.items(): + localReadCodeAB.add(localReadCodeB.items().pop(0)) + while localReadCodeA.items(): + localReadCodeAB.add(localReadCodeA.items().pop(0)) + while localReadCodeB.items(): + localReadCodeAB.add(localReadCodeB.items().pop(0)) + localReadItems = localReadCodeAB.flatitems() + localReadItemsThisLoop = localReadItems if iteration < isBarrier else [] + localReadItemsNextLoop = localReadItems if iteration >= isBarrier else [] + + ##### + # Prepare pack Code for B: + # since the mfma reuse B first => for A: mfma[A][B] + # we need 1 vector A and 1 vector B for first mfma + # then we prepare remaining A, then remaining B + # BF16 or FP16: each packCode is for one 32-bit reg, 1 packing inst: half-to-single x1 + # INT8 : each packCode is for one 32-bit regs, 3 packing inst: byte-to-half x2 + half-to-single x1 + #### + instPerRegPack = 1 / kernel["ProblemType"]["DataType"].numRegisters() - 1 + instPerPack = int(kernel["MIInputPerThread"] * kernel["ProblemType"]["DataType"].numRegisters() * instPerRegPack) + packItems = [] + for iui in range(kernel["InnerUnroll"]): + packINtems = [ [] for j in range(max(self.numReadsIterCoalescedA,self.numReadsIterCoalescedB)) ] + packA = packCode.findNamedItem("packA_I%s"%(iui)) + packB = packCode.findNamedItem("packB_I%s"%(iui)) + # In case localReadDo not generate pack Module + # and findNamedItem will return None type + # TODO: let all type have pack Module + if not packA: + packA = Module() + packAItems = packA.flatitems() + if not packB: + packB = Module() + packBItems = packB.flatitems() + if packAItems: + for j in range(self.numReadsIterCoalescedA): + for n in range(instPerPack): + packINtems[j].append(packAItems.pop(0)) + if packBItems: + for j in range(self.numReadsIterCoalescedB): + for n in range(instPerPack): + packINtems[j].append(packBItems.pop(0)) + while packAItems: + for j in range(self.numReadsIterCoalescedA): + for n in range(instPerPack): + packINtems[j].append(packAItems.pop(0)) + while packBItems: + for j in range(self.numReadsIterCoalescedB): + for n in range(instPerPack): + packINtems[j].append(packBItems.pop(0)) + for j in range(max(self.numReadsIterCoalescedA,self.numReadsIterCoalescedB)): + packItems += packINtems.pop(0) + + # remove s_nop for packing + # we will add s_nop if needed + if macIterItems: + macIterItems.pop(0) + + #### + # scheduled local read to previous iterations + #### + if self.numVgprBuffer >= kernel["LoopIters"]: + for vacancy in self.localReadsVacancy: + # {"items","latencyLeft","atIter","atMfmaIndex","noReadsAtThisIter"} + for localRead in list(localReadItemsThisLoop): + if vacancy["latencyLeft"] > localRead.issueLatency() * 2: + if not localRead.readToTempVgpr: + vacancy["latencyLeft"] -= localRead.issueLatency() * 2 + vacancy["items"].add(localRead) + localReadItemsThisLoop.remove(localRead) + if vacancy["atMfmaIndex"] > self.lwStartMfmaIndex - 1 and kernel["1LDSBuffer"]: + self.overflowedResources = 5 + # update waitCnt + if self.numItersPLR: + for readsIter in range(vacancy["atIter"], iteration + self.numItersPLR): + if (vacancy["atMfmaIndex"] % numMfmaPerIter == 0 or readsIter != vacancy["atIter"]) and \ + (vacancy["noReadsAtThisIter"] or readsIter <= vacancy["atIter"] + self.numItersPLR): + if isinstance(self.localReadsWait[readsIter], SWaitCnt): + self.localReadsWait[readsIter].lgkmcnt += 1 + # This line is added for backward compatibility + self.localReadsWait[readsIter].vscnt = self.localReadsWait[readsIter].vmcnt \ + if self.localReadsWait[readsIter].lgkmcnt != -1 and \ + self.localReadsWait[readsIter].vmcnt != -1 and \ + self.archCaps["SeparateVscnt"] else -1 + else: + # make sure the localread sequence remain the same + vacancy["latencyLeft"] = 0 + numReadsInst = len(localReadItemsThisLoop) if iteration < isBarrier else len(localReadItemsNextLoop) + + for i in range(numMfmaPerIter): + mfmaIndex = iteration * numMfmaPerIter + i + iterCode.addComment0(" mfmaIndex:%u " %(mfmaIndex)) + + #### + # scheduled local read + #### + readLeft = numReadsInst + latencyLeft = self.miLatencyLeft + # with PrefetchLocalRead, localreads can interleave with mfma + if self.numItersPLR and iteration < isBarrier: + # take ds_write into account to schedule ds_read, assume A and B localwrite have same width (TLDS=1) + if (mfmaIndex >= self.lwStartMfmaIndex) and not globalReadCode.countType(GlobalReadInstruction): + for j in range(min(len(writeItems),self.numLocalWriteModPerMfma)): + if writeItems[j].countType(LocalWriteInstruction): + latencyLeft -= (tPA["localWriteInstruction"].issueLatency*2) + readLeftLROPT = 0 + for j in range(len(localReadItemsThisLoop)): + latencyLeft -= localReadItemsThisLoop[j].issueLatency()*2 + readLeftLROPT += 1 if latencyLeft >= 0 else 0 + # at least 1 instruction + readLeftLROPT = max(readLeftLROPT,1) + # evenly schedule localread with each mfma + readLeftLREven = numReadsInst // numMfmaPerIter + if (numReadsInst % (numMfmaPerIter)) > i: + readLeftLREven += 1 + # we want no localreads at first mfma + if (iteration == 0) and numMfmaPerIter != 1: + numMfmaForLR = numMfmaPerIter - 1 + if i < numMfmaPerIter - numMfmaForLR: + readLeftLREven = 0 + readLeftLROPT = 0 + # rest mfma help to schedule those localReads + else: + readLeftLREven = numReadsInst // (numMfmaPerIter-1) + if (numReadsInst % (numMfmaPerIter-1)) >= i: + readLeftLREven += 1 + # if there are too many localreads, change strategy to even. + readLeft = max(readLeftLREven,readLeftLROPT) + if not self.numItersPLR and iteration < isBarrier: + for j in range(len(localReadItemsThisLoop)): + latencyLeft -= localReadItemsThisLoop[j].issueLatency()*2 + # if start to schedule localwrite, but still have localreads not scheduled yet, + # reject to use 1LDSB, since it will write and read same lds buffer at same time. + # TODO: force to schedule all remaining localreads before start to schedule localwrite. + if mfmaIndex >= self.lwStartMfmaIndex and mfmaIndex <= max(self.lwEndMfmaIndex,self.barrierMfmaIndex) and \ + localReadItemsThisLoop and localWriteCode.countType(LocalWriteInstruction) and kernel["1LDSBuffer"]: + self.overflowedResources = 5 + # DirectToVgpr case, localReadItemsThisLoop and localWriteCode.countType(LocalWriteInstruction) do not satisfy at the same time. + # However, it is still invaid if localReadItemsThisLoop exists when mfmaIndex > lwStartMfmaIndex + elif (kernel["DirectToVgprA"] or kernel["DirectToVgprB"]) and \ + mfmaIndex > self.lwStartMfmaIndex and mfmaIndex <= max(self.lwEndMfmaIndex,self.barrierMfmaIndex) and \ + localReadItemsThisLoop and kernel["1LDSBuffer"]: + self.overflowedResources = 5 + for j in range(readLeft): + if localReadItemsThisLoop: + item = localReadItemsThisLoop.pop(0) + iterCode.add(item) + if (i == 0): + localReadsWaitcnt += 1 + if not localReadItemsThisLoop and latencyLeft > 0 and iteration < isBarrier and \ + not(mfmaIndex > self.lwStartMfmaIndex and kernel["1LDSBuffer"]): + item = Module() + item.addComment0("localReadsVacancy: latencyLeft %d"%(latencyLeft)) + iterCode.add(item) + self.localReadsVacancy.append({ "items": item, \ + "latencyLeft": latencyLeft, \ + "atIter": iteration, \ + "atMfmaIndex": mfmaIndex, \ + "noReadsAtThisIter": numReadsInst == 0, \ + }) + + #### + # scheduled global read + #### + for j in range(self.numGlobalReadInsPerMfma): + if globalReadCode.items(): + loadModule = globalReadCode.items().pop(0) + if isDTVodd: + # need to swap Vgpr set for odd code + loadModule = self.flipVregSetForDirectToVgprInGlobalRead(kernel, loadModule) + iterCode.add(loadModule) + # schedule remaining globalReadInst + if mfmaIndex == self.grEndMfmaIndex: + while globalReadCode.items() and \ + (globalReadCode.countType(GlobalReadInstruction) or kernel["PrefetchGlobalRead"] == 2): + loadModule = globalReadCode.items().pop(0) + if isDTVodd: + # need to swap Vgpr set for odd code + loadModule = self.flipVregSetForDirectToVgprInGlobalRead(kernel, loadModule) + iterCode.add(loadModule) + # schedule remaining globalReadIncInst + if i == numMfmaPerIter - 1: + while globalReadCode.items(): + loadModule = globalReadCode.items().pop(0) + if isDTVodd: + # need to swap Vgpr set for odd code + loadModule = self.flipVregSetForDirectToVgprInGlobalRead(kernel, loadModule) + iterCode.add(loadModule) + + #### + # scheduled local write + #### + if kernel["1LDSBuffer"] and mfmaIndex == self.lwStartMfmaIndex - 1: + barrier = Module() + barrier.addComment0("1 LDS buffer: read-sync-write") + barrier.add(SWaitCnt(lgkmcnt=0, comment="")) + barrier.add(SBarrier()) + iterCode.add(barrier) + + if kernel["StorePriorityOpt"]: + flagInsert = False + if kernel["PrefetchGlobalRead"] == 2: + lwStartOffset = 0 + if kernel["DirectToLds"]: + lwStartOffset = 2 + # if (mfmaIndex == self.lwStartMfmaIndex or mfmaIndex == self.barrierMfmaIndex+2): + if (mfmaIndex == self.lwStartMfmaIndex + lwStartOffset or mfmaIndex == self.barrierMfmaIndex+1) : + flagInsert = True + elif kernel["PrefetchGlobalRead"] == 1 and numMfmaPerIter >= 4: + # this setting is good for fixed clock, but not good for auto clock + #if (mfmaIndex == self.grEndMfmaIndex or mfmaIndex == self.barrierMfmaIndex+1) : + withGL = (not NLLlast) + withDTLload = kernel["DirectToLds"] and withGL + startIndex = 0 if withDTLload else 1 + if (mfmaIndex == startIndex or withGL and mfmaIndex == self.barrierMfmaIndex+1): + flagInsert = True + if flagInsert: + iterCode.add(SSetPrior(prior=3, comment="store optimization")) + + if (mfmaIndex >= self.lwStartMfmaIndex): + for j in range(self.numLocalWriteModPerMfma): + # in case there are localWrite and globalread in same iteration + # we need to make sure globalRead before localWrite + if writeItems and not globalReadCode.countType(GlobalReadInstruction): + writeItem = writeItems.pop(0) + iterCode.add(writeItem) + # if there is localWrite at first mfma, need to skip it in waitcnt. + if i == 0: + skipLocalWriteWaitcnt += writeItem.countType(LocalWriteInstruction) + if not localReadItemsThisLoop: + self.perIterLocalWriteCanSkip[iteration] += writeItem.countType(LocalWriteInstruction) + if mfmaIndex == self.lwEndMfmaIndex: + while writeItems: + writeItem = writeItems.pop(0) + # generate all remaining pre code before the first Store C + iterCode.add(writeItem) + if i == 0: + skipLocalWriteWaitcnt += writeItem.countType(LocalWriteInstruction) + if not localReadItemsThisLoop: + self.perIterLocalWriteCanSkip[iteration] += writeItem.countType(LocalWriteInstruction) + + #### + # scheduled pointer + #### + if mfmaIndex == self.lwEndMfmaIndex: + iterCode.add(pointerLWCode) + if i == numMfmaPerIter - 1: + iterCode.add(pointerLRCode) + + #### + # scheduled sync + #### + if mfmaIndex == self.barrierMfmaIndex and self.numItersPLR: + iterCode.add(waitLWCode) + iterCode.add(syncCode) + + #### + # scheduled local read for next loop + # localReads for next loop should after barrier + #### + latencyLeft = self.miLatencyLeft + if self.numItersPLR and iteration >= isBarrier: + readLeftLROPT = 0 + for j in range(len(localReadItemsNextLoop)): + latencyLeft -= localReadItemsNextLoop[j].issueLatency()*2 + readLeftLROPT += 1 if latencyLeft >= 0 else 0 + # at least 1 instruction + readLeftLROPT = max(readLeftLROPT,1) + # evenly schedule localread with each mfma + readLeftLREven = numReadsInst // numMfmaPerIter + if (numReadsInst % (numMfmaPerIter)) > i: + readLeftLREven += 1 + # we want no localreads at barrier mfma + if (iteration == isBarrier) and numMfmaPerIter != 1: + numMfmaForLR = self.numMfmaForNextLoopLR + if i < numMfmaPerIter - numMfmaForLR: + readLeftLREven = 0 + readLeftLROPT = 0 + # rest mfma help to schedule those localReads + else: + readLeftLREven = numReadsInst // (numMfmaPerIter-1) + if (numReadsInst % (numMfmaPerIter-1)) >= i: + readLeftLREven += 1 + # if there are too many localreads, change strategy to even. + readLeft = max(readLeftLREven,readLeftLROPT) + for j in range(readLeft): + if localReadItemsNextLoop: + item = localReadItemsNextLoop.pop(0) + iterCode.add(item) + if (i == 0): + localReadsWaitcnt += 1 + + #### + # scheduled wait localReads + #### + if i == 0: + iterCode.add(waitCode) + + #### + # scheduled pack + #### + if packItems: + # how many pack have to be done + # calculate the data index of this mfma used for A and B + # if i // kernel["MIWaveTile"][0]==0, mfma will use new A (need to take iu into account) + # if i % kernel["MIWaveTile"][0]==0, mfma will use new B + packAIdx += instPerPack if i//(kernel["MIWaveTileA"]+kernel["MIWaveTileA"]*kernel["MIWaveTileB"]*(i//(kernel["MIWaveTileA"]*kernel["MIWaveTileB"]))) == 0 else 0 + packBIdx += instPerPack if i % kernel["MIWaveTileA"] == 0 else 0 + # blockWidth < 1, means 0.5 or 0.25 (BF,H,Int8) + packAIdx = packAIdx if tPA["localReadInstruction"].blockWidth < 1 else 0 + packBIdx = packBIdx if tPB["localReadInstruction"].blockWidth < 1 else 0 + numPack = (packAIdx + packBIdx) + iterCode.addComment0("pack scheduling: packAIdx:%u, packBIdx:%u" %(packAIdx,packBIdx)) + # we put 2 pack in each mfma + if packItems: + for j in range(instPerPack): + iterCode.add(packItems.pop(0)) + curPackIdx += 1 + if packItems: + for j in range(instPerPack): + iterCode.add(packItems.pop(0)) + curPackIdx += 1 + # since packed register need to wait 2 quad cycle to finish packing + # we insert pack instruction if we can, or s_nop + while curPackIdx < numPack+2: + if packItems: + for j in range(instPerPack): + iterCode.add(packItems.pop(0)) + curPackIdx += 1 + else: + iterCode.add(SNop(waitState=0, comment="VALU packing writes to be consumed by matrix instruction")) + curPackIdx += 1 + if i == numMfmaPerIter - 1: + while packItems: + iterCode.add(packItems.pop(0)) + + #### + # scheduled mfma + #### + iterCode.add(macIterItems.pop(0) if macIterItems else Module()) + + #### + # scheduled global read for DirectToVgpr (PGR=2 only) + #### + numLoadVgpr = len(list(globalReadCodeDTV.items())) + if numLoadVgpr > 0: + interval = roundUp(numMfmaPerIter / origLenGlobalReadCodeDTV) + tileIndex = 0 if kernel["DirectToVgprA"] else 1 + if (kernel["MIWaveTile"][tileIndex] // kernel["VectorWidth"]) > 1: + if kernel["ProblemType"]["DataType"].isDoubleComplex(): + # adjustment for double complex + # limit the max of interval up to 4 if (kernel["MIWaveTile"][0] // kernel["VectorWidth"]) > 1 + interval = min(4, interval) + elif kernel["ProblemType"]["DataType"].isDouble(): + # adjustment for double + # in this case, interval must be 1 to avoid overwritting vreg by global read + interval = 1 + # DirectToVgprA + TLU=False + VW > 1 case, need to use interval = 1 + if kernel["DirectToVgprA"] and (not kernel["ProblemType"]["TLUA"]) and kernel["VectorWidth"] > 1: + interval = 1 + # if number of mfma after self.grEndMfmaIndex is smaller than numMfmaPerIter, we need to use smaller interval to insert DTV load. + # this is to ensure DTV load is generated after lwStartMfmaIndex + intervalAfterGrEnd = kernel["LoopIters"] * numMfmaPerIter - self.lwStartMfmaIndex + intervalMfma = min(numMfmaPerIter, intervalAfterGrEnd) + numInstToInsert = roundUp(origLenGlobalReadCodeDTV / intervalMfma) + remainingTimesToInsert = roundUp(numLoadVgpr / numInstToInsert) + insertMfmaIndex = kernel["LoopIters"] * numMfmaPerIter - 1 - interval * (remainingTimesToInsert - 1) + # avoid insertMfmaIndex getting smaller than (kernel["LoopIters"] - 1) * numMfmaPerIter + insertMfmaIndex = max(insertMfmaIndex, (kernel["LoopIters"] - 1) * numMfmaPerIter) + if mfmaIndex == insertMfmaIndex: + for i in range(min(numLoadVgpr, numInstToInsert)): + loadDTVModule = globalReadCodeDTV.items().pop(0) + if isDTVodd: + # need to swap Vgpr set for odd code + loadDTVModule = self.flipVregSetForDirectToVgprInGlobalRead(kernel, loadDTVModule) + iterCode.add(loadDTVModule) + + if kernel["StorePriorityOpt"]: + flagInsert = False + if kernel["PrefetchGlobalRead"] == 2: + # if (mfmaIndex == self.barrierMfmaIndex or mfmaIndex == (kernel["LoopIters"] * numMfmaPerIter - 1)): + if (mfmaIndex == self.barrierMfmaIndex - 1 or (not NLLlast) and mfmaIndex == (kernel["LoopIters"] * numMfmaPerIter - 1)) : + flagInsert = True + elif kernel["PrefetchGlobalRead"] == 1 and numMfmaPerIter >= 4: + # this setting is good for fixed clock, but not good for auto clock + #if (mfmaIndex == mfmaIndex == self.barrierMfmaIndex - 1 or mfmaIndex == (kernel["LoopIters"] * numMfmaPerIter - 1)) : + insertPos1 = self.grEndMfmaIndex + if not kernel["NoLdsWriteCode"]: + insertPos1 = self.lwStartMfmaIndex - 1 + withGL = (not NLLlast) + if withGL and (mfmaIndex == insertPos1 or (not NLLlast) and mfmaIndex == (kernel["LoopIters"] * numMfmaPerIter - 1)) or \ + (not withGL) and mfmaIndex == (kernel["LoopIters"] * numMfmaPerIter // 2 - 1): + flagInsert = True + if flagInsert: + iterCode.add(SSetPrior(prior=0, comment="store optimization")) + else: + assert 0, "Unsupported scheduleIterAlg=%u"%self.scheduleIterAlg + + if isinstance(waitCode, SWaitCnt): + + # Set the waitCount, based on the new iter schedule + lgkmcnt = waitCode.lgkmcnt + localReads = 0 + localWrites = 0 + if kernel["EnableMatrixInstruction"]: + # dataAtIter : the data we wait is read at which iteration + # numReadsIter : in this loop, number of iteration we have read (data used in current loop) + dataAtIterA = iteration//self.numIterPerCoalescedReadA - self.numItersPLR + dataAtIterB = iteration//self.numIterPerCoalescedReadB - self.numItersPLR + numReadsIterA = min(iteration+1, kernel["LoopIters"]//self.numIterPerCoalescedReadA - self.numItersPLR) + numReadsIterB = min(iteration+1, kernel["LoopIters"]//self.numIterPerCoalescedReadB - self.numItersPLR) + skipReadsIterA = numReadsIterA - dataAtIterA - 1 if not dataAtIterA < max(dataAtIterA,dataAtIterB) else 0 + skipReadsIterB = numReadsIterB - dataAtIterB - 1 if not dataAtIterB < max(dataAtIterA,dataAtIterB) else 0 + # numPrefetchIter : in this loop, number of prefetch iteration we have read (data used in next loop) + # currently we have localReadA and localReadB if iteration >= isBarrier + # some case will not have localReads if PGR=0 or NoLoadLoop + # known bug: wider localread + numItersPLR>1 may have chance to fail. + numPrefetchIter = (iteration//(kernel["LoopIters"]-self.numItersPLR))*((iteration+1)-(kernel["LoopIters"]-self.numItersPLR)) if kernel["PrefetchGlobalRead"] else 0 + numPrefetchIter = 0 if iteration >= isBarrier and not hasLocalRead else numPrefetchIter + skipReadsIterA += numPrefetchIter + skipReadsIterB += numPrefetchIter + # here the reads are prefetches so can skip them in the waitcnt + # how many localreads can skip is based on how many iterations we prefetch. + localReads += self.numReadsPerIterA * skipReadsIterA + localReads + self.numReadsPerIterB * skipReadsIterB + # some of localReads is interleaved after waitcnt in SIA3 + if kernel["ScheduleIterAlg"] == 3 and self.numItersPLR and\ + (iteration < numReadsIterA or iteration < numReadsIterB or numPrefetchIter): + if (iteration < numReadsIterA and not dataAtIterA < max(dataAtIterA,dataAtIterB)) or numPrefetchIter: + localReads -= self.numReadsPerIterA + if (iteration < numReadsIterB and not dataAtIterB < max(dataAtIterA,dataAtIterB)) or numPrefetchIter: + localReads -= self.numReadsPerIterB + localReads += localReadsWaitcnt + lgkmcnt += localReads + iterCode.addComment0("numPrefetchIter=%u" % numPrefetchIter) + iterCode.addComment0("dataAtIterA=%u numReadsIterA=%u skipReadsIterA=%u readsPerIterA=%u" % (dataAtIterA, numReadsIterA, skipReadsIterA, self.numReadsPerIterA)) + iterCode.addComment0("dataAtIterB=%u numReadsIterB=%u skipReadsIterB=%u readsPerIterB=%u" % (dataAtIterB, numReadsIterB, skipReadsIterB, self.numReadsPerIterB)) + if kernel["ScheduleIterAlg"] == 0 or kernel["ScheduleIterAlg"] == 1: + for i in range (max(dataAtIterA,dataAtIterB),iteration+1): + localWrites += self.perIterLocalWriteCode[i].countType(LocalWriteInstruction) + # ScheduleIterAlg=2, localwrite is after waitCnt, no need to count it's current iteration. + if kernel["ScheduleIterAlg"] == 3: + for i in range (max(dataAtIterA,dataAtIterB)+1,iteration): + localWrites += self.perIterLocalWriteCode[i].countType(LocalWriteInstruction) + if kernel["ScheduleLocalWrite"] > 0: + # current iteration localWrite count + localWrites += skipLocalWriteWaitcnt + # dataAtIter iteration localWrite count + if self.numItersPLR: + skipPreIterLW = self.perIterLocalWriteCanSkip[max(dataAtIterA,dataAtIterB)] + if kernel["PrefetchGlobalRead"] == 2 and kernel["LocalReadVectorWidth"] == 2 and \ + (kernel["DirectToVgprA"] or kernel["DirectToVgprB"]): + # PGR==2 and LRVW==2 and DirectToVgpr enabled case, count local write before max(dataAtIterA,dataAtIterB) + # NOTE: This logic assumes that local write is scheduled after local read. + for up in range(max(dataAtIterA,dataAtIterB)): + skipPreIterLW += self.perIterLocalWriteCanSkip[up] + localWrites += skipPreIterLW + lgkmcnt += localWrites + else: + for item in list(iterCode.items()): + localReads = item.countType(LocalReadInstruction) + localWrites = item.countType(LocalWriteInstruction) + if self.numVgprBuffer: + # SQ: If PrefetchLocalRead = 1 and DepthU == LocalSplitU, then there is no double + # buffering and we must wait for all localReads but not localWrites. + # In that case, LoopIters == 1: + if kernel["LoopIters"] > 1: + # here the reads are prefetches so can skip them in the waitcnt + lgkmcnt += localReads + # and the writes are targetting another section of LDS and are + # synchronized through a different waitcnt than this one + # (which is always just before the macs) + lgkmcnt += localWrites + else: + # if UnrollLoopEfficiencyEnable == True use waitCode passed lgkmCnt + # else: + # we need to wait for all preceding reads before the macs + # so only opportunity for optimization is if the writes are at the end + if globalParameters["UnrollLoopEfficiencyEnable"]: + lgkmcnt = waitCode.lgkmcnt + else: + if localReads: + lgkmcnt = 0 # reset to wait for all reads + else: + lgkmcnt = localWrites # this only survives if writes are at the end + + waitCode.comment += " old=%u, new=%u newLW=%u newLR=%u" % (waitCode.lgkmcnt, lgkmcnt,localWrites,localReads) + waitCode.lgkmcnt = lgkmcnt + # This line is added for backward compatibility + waitCode.vscnt = waitCode.vmcnt if waitCode.lgkmcnt != -1 and waitCode.vmcnt != -1 and self.archCaps["SeparateVscnt"] else -1 + + return iterCode + + ############################################################################## + # returns list of modules or text + ############################################################################## + def setupNewTile(self, kernel, tensorParametersA, tensorParametersB, isOptNLL=False, forceNoTileCode=False, forceNoGRCode=False): + module = Module("setupNewTile") + + #################################### + # Global Read Addresses + #################################### + module.addComment2("Begin setupNewTile") + + # work-group assignments + module.addComment1("global read addresses: work-group") + if not forceNoTileCode: + module.add(self.graWorkGroup(kernel)) + + self.dontAppendCode = forceNoTileCode + # tile assignments + module.addComment1("global read addresses: tile offset assignment a") + module.add(self.graTileAssignment(kernel, tensorParametersA)) + module.addComment1("global read addresses: tile offset assignment b") + module.add(self.graTileAssignment(kernel, tensorParametersB)) + + # unroll assignments + module.addComment1("global read addresses: unroll assignment a") + module.add(self.graUnrollAssignment(kernel, tensorParametersA)) + module.addComment1("global read addresses: unroll assignment b") + module.add(self.graUnrollAssignment(kernel, tensorParametersB)) + + # other free indices + if kernel["ProblemType"]["NumIndicesC"] > 2: + module.addComment1("global read addresses: other free assignments") + module.add(self.graOtherFreeAssignments()) + + # other summation indices + if self.otherSummations: + module.addComment1("global read addresses: other summation assignments") + module.add(self.graOtherSummationAssignments(kernel)) + + # tile offsets + module.addComment1("global read addresses: tile offsets a") + module.add(self.graTileOffsets(kernel, tensorParametersA)) + module.addComment1("global read addresses: tile offsets b") + module.add(self.graTileOffsets(kernel, tensorParametersB)) + + # unroll offsets + module.addComment1("global read addresses: unroll offsets a") + module.add(self.graUnrollOffsets(kernel, tensorParametersA)) + module.addComment1("global read addresses: unroll offsets b") + module.add(self.graUnrollOffsets(kernel, tensorParametersB)) + + # tile edges + if kernel["EdgeType"] == "ShiftPtr": + # Shift here has two purposes: + # 1. Ensure the loads are in-bounds to prevent fault. + # BufferLoad uses the buffer limit hardware and does not require bounds checking for this case + # 2. Shift-left a wide vector load to ensure it is completely in-bounds. + # If this occurs we need to 'unshift' the C values (see shiftVectorComponents) + # BufferLoad does support this shifting, but if GuaranteeNoPartial=1 then + # it can be guaranteed that no shifting is required. + if not (kernel["BufferLoad"] and kernel["GuaranteeNoPartialA"]) and not forceNoTileCode: + module.addComment1("global read addresses: shift a") + module.add(self.graShift(kernel, tensorParametersA)) + if not (kernel["BufferLoad"] and kernel["GuaranteeNoPartialB"]) and not forceNoTileCode: + module.addComment1("global read addresses: shift b") + module.add(self.graShift(kernel, tensorParametersB)) + + # final offsets + module.addComment1("global read addresses: final offsets a") + module.add(self.graFinalOffsets(kernel, tensorParametersA)) + module.addComment1("global read addresses: final offsets b") + module.add(self.graFinalOffsets(kernel, tensorParametersB)) + self.dontAppendCode = False + self.dontAppendCode = self.dontAppendCode or forceNoTileCode + + # addresses + if not forceNoTileCode: + module.addComment1("global read addresses: addresses a") + module.add(self.graAddresses(kernel, tensorParametersA)) + module.addComment1("global read addresses: addresses b") + module.add(self.graAddresses(kernel, tensorParametersB)) + + # increments + module.addComment1("global read addresses: increments a") + for i in reversed(range(kernel["ProblemType"]["NumIndicesSummation"])): + module.add(self.graIncrements(kernel, i, tensorParametersA)) + module.addComment1("global read addresses: increments b") + for i in reversed(range(kernel["ProblemType"]["NumIndicesSummation"])): + module.add(self.graIncrements(kernel, i, tensorParametersB)) + + #################################### + # Local Write Addresses + #################################### + module.addComment2("Local Write Addresses") + + # tile assignments + module.add(self.lwaTileAssignment(tensorParametersA)) + module.add(self.lwaTileAssignment(tensorParametersB)) + + # unroll assignments + module.add(self.lwaUnrollAssignment(kernel, tensorParametersA)) + module.add(self.lwaUnrollAssignment(kernel, tensorParametersB)) + + # first offsets + module.addComment1("local write addresses: first offset a") + module.add(self.lwaFirstOffset(kernel, tensorParametersA)) + module.addComment1("local write addresses: first offset b") + module.add(self.lwaFirstOffset(kernel, tensorParametersB)) + self.dontAppendCode = False + self.dontAppendCode = self.dontAppendCode or forceNoTileCode + + ########################################################################### + # summations loops: open + ########################################################################### + + # declare loop num iter + if not forceNoTileCode: + module.addComment0("declare loop num iterations") + + # perform initC in the shadow of the prefetch + # Prefetch occurs at start of unroll loop + # If we have multiple summation indices (otherSummationLoops>0), + # we can't init in shadow of this prefetch + # since that would initC inside the other summation loops + + if self.doShadowInit != 2: + module.add(self.initC(kernel)) + + # open non-unrolled summation loops + if not forceNoTileCode: + for i in range(kernel["ProblemType"]["NumIndicesSummation"]-1): + module.addComment1("summation loop %u"%i) + module.add(self.calculateLoopNumIter(kernel, tensorParametersA, tensorParametersB, i)) + if self.actualSummationLoops>1: + module.add(self.openLoop(kernel, tensorParametersA, tensorParametersB, i)) + module.add(self.calculateLoopNumIter(kernel, tensorParametersA, tensorParametersB, self.unrollIdx)) + + if not forceNoTileCode: + if self.staggerU: + module.add(self.declareStaggerParms(kernel)) + module.add(self.calculateStagger(kernel, tensorParametersA)) + module.add(self.calculateStagger(kernel, tensorParametersB)) + + # LRO and LWA as assigned + # init lds read pointers before each unrolled loop + module.addComment0("local read addresses: init pointers a") + module.add(self.localReadInitPointers(kernel, tensorParametersA, tensorParametersA)) + module.addComment0("local read addresses: init pointers b") + module.add(self.localReadInitPointers(kernel, tensorParametersA, tensorParametersB)) + + #################################### + # prefetch: unrolled loop prefix + #################################### + if kernel["PrefetchGlobalRead"]: + pfi = 1 + module.addComment1("prefetch: global -> local") + module.add(self.openSumAtLeastUnroll(kernel, prefetch=True, isOptNLL=isOptNLL)) + # if DirectToVgprA is enabled, swap the order of global read (B->A) + tensorParameters1st = tensorParametersA + tensorParameters2nd = tensorParametersB + if kernel["DirectToVgprA"]: + tensorParameters1st, tensorParameters2nd = tensorParameters2nd, tensorParameters1st + moduleTmp = self.directToLdsM0Update(kernel, 0, tensorParameters1st, usePlaceHolder=False) + module.add(replaceHolder(moduleTmp, 0)) + module.add(self.globalReadDo(kernel, 0, tensorParameters1st, 0)) + moduleTmp = self.directToLdsM0Update(kernel, 0, tensorParameters2nd, usePlaceHolder=False) + module.add(replaceHolder(moduleTmp, 0)) + module.add(self.globalReadDo(kernel, 0, tensorParameters2nd, 0)) + module.add(self.globalReadIncrementAB(kernel, tensorParametersA, tensorParametersB, self.unrollIdx, pfi)) + + module.addComment2("End setupNewTile") + + return module + + + ############################################################################## + # get conditions to skip local write wait + ############################################################################## + def getConditionToSkipLocalWriteWait( self, kernel, u): + cond1 = not (u == 0 and kernel["PrefetchLocalRead"] != 0 and \ + (kernel["DirectToVgprA"] and kernel["DirectToLdsB"] or kernel["DirectToVgprB"] and kernel["DirectToLdsA"])) \ + or kernel["PrefetchGlobalRead"]==2 + # no need local read wait if LocalReadVectorWidth==2 and u is odd. + # In that case, Prefetch local read covers both u = 0 and 1 (limit to MFMA+double+DirectToVgpr only) + # (The other side of numReadsIterCoalesced must be 0 to skip local read wait) + condSkip = kernel["LocalReadVectorWidth"]==2 and (u%2 != 0) and kernel["EnableMatrixInstruction"] and \ + kernel["ProblemType"]["DataType"].isDouble() and \ + (kernel["DirectToVgprA"] and self.numReadsIterCoalescedB % 2 == 0 or \ + kernel["DirectToVgprB"] and self.numReadsIterCoalescedA % 2 == 0) + return cond1 and (not condSkip) + + ############################################################################## + # No Load Loop Body + ############################################################################## + def noLoadLoopBody( self, kernel, tensorParametersA, tensorParametersB, pack, isOptNLL, isNGLL, NLLfirst, NLLlast, isDTVodd=False): + module = Module("noLoadLoopBody") + expand = kernel["ExpandPointerSwap"] + lastuIdx = False + pflr = self.numItersPLR + localWriteEndIter = kernel["LoopIters"] - self.numItersPLR - 1 + + for uIdx in range(0, kernel["LoopIters"]*kernel["DepthULdsDivisor"]): + u = uIdx % kernel["LoopIters"] # u: index in compute loop (in contrast to the notion of global read loop) + uDu = uIdx // kernel["LoopIters"] # uDu: index of compute loop + isLastLoop = (uDu == kernel["DepthULdsDivisor"] -1 ) and not isNGLL + if u == 0: + if uDu > 0: + assert len(self.globalReadACode.items()) > 0 and len(self.globalReadBCode.items()) > 0 # already issued in first uDu + self.globalReadACode = StructuredModule() # empty + self.globalReadBCode = StructuredModule() # empty + self.globalReadIncrements = Module() # empty + self.globalReadIncrements.add(Module("globalReadIncrementA")) + self.globalReadIncrements.add(Module("globalReadIncrementB")) + if not isLastLoop: + self.localWriteACode = self.localWriteDo(kernel, tensorParametersA, (uDu+1)%kernel["DepthULdsDivisor"]) # local write in loopcnt N targets data for loopcnt N+1 + self.localWriteBCode = self.localWriteDo(kernel, tensorParametersB, (uDu+1)%kernel["DepthULdsDivisor"]) + else: + self.localWriteACode = Module() + self.localWriteBCode = Module() + + # TODO schedule waitcnt/barrier in makeSubIterSchedule() + if kernel["PrefetchGlobalRead"] and kernel["LoopIters"] in [1, 2] and uDu > 0: + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, 1, 0, -1, "wait for local write")) + module.add(self.syncThreads(kernel, "sync for local read after write")) + + if not isNGLL: + # PAP would have GlobalRead and GlobalInc, but no localWrite + # Get the perIterGlobalReadCode code for PAP (if PAP=On), else would be empty + self.makeSchedule(kernel, tensorParametersA, tensorParametersB, localWriteEndIter, uDu, skipGlobalReadInc=False, lastLoop=NLLlast) + module.add(self.unrollLoopHeaderCode) + + # which loop iteration to reset the LRO, + # note if PLR=0, isResetLroIter is False for all u + isResetLroIter = (u == localWriteEndIter) + isSwapAndResetLwoIter = isResetLroIter + isSwapLroIter = isResetLroIter + if kernel["ScheduleIterAlg"] == 3: + isSwapAndResetLwoIter = (u == self.lwEndMfmaIndex//(self.numMfmaPerIter)) + + extraComment = "" + if isLastLoop: + extraComment += " (last unrolled loop)" + else: + if kernel.enabledSplitLDS: + extraComment += f" (uDu={uDu}) " + if isResetLroIter: + extraComment += " (reset local read pointers iteration) " + if isSwapAndResetLwoIter: + extraComment += " (swap and reset local write pointers iteration) " + if isSwapLroIter: + extraComment += " (swap local read pointers iteration) " + + module.addComment1("iter %u%s"%(u,extraComment)) + plrIdx = ((u+pflr) % (self.numVgprBuffer+1)) % kernel["LoopIters"] + localReads = Module() + + pointerLWCode = Module() + pointerLRCode = Module() + waitCode = Module() # may be overwritten (not added to) below + macIterCode = Module() + waitLWCode = Module() + syncCode = Module() + + hasLiveLdsData = kernel["PrefetchGlobalRead"] or (uDu < kernel["DepthULdsDivisor"]-1) + hasLiveLdsData = hasLiveLdsData and not isLastLoop + # reads for current loop are done in previous iteration because of wider local read + doReadA = (u < kernel["LoopIters"]/self.numIterPerCoalescedReadA - self.numItersPLR) + doReadB = (u < kernel["LoopIters"]/self.numIterPerCoalescedReadB - self.numItersPLR) + # reads for next loop + doReadA = doReadA or (hasLiveLdsData and u > localWriteEndIter) + doReadB = doReadB or (hasLiveLdsData and u > localWriteEndIter) + # disable LocalRead if DirectToVgpr is enabled + doReadA = doReadA and (not kernel["DirectToVgprA"]) + doReadB = doReadB and (not kernel["DirectToVgprB"]) + for iui in range(0,kernel["InnerUnroll"]): + doReadA = doReadA and iui*self.numReadsIterCoalescedA < kernel["InnerUnroll"] + doReadB = doReadB and iui*self.numReadsIterCoalescedB < kernel["InnerUnroll"] + if doReadA: + localReads.addComment1("local read a") + localReadCodeA, packCodeA = self.localReadDo(kernel, plrIdx*self.numIterPerCoalescedReadA, iui*self.numReadsIterCoalescedA, 0, tensorParametersA) + localReads.add(localReadCodeA) + pack[plrIdx*self.numIterPerCoalescedReadA].add(packCodeA) + if doReadB: + localReads.addComment1("local read b") + localReadCodeB, packCodeB = self.localReadDo(kernel, plrIdx*self.numIterPerCoalescedReadB, iui*self.numReadsIterCoalescedB, 0, tensorParametersB) + localReads.add(localReadCodeB) + pack[plrIdx*self.numIterPerCoalescedReadB].add(packCodeB) + if (not isResetLroIter or iui != kernel["InnerUnroll"]-1): + if doReadA: + localReads.addComment1("local read increment a") + localReads.add(self.localReadInc(kernel, iui, tensorParametersA)) + if doReadB: + localReads.addComment1("local read increment b") + localReads.add(self.localReadInc(kernel, iui, tensorParametersB)) + + if not isLastLoop: + if kernel["PrefetchGlobalRead"]: + # put barrier at localWriteEndIter+1 + if u == localWriteEndIter+1 or (u == (localWriteEndIter+1)%kernel["LoopIters"] and kernel["ScheduleIterAlg"] == 2): + # skip local write wait if DirectToVgpr + DirectToLds is enabled + if not kernel["NoLdsWriteCode"]: + waitLWCode.add(self.wait(kernel, tensorParametersA, tensorParametersB, -1, 0, -1, "3wait for local write")) + if (kernel["DirectToVgprA"] or kernel["DirectToVgprB"]) and (kernel["DirectToLdsA"] or kernel["DirectToLdsB"]): + # DirectToVgpr + DirectToLds case, add waitcnt vmcnt before s_barrier + retInst = self.getWaitcntCodeForDirectToVgpr(kernel, localWriteEndIter, u, firstIter=False, beforeBarrier=True) + waitLWCode.add(retInst) + syncCode.add(self.syncThreads(kernel)) + + if isSwapAndResetLwoIter: # ResetLroIter + # local write for next iter, used to have local writes here + pointerLWCode.addComment1("local write swap offsets a") + pointerLWCode.add(self.localWriteSwapOffsets(kernel, expand, tensorParametersA)) + pointerLWCode.addComment1("local write swap offsets b") + pointerLWCode.add(self.localWriteSwapOffsets(kernel, expand, tensorParametersB)) + + if isSwapLroIter: # ResetLroIter + # Swap, reset, or increment the LRO: + # force internalPointerSwap = False in NGLL case + internalPointerSwap = expand and not isNGLL + pointerLRCode.addComment1("local read swap offsets a") + pointerLRCode.add(self.localReadSwapOffsets(kernel, internalPointerSwap, tensorParametersA)) + pointerLRCode.addComment1("local read swap offsets b") + pointerLRCode.add(self.localReadSwapOffsets(kernel, internalPointerSwap, tensorParametersB)) + + if isResetLroIter: # ResetLroIter + pointerLRCode.addComment1("local read init pointers a") + pointerLRCode.add(self.localReadInitPointers(kernel, tensorParametersA, tensorParametersA)) + pointerLRCode.addComment1("local read init pointers b") + pointerLRCode.add(self.localReadInitPointers(kernel, tensorParametersA, tensorParametersB)) + + # we initiate lgkmcnt to 0, then assigning it correct value in makeSubIterSchedule() + waitCode = self.wait(kernel, tensorParametersA, tensorParametersB, \ + -1, 0, 0, \ + "wait for prior local read local write") + # DirectToVgpr case, wait for global read as well as local read/write + if kernel["DirectToVgprA"] or kernel["DirectToVgprB"]: + # not generate wait here + # 1) local write code in previous u (u-1) has waitcnt vmcnt + prevVmcnt = False + prevLocalWrite = "" + if (u > 0): + prevLocalWrite = ' '.join([str(x) for x in self.perIterLocalWriteCode[u-1].flatitems()]) + prevVmcnt = "vmcnt" in prevLocalWrite + if not prevVmcnt: + retInst = self.getWaitcntCodeForDirectToVgpr(kernel, localWriteEndIter, u, False, isNGLL, NLLlast=NLLlast) + module.add(retInst) + + luIdx = (u) % (self.numVgprBuffer+1) # local to use for MACs + if kernel["EnableMatrixInstruction"]: + # NGLL case, use first set + setId = 0 if isNGLL else 1 + # flip setId if isDTVodd is True + if isDTVodd: + setId = 1 - setId + # use second set for DirectToVGPR + vregSetIdxMFMA = setId # use first set for NGLL, second set for other cases + macIterCode.add(self.mfmaIter(kernel, tensorParametersA, tensorParametersB, u, kernel["InnerUnroll"], vregSetIdxMFMA)) + else: + printExit("TensileLite does not support MAC instructions.") + + subIterCode = self.makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ + u, pointerLWCode, pointerLRCode, waitCode, macIterCode, waitLWCode, syncCode, pack[luIdx], isDTVodd, NLLlast) + module.add(subIterCode) + # vgpr.checkin for all the checked-out vgpr in LocalRead + for item in list(pack[luIdx].items()): + if item.tempVgpr != None: + self.vgprPool.checkIn(item.tempVgpr) + item.tempVgpr = None + pack[luIdx] = Module() + return module + + ############################################################################## + # noLoadLoop + # Create the no load loop (NLL) + # + # isOptNLL : the NLL is to be optimized for the alpha=1 and non-edge case + ############################################################################## + def noLoadLoop( self, kernel, tensorParametersA, tensorParametersB, isOptNLL, isNGLL, pack ): + module = Module("noLoadLoop") + LoopNameComment = "NoGlobalLoadLoop" if isNGLL else "NoLoadLoop" + isOptNLLComment = "Opt" if isOptNLL else "Ord" + startComment = "%s. %s - Begin " % (isOptNLLComment, LoopNameComment) + module.addComment2(startComment) + NLLfirst = True + NLLlast = True + if kernel["PrefetchGlobalRead"] == 2: + # PGR=2 case NoLoadLoop(NLL) is generated twice + # we need to distinguish them to generate proper code at each NLL + if isNGLL: + NLLlast = False + else: + # PGR=2 and not isNGLL means second NoLoadLoop for PGR2. + # Need to avoid generating duplicated code which is already generated in NGLL(first NoLoadLoop for PGR=2) + NLLfirst = False + if isNGLL: + self.perIterLocalWriteCode = self.perIterLocalWriteCodeNGLL + self.perIterLocalWriteCanSkip = [ 0 for i in range (kernel["LoopIters"]) ] + #else: + if not isNGLL: + self.dtlsM0UpdateACode = StructuredModule() + self.globalReadACode = StructuredModule() # empty + self.dtlsM0UpdateBCode = StructuredModule() + self.globalReadBCode = StructuredModule() # empty + self.globalReadIncrements = Module() + self.globalReadIncrements.add(Module("globalReadIncrementA")) + self.globalReadIncrements.add(Module("globalReadIncrementB")) + self.localWriteACode = Module() + self.localWriteBCode = Module() + + kStrOpenSum = self.openSumAtLeastUnroll(kernel, prefetch=False, isOptNLL=isOptNLL) + + # skip generating OpenSum code here for SingleNLLOpt + if not (isOptNLL and self.enableSingleNLLOpt): + module.add(kStrOpenSum) + kStrOpenSum = "" # empty OpenSum str to avoid inserting it again + + if not self.numItersPLR: + if kernel["DirectToLdsA"] or kernel["DirectToLdsB"]: + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, 0, -1, -1, "10wait for global read")) + # TODO: need to check if we correctly checked-in the temp VGPR used for Int8 LocalWrite (uDu, PGR=2) + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, -1, 0, -1, "4wait for local write")) + module.add(self.syncThreads(kernel)) + + # if DirectToVgpr and ASEM is not multiple of DepthU*2, generate noLoadLoopBody twice for odd and even exit separately + if ( kernel["DirectToVgprA"] or kernel["DirectToVgprB"]) and (kernel["AssertSummationElementMultiple"] % (kernel["DepthU"] * 2) != 0): + # generate additional No Load Loop Body code for odd case (to use the other Vreg set for DirectToVgpr) + # 1. generate odd check + name = "" + if isNGLL: + name += "NoGlobalLoadLoop" + else: + name += "NoLoadLoop" + if isOptNLL: + name += "Opt" + else: + name += "Ord" + module.add(self.openOddNoLoadLoopForDTV(name)) + # 2. generate no Load Loop Body code for odd + # backup + self.saveLocalPointers(kernel, tensorParametersA, tensorParametersB) + # deepCopy packCode for OptNLL noLoadLoop + deepCopyPack = copy.deepcopy(pack) + module.add(self.noLoadLoopBody(kernel, tensorParametersA, tensorParametersB, deepCopyPack, isOptNLL, isNGLL, NLLfirst, NLLlast, isDTVodd=True)) + # restore + self.restoreLocalPointers(kernel, tensorParametersA, tensorParametersB) + # 3. generate even start label + module.add(self.closeOddNoLoadLoopForDTV(name)) + # 4. generate no Load Loop Body code for odd + # need to re-initialize perIterLocalWriteCanSkip to avoid having incorrect lgkmcnt + self.perIterLocalWriteCanSkip = [ 0 for i in range (kernel["LoopIters"]) ] + module.add(self.noLoadLoopBody(kernel, tensorParametersA, tensorParametersB, pack, isOptNLL, isNGLL, NLLfirst, NLLlast)) + # 5. generate even end label + module.add(self.generateEvenEndLabeNoLoadLoopForDTV(name)) + else: + # generate no Load Loop Body code + module.add(self.noLoadLoopBody(kernel, tensorParametersA, tensorParametersB, pack, isOptNLL, isNGLL, NLLfirst, NLLlast)) + + # add OpenSum code here if it is not empty + if kStrOpenSum != "": + module.add(kStrOpenSum) + + # Close code is necessary for both first and last (NGLL case(=NLLfirst) needs label) + module.add(self.closeSumAtLeastUnroll(kernel, tensorParametersA, tensorParametersB, prefetch=False, isOptNLL=isOptNLL, isNGLL=isNGLL)) + + return module + + ############################################################################## + # Loop Body + ############################################################################## + def loopBody( self, kernel, tensorParametersA, tensorParametersB, pack, lc, loopCopies, finalLoop, firstIter=False ): + module = Module("loopBody") + expand = kernel["ExpandPointerSwap"] + + # not generate openLoop for firstIter + if not firstIter: + module.addComment2("Unrolled Loop %u/%u - Begin" % (lc+1, loopCopies)) + if kernel["PrefetchGlobalRead"] and not self.numItersPLR and not kernel["ScheduleIterAlg"] == 2: + if kernel["DirectToLdsA"] or kernel["DirectToLdsB"]: + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, 0, -1, -1, "11wait for global read")) + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, 1, 0, -1, "1wait for local write")) + module.add(self.syncThreads(kernel, "4sync for global read")) + + module.addComment1("Begin Each Unroll: Check VGPR.checkin for INT8 LW") + + # if DirectToVgprA is enabled, swap the order of global read (B->A) + tensorParameters1st = tensorParametersA + tensorParameters2nd = tensorParametersB + tc1 = 'A' + tc2 = 'B' + if kernel["DirectToVgprA"]: + tensorParameters1st, tensorParameters2nd = tensorParameters2nd, tensorParameters1st + tc1, tc2 = tc2, tc1 + # unrolled loop: global read A, B + # M0 update for directToLds + vregSetIdxGR = 0 + if (kernel["DirectToVgpr%s"%tc1]): + vregSetIdxGR = (kernel["PrefetchGlobalRead"] + lc ) % 2 # toggle vreg set for DirectToVgpr. + self.dtlsM0UpdateACode = self.directToLdsM0Update(kernel, 1, tensorParameters1st, usePlaceHolder=True) + self.globalReadACode = self.globalReadDo(kernel, 1, tensorParameters1st, vregSetIdxGR) + vregSetIdxGR = 0 + if (kernel["DirectToVgpr%s"%tc2]): + vregSetIdxGR = (kernel["PrefetchGlobalRead"] + lc ) % 2 # toggle vreg set for DirectToVgpr. + self.dtlsM0UpdateBCode = self.directToLdsM0Update(kernel, 1, tensorParameters2nd, usePlaceHolder=True) + self.globalReadBCode = self.globalReadDo(kernel, 1, tensorParameters2nd, vregSetIdxGR) + + # unrolled loop: increment global read addresses + self.globalReadIncrements = self.globalReadIncrementAB(kernel, tensorParametersA, tensorParametersB, self.unrollIdx, 0) + + if not kernel["NoLdsWriteCode"]: + self.localWriteACode = self.localWriteDo(kernel, tensorParametersA) + self.localWriteBCode = self.localWriteDo(kernel, tensorParametersB) + else: + self.localWriteACode = Module() + self.localWriteBCode = Module() + + # localWriteEndIter is used to determine which iteration to put sync + # if PGR=0, GR,LW,sync,LR will put at front of loop. + localWriteEndIter = kernel["LoopIters"] - self.numItersPLR - 1 + + # Schedule the global read, global read inc, and writes: + unrollLoopHeaderCodeScheduled = False + if not kernel["PrefetchGlobalRead"]: + unrollLoopHeaderCodeScheduled = True + self.makeSchedule(kernel, tensorParametersA, tensorParametersB, localWriteEndIter, firstIter=firstIter) + module.add(self.unrollLoopHeaderCode) + + # if not prefetch global, localWrite before mac's + if not kernel["PrefetchGlobalRead"]: + # unrolled loop: local write A, B + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, 0, -1, -1, "5wait for global read")) + module.add(self.syncThreads(kernel, "PGR=0, prior iter done reading lds")) + if not kernel["NoLdsWriteCode"]: + module.addComment1("local write a") + tempLWCodeModA = self.localWriteDo(kernel, tensorParametersA) + module.add(tempLWCodeModA) + module.addComment1("local write b") + tempLWCodeModB = self.localWriteDo(kernel, tensorParametersB) + module.add(tempLWCodeModB) + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, -1, 0, -1, "2prefetch wait for local write")) + module.add(self.syncThreads(kernel)) + # debug Local state + """ + module.add(" /* print Local state */" + self.endLine) + module.add(" for (unsigned int i = serial; i < LDS_NUM_ELEMENTS; i+=NUM_THREADS) {%s" % self.endLine) + module.add(" printf(\\\"localMemory[%%06u] = %%.0f\\\\n\\\", i, localMemory[i]);%s" ) + % self.endLine + module.add(" }" + self.endLine) + """ + + # unrolled loop: prefetch local + if self.numItersPLR and not kernel["PrefetchGlobalRead"]: + for plrIdx in range(0, self.numItersPLR): + pack[plrIdx] = Module() + for iui in range(0,kernel["InnerUnroll"]): + if iui*self.numReadsIterCoalescedA < kernel["InnerUnroll"] and (not kernel["DirectToVgprA"]) : # no local read code if DirectToVgpr is enabled + module.addComment1("prefetch local a") + localReadCodeA, packCodeA = self.localReadDo(kernel, plrIdx*self.numIterPerCoalescedReadA, iui*self.numReadsIterCoalescedA, 0, tensorParametersA) + module.add(localReadCodeA) + pack[plrIdx].add(packCodeA) + if iui*self.numReadsIterCoalescedB < kernel["InnerUnroll"] and (not kernel["DirectToVgprB"]) : # no local read code if DirectToVgpr is enabled + module.addComment1("prefetch local b") + localReadCodeB, packCodeB = self.localReadDo(kernel, plrIdx*self.numIterPerCoalescedReadB, iui*self.numReadsIterCoalescedB, 0, tensorParametersB) + module.add(localReadCodeB) + pack[plrIdx].add(packCodeB) + if iui*self.numReadsIterCoalescedA < kernel["InnerUnroll"] and (not kernel["DirectToVgprA"]) : # no local read code if DirectToVgpr is enabled + module.addComment0("local read increment a") + module.add(self.localReadInc(kernel, iui, tensorParametersA)) + if iui*self.numReadsIterCoalescedB < kernel["InnerUnroll"] and (not kernel["DirectToVgprB"]) : # no local read code if DirectToVgpr is enabled + module.addComment0("local read increment b") + module.add(self.localReadInc(kernel, iui, tensorParametersB)) + + pflr = self.numItersPLR # how many pf already done above + + ############################################################################ + # unrolled loop: mac iterations + ############################################################################ + + # double/quadruple the number of compute loop for each DepthU's worth of data read + for uIdx in range(0, kernel["LoopIters"]*kernel["DepthULdsDivisor"]): + u = uIdx % kernel["LoopIters"] # u: index in compute loop (in contrast to the notion of global read loop) + uDu = uIdx // kernel["LoopIters"] # uDu: index of compute loop + if u==0: # if at start of subloop... + # ...update local write code + if not kernel["NoLdsWriteCode"]: + self.localWriteACode = self.localWriteDo(kernel, tensorParametersA, (uDu+1)%kernel["DepthULdsDivisor"]) # local write in loopcnt N targets data for loopcnt N+1 + self.localWriteBCode = self.localWriteDo(kernel, tensorParametersB, (uDu+1)%kernel["DepthULdsDivisor"]) + else: + self.localWriteACode = Module() + self.localWriteBCode = Module() + + # TODO schedule waitcnt/barrier in makeSubIterSchedule() + if kernel["PrefetchGlobalRead"] and kernel["LoopIters"] in [1, 2] and uDu > 0: + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, 1, 0, -1, "wait for local write")) + module.add(self.syncThreads(kernel, "sync for local read after write")) + + if not unrollLoopHeaderCodeScheduled: + self.makeSchedule(kernel, tensorParametersA, tensorParametersB, localWriteEndIter, uDu, firstIter=firstIter, lastLoop=False, lastLc=(lc==loopCopies-1)) + module.add(self.unrollLoopHeaderCode) + + # for PGR=0 where generator can't schedule the instructions (yet), + # we duplicate the local write codegen and append to string list directly + if not kernel["PrefetchGlobalRead"]: + doWrite = False + if uDu0 and self.numItersPLR==0 and u==0: + assert doWrite==False # should be exclusive with the previous condition + doWrite = True + writeForNextLoop = 0 + # unrolled loop: local write A, B + if doWrite: + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, -1, -1, 0, "5wait for local read")) + module.add(self.syncThreads(kernel, "PGR=0, prior iter done reading lds")) + if not kernel["NoLdsWriteCode"]: + module.addComment1("local write a") + tempLWCodeModA = self.localWriteDo(kernel, tensorParametersA, (uDu+writeForNextLoop)%kernel["DepthULdsDivisor"]) + module.add(tempLWCodeModA) + module.addComment1("local write b") + tempLWCodeModB = self.localWriteDo(kernel, tensorParametersB, (uDu+writeForNextLoop)%kernel["DepthULdsDivisor"]) + module.add(tempLWCodeModB) + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, -1, 0, -1, "2prefetch wait for local write")) + module.add(self.syncThreads(kernel)) + + # which loop iteration to reset the LRO, + # note if PLR=0, isResetLroIter is False for all u + isResetLroIter = (u == localWriteEndIter) + isSwapAndResetLwoIter = isResetLroIter + isSwapLroIter = isResetLroIter + if kernel["ScheduleIterAlg"] == 3: + isSwapAndResetLwoIter = (u == self.lwEndMfmaIndex//(self.numMfmaPerIter)) + extraComment = "" + if kernel.enabledSplitLDS: + extraComment += f" (uDu={uDu}) " + if isResetLroIter: + extraComment += " (reset local read pointers iteration) " + if isSwapAndResetLwoIter: + extraComment += " (swap and reset local write pointers iteration) " + if isSwapLroIter: + extraComment += " (swap local read pointers iteration) " + + module.addComment1("iter %u%s"%(u,extraComment)) + plrIdx = ((u+pflr) % (self.numVgprBuffer+1)) % kernel["LoopIters"] + + localReads = Module() + localReadsA = Module() + localReadsB = Module() + + pointerLWCode = Module() + pointerLRCode = Module() + waitCode = Module() # may be overwritten (not added to) below + macIterCode = Module() + waitLWCode = Module() + syncCode = Module() + + hasLiveLdsData = kernel["PrefetchGlobalRead"] or (uDu < kernel["DepthULdsDivisor"]-1) + # reads for current loop are done in previous iteration because of wider local read + doReadA = (u < kernel["LoopIters"]/self.numIterPerCoalescedReadA - self.numItersPLR) + doReadB = (u < kernel["LoopIters"]/self.numIterPerCoalescedReadB - self.numItersPLR) + # reads for next loop + doReadA = doReadA or (hasLiveLdsData and u > localWriteEndIter) + doReadB = doReadB or (hasLiveLdsData and u > localWriteEndIter) + # disable LocalRead if DirectToVgpr is enabled + doReadA = doReadA and (not kernel["DirectToVgprA"]) + doReadB = doReadB and (not kernel["DirectToVgprB"]) + # double the number of VgprValu if self.vgprValuDouble is true + plrIdxLR = plrIdx + if self.vgprValuDouble and (lc & 1) == 0: + # use the next buffer set (do not change the index of pack[]) + plrIdxLR += 1 + for iui in range(0,kernel["InnerUnroll"]): + doReadA = doReadA and iui*self.numReadsIterCoalescedA < kernel["InnerUnroll"] + doReadB = doReadB and iui*self.numReadsIterCoalescedB < kernel["InnerUnroll"] + if doReadA: + localReads.addComment1("local read a") + localReadCodeA, packCodeA = self.localReadDo(kernel, plrIdxLR*self.numIterPerCoalescedReadA, iui*self.numReadsIterCoalescedA, 0, tensorParametersA) + localReads.add(localReadCodeA) + localReadsA.add(localReadCodeA) + pack[plrIdx*self.numIterPerCoalescedReadA].add(packCodeA) + if doReadB: + localReads.addComment1("local read b") + localReadCodeB, packCodeB = self.localReadDo(kernel, plrIdxLR*self.numIterPerCoalescedReadB, iui*self.numReadsIterCoalescedB, 0, tensorParametersB) + localReads.add(localReadCodeB) + localReadsB.add(localReadCodeB) + pack[plrIdx*self.numIterPerCoalescedReadB].add(packCodeB) + # Don't increment the LRO if we are going to reset them below: + if not isResetLroIter or iui != kernel["InnerUnroll"]-1: + if doReadA: + localReads.addComment1("local read increment a") + localReads.add(self.localReadInc(kernel, iui, tensorParametersA)) + if doReadB: + localReads.addComment1("local read increment b") + localReads.add(self.localReadInc(kernel, iui, tensorParametersB)) + + if kernel["PrefetchGlobalRead"]: + # wait code for DirectToVgpr + if kernel["DirectToVgprA"] or kernel["DirectToVgprB"]: + # not generate wait here + # 1) local write code in previous u (u-1) has waitcnt vmcnt + prevVmcnt = False + prevLocalWrite = "" + if (u > 0): + for up in range(u): + prevLocalWrite += ' '.join([str(x) for x in self.perIterLocalWriteCode[up].flatitems()]) + prevVmcnt = "vmcnt" in prevLocalWrite + if not prevVmcnt: + retInst = self.getWaitcntCodeForDirectToVgpr(kernel, localWriteEndIter, u, firstIter) + module.add(retInst) + # put barrier at localWriteEndIter+1 + if u == localWriteEndIter+1 or (u == (localWriteEndIter+1)%kernel["LoopIters"] and kernel["ScheduleIterAlg"] == 2): + if kernel["DirectToLdsA"] or kernel["DirectToLdsB"]: + # skip generating wait for global read again here in DirectToVgpr case + if not(kernel["DirectToVgprA"] or kernel["DirectToVgprB"]): + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, 0, -1, -1, "12wait for global read")) + else: + # DirectToVgpr + DirectToLds case, add waitcnt vmcnt before s_barrier + retInst = self.getWaitcntCodeForDirectToVgpr(kernel, localWriteEndIter, u, firstIter, beforeBarrier=True) + waitLWCode.add(retInst) + # skip local write wait if DirectToVgpr + DirectToLds is enabled + # (no local write code. Global read wait for DirectToLds is already done) + if not kernel["NoLdsWriteCode"]: + waitLWCode.add(self.wait(kernel, tensorParametersA, tensorParametersB, -1, 0, -1, "3wait for local write")) + if kernel["DirectToVgprA"] or kernel["DirectToVgprB"]: + # put only barrier for DirectToVgpr (to avoid generating waitcnt for global read) + syncCode.add("s_barrier" + self.endLine) + else: + syncCode.add(self.syncThreads(kernel)) + + if isSwapAndResetLwoIter: # ResetLroIter + # local write for next iter, used to have local writes here + pointerLWCode.addComment1("local write swap offsets a") + pointerLWCode.add(self.localWriteSwapOffsets(kernel, expand, tensorParametersA)) + pointerLWCode.addComment1("local write swap offsets b") + pointerLWCode.add(self.localWriteSwapOffsets(kernel, expand, tensorParametersB)) + + if isSwapLroIter: # ResetLroIter + # Swap, reset, or increment the LRO: + pointerLRCode.addComment1("local read swap offsets a") + pointerLRCode.add(self.localReadSwapOffsets(kernel, expand, tensorParametersA)) + pointerLRCode.addComment1("local read swap offsets b") + pointerLRCode.add(self.localReadSwapOffsets(kernel, expand, tensorParametersB)) + + if isResetLroIter: # ResetLroIter + pointerLRCode.addComment1("local read init pointers a") + pointerLRCode.add(self.localReadInitPointers(kernel, tensorParametersA, tensorParametersA)) + pointerLRCode.addComment1("local read init pointers b") + pointerLRCode.add(self.localReadInitPointers(kernel, tensorParametersA, tensorParametersB)) + + # we initiate lgkmcnt to 0, then assigning it correct value in makeSubIterSchedule() + if self.getConditionToSkipLocalWriteWait(kernel, u): + waitCode = self.wait(kernel, tensorParametersA, tensorParametersB, \ + -1, 0, 0, \ + "wait for prior local read local write") + + luIdx = (u) % (self.numVgprBuffer+1) # local to use for MACs + if kernel["EnableMatrixInstruction"]: + vregSetIdxMFMA = lc + macIterCode.add(self.mfmaIter(kernel, tensorParametersA, tensorParametersB, u, kernel["InnerUnroll"], vregSetIdxMFMA)) + else: + printExit("TensileLite does not support MAC instructions.") + + ###### unroll loop efficiency implementation###################################### + # unroll loop efficiency implementation + ## split A&B fetch&MAC code into multiple groups + ## splitting strategy based on TT size + ## 6x4 -> split MAC blob(s) into group of 8(s) and 16 FMA instructions. + ## LDS fetch(es) into group of A{1-2)B(0) , A(3),B(1) (not implemented yet) + ## 4x6 -> split MAC blob(s) into group of 8(s) and 16 FMA instructions. + ## LDS fetch(es) into group of B{1-2)A(0) , B(3),A(1) + ## 4x4 -> split into group of 8 and 8 MAC(s) + ## 6x6 -> split into group of 12 MAC(s) + ## 8x4/4x8 -> split into group of 16 and 16 MAC(s) + ## 8x8 -> split into group of 16 MAC(s) + ## supports only PLR=0 + ############################################################################### + if self.numItersPLR or (not globalParameters["UnrollLoopEfficiencyEnable"]): + subIterCode = self.makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ + u, pointerLWCode, pointerLRCode, waitCode, macIterCode, waitLWCode, syncCode, pack[luIdx]) + module.add(subIterCode) # add scheduled "other", local reads, local writes + for item in list(pack[luIdx].items()): + if item.tempVgpr != None: + self.vgprPool.checkIn(item.tempVgpr) + item.tempVgpr = None + pack[luIdx] = Module() + else: + printExit("TensileLite does not support MAC instructions.") + + # close unrolled loop + if expand: + if not finalLoop: + module.addComment2("Unrolled Loop - End %u/%u"%(lc+1, loopCopies)) + else: + module.addComment2("Unrolled Loop - End %u/%u (final)"%(lc+1, loopCopies)) + + else: + module.addComment2("Unrolled Loop - End") + + oddLabel = lc == 0 + module.add(self.closeLoop(kernel, tensorParametersA, tensorParametersB, self.unrollIdx, finalLoop, oddLabel=oddLabel)) + return module + + ############################################################################## + # Kernel Body + ############################################################################## + def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): + expand = kernel["ExpandPointerSwap"] + + #################################### + # Begin String + moduleKernelBody = KernelBody("kernelBody") + + #################################### + # Function Signature + #################################### + fs = self.functionSignature() + moduleKernelBody.addSignature(fs) + + module = Module("body") + module.add(self.defineAndResources(kernel, tensorParametersA, tensorParametersB)) + + #################################### + # Local Read Addresses + #################################### + module.addComment2("Local Read Addresses") + + # tile assignments + module.addComment1("local read addresses: tile assignments a/b") + module.add(self.lraTileAssignment(kernel, tensorParametersA, tensorParametersB)) + + # final offsets + module.addComment1("local read addresses: final offsets a") + module.add(self.lraFinalOffset(kernel, tensorParametersA)) + module.addComment1("local read addresses: final offsets b") + module.add(self.lraFinalOffset(kernel, tensorParametersB)) + + # declare addresses + module.addComment1("local read addresses: declare addresses a") + module.add(self.lraDeclareAddresses(kernel, tensorParametersA)) + module.addComment1("local read addresses: declare addresses b") + module.add(self.lraDeclareAddresses(kernel, tensorParametersB)) + + # doShadowInit performs initialization in the 'shadow' of the global mem prefetch + self.doShadowInit = 0 + if kernel["PrefetchGlobalRead"]: + if self.actualSummationLoops == 1: + self.doShadowInit = 2 # 2 is both store setup and initC + else: + # can't do shadow initC with multiple summation since this resets the ValuC counters + # on each unroll iteration. + self.doShadowInit = 1 # 1 is just store setup + + module.add(self.setupNewTile(kernel, tensorParametersA, tensorParametersB, isOptNLL=False)) + + pack = [ Module() for i in range (self.numVgprBuffer+1) ] + self.preLoopLocalWriteCode = None + + if kernel["PrefetchGlobalRead"]: + if self.doShadowInit: + module.add(self.openShadowInit()) + module.add(self.globalWriteWorkGroupInit(kernel)) + if self.doShadowInit == 2: + module.add(self.initC(kernel)) # initC while waiting for global reads + module.add(self.closeShadowInit(kernel)) + + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, 0, -1, -1, "8wait for global read")) + # These cases loop back and run the prefetch loop again + # we need an extra barrier to ensure that the ds_reads (either for SR or MFMA) from previous iteration + # have finished before we generate the prefetch for the next summation index. + if self.actualSummationLoops>1: + module.add(SBarrier()) + + # local write + self.preLoopLocalWriteCode = self.preLoopLocalWriteDo(kernel, tensorParametersA, tensorParametersB) + module.add(self.preLoopLocalWriteCode) + # swap local ptrs + module.addComment1("local write swap a") + module.add(self.localWriteSwapOffsets(kernel, expand, tensorParametersA)) + module.addComment1("local write swap b") + module.add(self.localWriteSwapOffsets(kernel, expand, tensorParametersB)) + + if kernel["PrefetchGlobalRead"] == 2: + module.add(self.openPrefetchGlobalRead2(kernel)) + # if DirectToVgprA is enabled, swap the order of global read (B->A) + tensorParameters1st = tensorParametersA + tensorParameters2nd = tensorParametersB + if kernel["DirectToVgprA"]: + tensorParameters1st, tensorParameters2nd = tensorParameters2nd, tensorParameters1st + module.add(self.directToLdsM0Update(kernel, 1, tensorParameters1st)) + module.add(self.globalReadDo(kernel, 0, tensorParameters1st, 1)) + module.add(self.directToLdsM0Update(kernel, 1, tensorParameters2nd)) + module.add(self.globalReadDo(kernel, 0, tensorParameters2nd, 1)) + + # swap local ptrs again if DirectToLds is enabled + if kernel["DirectToLdsA"]: + module.addComment1("local write swap a") + module.add(self.localWriteSwapOffsets(kernel, expand, tensorParametersA)) + if kernel["DirectToLdsB"]: + module.addComment1("local write swap b") + module.add(self.localWriteSwapOffsets(kernel, expand, tensorParametersB)) + + module.add(self.closePrefetchGlobalRead2()) + + # prefetch-local + if self.numItersPLR: + # not generate wait for local write if LDS write code is not generated + if not kernel["NoLdsWriteCode"]: + # TODO: need to check if we correctly checked-in the temp VGPR used for Int8 LocalWrite (uDu, PGR=2) + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, -1, 0, -1, "0prefetch wait for local write")) + module.add(self.syncThreads(kernel)) + + # in some cases need an extra copy of the LDS read with appropriate double buffer offsets + for plrIdx in range(0, self.numItersPLR): + pack[plrIdx] = Module() + for espi in range(0, 1): + for iui in range(0,kernel["InnerUnroll"]): + if iui*self.numReadsIterCoalescedA < kernel["InnerUnroll"] and (not kernel["DirectToVgprA"]) : # no local read code if DirectToVgpr is enabled + module.addComment1("local read prefetch a") + localReadCodeA, packCodeA = self.localReadDo(kernel, plrIdx*self.numIterPerCoalescedReadA, iui*self.numReadsIterCoalescedA, espi, tensorParametersA) + module.add(localReadCodeA) + pack[plrIdx].add(packCodeA) + if iui*self.numReadsIterCoalescedB < kernel["InnerUnroll"] and (not kernel["DirectToVgprB"]) : # no local read code if DirectToVgpr is enabled + module.addComment1("local read prefetch b") + localReadCodeB, packCodeB = self.localReadDo(kernel, plrIdx*self.numIterPerCoalescedReadB, iui*self.numReadsIterCoalescedB, espi, tensorParametersB) + module.add(localReadCodeB) + pack[plrIdx].add(packCodeB) + if iui*self.numReadsIterCoalescedA < kernel["InnerUnroll"] and (not kernel["DirectToVgprA"]) : # no local read code if DirectToVgpr is enabled + module.addComment1("local read inc a") + module.add(self.localReadInc(kernel, iui, tensorParametersA)) + if iui*self.numReadsIterCoalescedB < kernel["InnerUnroll"] and (not kernel["DirectToVgprB"]) : # no local read code if DirectToVgpr is enabled + module.addComment1("local read inc b") + module.add(self.localReadInc(kernel, iui, tensorParametersB)) + module.add(self.closeSumAtLeastUnroll(kernel, tensorParametersA, tensorParametersB, prefetch=True, isOptNLL=False, isNGLL=False)) + + loopCopies = 2 if expand else 1 + + if self.useInitAccVgprOpt: + # generate first iteration code for init accvgpr opt + module.addComment2("First Unrolled Iter for InitAccVgprOpt - Begin") + # open loop without Label + module.add(self.openLoop(kernel, tensorParametersA, tensorParametersB, self.unrollIdx, noLabelGen=True)) + module.add(self.loopBody( kernel, tensorParametersA, tensorParametersB, pack, 0, loopCopies, False, firstIter=True )) + + # open unrolled summation loop + module.addComment2("Unrolled Loop(s) - Begin") + module.add(self.openLoop(kernel, tensorParametersA, tensorParametersB, self.unrollIdx, beginLabelOnly=False)) + + lcStart = 0 + if self.useInitAccVgprOpt: + lcStart = 1 if loopCopies == 2 else 0 + for lc in range(0, loopCopies): + loopIndex = lcStart + lc + if loopIndex >= loopCopies: + loopIndex -= loopCopies + # loop body code generation + finalLoop = lc == loopCopies - 1 + module.add(self.loopBody( kernel, tensorParametersA, tensorParametersB, pack, loopIndex, loopCopies, finalLoop )) + + module.addComment1("Before NLL: Check VGPR.checkin for INT8 LW") + + # swap local write, read again before noLoadLoop if PrefetchGlobalRead and DirectToLds is enabled + # In DirectToLds enabled case, local write address is necessary for prefetch global read (for m0). + # However, even exit with DirectToLds will not pass with this code (limitation). + # So far, this code is to make odd exit case (i.e. k is multiple of 2*depthU) pass for DirectToVgpr + if not self.useInitAccVgprOpt and kernel["PrefetchGlobalRead"] and kernel["ExpandPointerSwap"]: + # local write for next iter, used to have local writes here + if(kernel["DirectToLdsA"]): + module.addComment1("local write swap offsets a") + module.add(self.localWriteSwapOffsets(kernel, expand, tensorParametersA)) + if(kernel["DirectToLdsB"]): + module.addComment1("local write swap offsets b") + module.add(self.localWriteSwapOffsets(kernel, expand, tensorParametersB)) + # swap local read point for self.useInitAccVgprOpt + if self.useInitAccVgprOpt and kernel["ExpandPointerSwap"]: + module.addComment1("local read swap offsets a") + module.add(self.localReadSwapOffsets(kernel, expand, tensorParametersA)) + module.addComment1("local read swap offsets b") + module.add(self.localReadSwapOffsets(kernel, expand, tensorParametersB)) + + if kernel["PrefetchGlobalRead"] == 2: + module.add(self.noLoadLoop(kernel, tensorParametersA, tensorParametersB, isOptNLL=False, isNGLL=True, pack=pack)) + + # This "NoLoad" loop is a copy of the unroll loop but with global loads + LDS writes removed + # doShadowInit is required since this pushes up the store SRD initialization before the NLL + # OptNLL only allowed for single summation index - for multiple summation we (currently) + # execute the NLL inside each unroll iteration not just once at the end. + if kernel["PrefetchGlobalRead"]: + if not kernel["SuppressNoLoadLoop"]: + + firstNLLgenerated = False + if kernel["KernelLanguage"] == "Assembly" and kernel["OptNoLoadLoop"] and \ + kernel["BufferLoad"] and kernel["BufferStore"] and self.doShadowInit and \ + kernel["LocalSplitU"]==1 and kernel["GlobalSplitU"] == 1 and \ + self.actualSummationLoops==1: + + firstNLLgenerated = True + + # two different noLoadLoops: + # 1. OptNLL & PAP global-read interleaved (only for PAP=ON) + # (2. OptNLL : No PAP global-read (For PAP=OFF, or PAP=ON but the last tile)) + # -> this is unified with 1. global-read is invalidated at the last tile. + # 3. OrdinaryNLL (Not Opt.) + self.saveLocalPointers(kernel, tensorParametersA, tensorParametersB) + # deepCopy packCode for OptNLL noLoadLoop + deepCopyPack = copy.deepcopy(pack) + module.add(self.noLoadLoop(kernel, tensorParametersA, tensorParametersB, isOptNLL=True, isNGLL=False, pack=deepCopyPack)) + self.restoreLocalPointers(kernel, tensorParametersA, tensorParametersB) + + # skip second NLL code if enableSingleNLLOpt + if not (self.enableSingleNLLOpt and firstNLLgenerated): + module.add(self.noLoadLoop(kernel, tensorParametersA, tensorParametersB, isOptNLL=False, isNGLL=False, pack=pack)) + else: + # generate PrefetchGlobalLastIterEnd label + module.add(self.closeSumAtLeastUnroll(kernel, tensorParametersA, tensorParametersB, prefetch=False, isOptNLL=False, isNGLL=False)) + + # if PGR, last few iterations will have PLR, + # and those PLR will not be used(register not checkIn) if without NoLoadLoop + else: + for i in range(self.numVgprBuffer): + for item in list(pack[i].items()): + if item.tempVgpr != None: + self.vgprPool.checkIn(item.tempVgpr) + item.tempVgpr = None + + if self.staggerU and self.actualSummationLoops>1: + module.addComment1("remove stagger offsets") + module.add(self.removeStagger(kernel, tensorParametersA)) + module.add(self.removeStagger(kernel, tensorParametersB)) + + if not self.noTailLoop: + ######################################## + # Tail Loop + # which means tail loop not needed. + ######################################## + self.inTailLoop = True + module.addComment2("Tail Loop") + + # Update local write pointers in case the upcoming global reads are writing directly to LDS: + if kernel["PrefetchGlobalRead"]: + module.addComment1("local write reset offsets a") + module.add(self.localWriteResetOffsets(kernel, kernel["ExpandPointerSwap"], tensorParametersA)) + if kernel["ExpandPointerSwap"]: + # reset local write offset in asm code as well + module.add(self.localWriteResetOffsets(kernel, False, tensorParametersA)) + module.addComment1("local write reset offsets b") + module.add(self.localWriteResetOffsets(kernel, kernel["ExpandPointerSwap"], tensorParametersB)) + if kernel["ExpandPointerSwap"]: + # reset local write offset in asm code as well + module.add(self.localWriteResetOffsets(kernel, False, tensorParametersB)) + + # tail: global read + module.add(self.calculateLoopNumIter(kernel, tensorParametersA, tensorParametersB, -1)) + if self.staggerU and self.actualSummationLoops==1: + module.addComment1("remove stagger offsets for tail loop") + module.add(self.removeStagger(kernel, tensorParametersA)) + module.add(self.removeStagger(kernel, tensorParametersB)) + + # if DirectToVgprA is enabled, swap the order of global read (B->A) + tensorParameters1st = tensorParametersA + tensorParameters2nd = tensorParametersB + tc1 = 'a' + tc2 = 'b' + if kernel["DirectToVgprA"]: + tensorParameters1st, tensorParameters2nd = tensorParameters2nd, tensorParameters1st + tc1, tc2 = tc2, tc1 + module.addComment1("Update M0 for DTLDS") + moduleTmp = self.directToLdsM0Update(kernel, 1, tensorParameters1st) + module.add(replaceHolder(moduleTmp, 0)) + module.addComment1("global read %s"%tc1) + vregSetIdx = 0 + module.add(self.globalReadDo(kernel, 2, tensorParameters1st, vregSetIdx)) + module.addComment1("Update M0 for DTLDS") + moduleTmp = self.directToLdsM0Update(kernel, 1, tensorParameters2nd) + module.add(replaceHolder(moduleTmp, 0)) + module.addComment1("global read %s"%tc2) + vregSetIdx = 0 + module.add(self.globalReadDo(kernel, 2, tensorParameters2nd, vregSetIdx)) + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, 0, -1, -1, "2wait for global read")) + module.add(self.syncThreads(kernel)) + + # the following read/write addresses could be modified in recalcLocal(Read|Write)Addresses due to policy change + self.oriLraA = None # back up original local read address vgpr + self.oriLraB = None + self.oriLwaA = None # back up original local write address vgpr + self.oriLwaB = None + for uDu in range(0, kernel["DepthULdsDivisor"]): + if kernel.enabledSplitLDS: + # change local write policy from interleave-K to fractional as tail loop + # iterate LDS read address one unit of K at a time + module.addComment1("Recalc local write offsets") + module.add(self.recalcLocalWriteAddresses(kernel, tensorParametersA, uDu)) + module.add(self.recalcLocalWriteAddresses(kernel, tensorParametersB, uDu)) + if uDu > 0: + module.addComment1("sync before local write") + module.add(self.syncThreads(kernel)) + if not kernel["NoLdsWriteCode"]: + # tail: local write + module.addComment1("local write a") + tempLWCodeModA = self.localWriteDo(kernel, tensorParametersA, None) + module.add(tempLWCodeModA) + module.addComment1("local write b") + tempLWCodeModB = self.localWriteDo(kernel, tensorParametersB, None) + module.add(tempLWCodeModB) + # change local read policy from wider local read to one unit of K at a time + module.addComment1("Recalc local read offsets") + module.add(self.recalcLocalReadAddressesAB(kernel, tensorParametersA, tensorParametersB)) + # TODO: need to check if we correctly checked-in the temp VGPR used for Int8 LocalWrite (uDu, PGR=2) + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, -1, 0, -1, "5wait for local write")) + module.add(self.syncThreads(kernel)) + #module.add(self.dumpLds(kernel, 0, 8)) + + # tail: re-init local read addresses + if kernel["PrefetchGlobalRead"]: + module.addComment1("local read reset offsets a") + module.add(self.localReadResetOffsets(kernel, tensorParametersA)) + module.addComment1("local read reset offsets b") + module.add(self.localReadResetOffsets(kernel, tensorParametersB)) + module.addComment1("local read init pointers a") + module.add(self.localReadInitPointers(kernel, tensorParametersA, tensorParametersA)) + module.addComment1("local read init pointers b") + module.add(self.localReadInitPointers(kernel, tensorParametersA, tensorParametersB)) + # tail: macs + module.addComment1("tail loop: macs") + module.add(self.openLoop(kernel, tensorParametersA, tensorParametersB, -1, uDu if kernel.enabledSplitLDS else None)) + + # Try to use InnerUnroll in the tail loop if allowed: + KinInnerUnroll = kernel["InnerUnroll"] + if kernel["EnableMatrixInstruction"]: + KinInnerUnroll *= kernel["MatrixInstK"] + + tailLoopInnerUnroll = 1 + if (kernel["AssertSummationElementMultiple"] % KinInnerUnroll == 0): + tailLoopInnerUnroll = kernel["InnerUnroll"] + # need to unroll tail loop for the following cases + mEnd = 1 + if kernel["DirectToVgprA"] or kernel["DirectToVgprB"]: + mEnd = kernel["DepthU"]//KinInnerUnroll + elif kernel["DirectToLds"] and kernel["EnableMatrixInstruction"] and kernel["InnerUnroll"] == 1 and\ + (kernel["GlobalLoadVectorWidthA"] * self.bpeAB > 4 or kernel["GlobalLoadVectorWidthB"] * self.bpeAB > 4) and \ + kernel["DepthU"] // kernel["MatrixInstK"] > 2: + mEnd = kernel["DepthU"] // (kernel["MatrixInstK"] * 2) + + for mValue in range(mEnd): + pack[0] = Module() + for iui in range(0, tailLoopInnerUnroll): + doReadA = not kernel["DirectToVgprA"] + doReadB = not kernel["DirectToVgprB"] + if doReadA: + # Reading 16-bit data from LDS requires packing when ECC enabled + module.addComment1("local read a") + localReadCodeA, packCodeA = self.localReadDo(kernel, 0, iui, 0, tensorParametersA) + module.add(localReadCodeA) + pack[0].add(packCodeA) + if doReadB: + module.addComment1("local read b") + localReadCodeB, packCodeB = self.localReadDo(kernel, 0, iui, 0, tensorParametersB) + module.add(localReadCodeB) + pack[0].add(packCodeB) + # adjustment for DirectToLds case + iuiParam = iui + tailLoopInnerUnroll * mValue + if doReadA: + module.addComment1("local read inc a") + module.add(self.localReadInc(kernel, iuiParam, tensorParametersA)) + if doReadB: + module.addComment1("local read inc b") + module.add(self.localReadInc(kernel, iuiParam, tensorParametersB)) + module.add(self.wait(kernel, tensorParametersA, tensorParametersB, -1, -1, 0, "4wait for local read")) + + if kernel["EnableMatrixInstruction"]: + module.add(pack[0]) + # vgpr.checkin for all the checked-out vgpr in LocalRead + for item in list(pack[0].items()): + if item.tempVgpr != None: + self.vgprPool.checkIn(item.tempVgpr) + item.tempVgpr = None + pack[0] = Module() + + if kernel["EnableMatrixInstruction"]: + # DirectToVgpr is not applicable for tail loop + vregSetIdxMFMA = 0 + module.add(self.mfmaIter(kernel, tensorParametersA, tensorParametersB, 0, tailLoopInnerUnroll, vregSetIdxMFMA, True)) + else: + printExit("TensileLite does not support MAC instructions.") + + finalLoop = mValue == mEnd - 1 + module.add(self.closeLoop(kernel, tensorParametersA, tensorParametersB, -1, finalLoop, uDu if kernel.enabledSplitLDS else None)) + # always emit the skip-tail-loop label + module.add(self.closeLoop(kernel, tensorParametersA, tensorParametersB, -1, None, emitEndLabelOnly=True)) + # tail: close + self.inTailLoop = False + + # extra summation loops: global increment and close + for i in reversed(range(self.otherSummationLoops)): + module.addComment1("global read inc AB") + module.add(self.globalReadIncrementAB(kernel, tensorParametersA, tensorParametersB, i, 0)) + module.add(self.closeLoop(kernel, tensorParametersA, tensorParametersB, i, True)) + + module.add(self.endSummation(kernel)) + if not self.doShadowInit: + module.add(self.globalWriteWorkGroupInit(kernel)) + + #################################### + # Shift Vector Components + #################################### + if kernel["EdgeType"] == "ShiftPtr": + # GuaranteeNoPartial means each component in the vector loads is always valid. In this case we + # don't need the unshift code + + # shift vector components d0 + if not kernel["GuaranteeNoPartialA"] and tensorParametersA["rtv"]: + module.addComment1("shift vector components d0") + module.add(self.shiftVectorComponents(kernel, tensorParametersA)) + + # shift vector components d1, for MFMA version, B never entered this + if not kernel["GuaranteeNoPartialB"] and tensorParametersB["rtv"]: + module.addComment1("shift vector components d1") + module.add(self.shiftVectorComponents(kernel, tensorParametersB)) + + #################################### + # LocalSplitU reduction + #################################### + #if kernel["NumThreads"]%kernel["MacroTile0"] == 0: + if kernel["LocalSplitU"] > 1: + module.addComment2("LocalSplitU Reduction") + module.add(self.syncThreads(kernel)) + + # LocalSplitU: local write + module.addComment1("LocalSplitU: local write") + module.add(self.localSplitULocalWrite(kernel)) + + # LocalSplitU: local read + module.addComment1("LocalSplitU: local read") + module.add(self.localSplitULocalRead(kernel)) + + # LocalSplitU: local read + module.addComment1("LocalSplitU: reduction") + module.add(self.localSplitUReduction(kernel)) + + # LocalSplitU: global write indices + module.addComment1("LocalSplitU: global write indices") + module.add(self.localSplitUGlobalWriteIndices(kernel)) + + # LocalSplitU: global write + module.addComment1("LocalSplitU: global write") + module.add(self.localSplitUGlobalWrite(kernel, tensorParametersA, tensorParametersB)) + + + else: + #################################### + # NOT LocalSplitU + #################################### + + # global write indices + module.addComment1("not-LocalSplitU: global write indices") + module.add(self.notLocalSplitUGlobalWriteIndices(kernel)) + + # global write + module.addComment1("not-LocalSplitU: global write") + module.add(self.notLocalSplitUGlobalWrite(kernel, tensorParametersA, tensorParametersB)) + + # function suffix + module.add(self.functionEnd(True)) + module.add(self.functionSuffix(kernel)) + + # Tensile pass + tpo = TensilePassOptions() + tpo.removeDupActFunc = kernel["ActivationFuncCall"] + TensilePass(module, tpo) + + moduleKernelBody.addBody(module) + self.checkResources(moduleKernelBody) + + # Tensile instruction pass + TensileInstructionsPass(moduleKernelBody) + + error = self.overflowedResources + return (error, str(moduleKernelBody)) + + ############################################################################## + # Init Kernel + ############################################################################## + @abc.abstractmethod + def initKernel(self, kernel, tensorParametersA, tensorParametersB ): + # ISA version, such as 803 + self.version = tuple(kernel["ISA"]) + ti = TensileInstructions() + ti.setKernelInfo(self.version, kernel["WavefrontSize"]) + self.staggerU = kernel["StaggerU"] and (kernel["KernelLanguage"]=="Source" or kernel["BufferLoad"]) + + # Only assembly supports scheduling + self.canSchedule = (kernel["KernelLanguage"] == "Assembly") + + if self.canSchedule: + self.scheduleGlobalRead = kernel["ScheduleGlobalRead"] \ + and kernel["PrefetchGlobalRead"] \ + and kernel["BufferLoad"] # flat updates lgkmcnt counts = hard to schedule flat loads + else: + self.scheduleGlobalRead = 0 + + if self.canSchedule: + self.scheduleLocalWrite = kernel["ScheduleLocalWrite"] \ + and kernel["PrefetchGlobalRead"] \ + and kernel["BufferLoad"] # flat updates lgkmcnt counts = hard to schedule writes and loads? + else: + self.scheduleLocalWrite = 0 + + if self.canSchedule: + self.scheduleIterAlg = kernel["ScheduleIterAlg"] + else: + self.scheduleIterAlg = 0 + + self.noTailLoop = kernel["NoTailLoop"] + + self.actualSummationLoops = kernel["ProblemType"]["NumIndicesSummation"] + self.otherSummationLoops = self.actualSummationLoops-1 + self.otherSummations = kernel["ProblemType"]["NumIndicesSummation"]-1 # not loops but summations vars + + if kernel["KernelLanguage"] == "Source": + self.language = globalParameters["RuntimeLanguage"] + else: + self.language = "ASM" + self.indexChars = [] + for i in range(0, len(globalParameters["IndexChars"])): + self.indexChars.append(globalParameters["IndexChars"][i]) + self.indexChars[kernel["ProblemType"]["Index0"]] \ + = "0" + self.indexChars[kernel["ProblemType"]["Index0"]] + self.indexChars[kernel["ProblemType"]["Index1"]] \ + = "1" + self.indexChars[kernel["ProblemType"]["Index1"]] + self.unrollIdx = kernel["ProblemType"]["NumIndicesSummation"]-1 + self.unrollChar = \ + self.indexChars[kernel["ProblemType"]["IndicesSummation"][\ + self.unrollIdx]] + self.tileChar0 = self.indexChars[kernel["ProblemType"]["Index0"]] + self.tileChar1 = self.indexChars[kernel["ProblemType"]["Index1"]] + self.tileCharA = self.tileChar0 if (kernel["ProblemType"]["Tensor0"]==0) \ + else self.tileChar1 + self.tileCharB = self.tileChar0 if (kernel["ProblemType"]["Tensor0"]==1) \ + else self.tileChar1 + + """ + if kernel["ProblemType"]["Tensor0"]==0: + kernel["ThreadTileA"] = kernel["ThreadTile0"] + kernel["ThreadTileB"] = kernel["ThreadTile1"] + kernel["SubGroupA"] = kernel["SubGroup0"] + kernel["SubGroupB"] = kernel["SubGroup1"] + kernel["MacroTileA"] = kernel["MacroTile0"] + kernel["MacroTileB"] = kernel["MacroTile1"] + else: + kernel["ThreadTileB"] = kernel["ThreadTile0"] + kernel["ThreadTileA"] = kernel["ThreadTile1"] + kernel["SubGroupB"] = kernel["SubGroup0"] + kernel["SubGroupA"] = kernel["SubGroup1"] + kernel["MacroTileB"] = kernel["MacroTile0"] + kernel["MacroTileA"] = kernel["MacroTile1"] + """ + + """ + # original parameters + NumLoadsCoalesced -> NumLoadsPerpendicular + # new intermediate parameters + numReadsTile # nrt + numReadsUnroll # nru + numWritesCoal # nwc + numWritesPerp # nwp + numWritesCoalVecComp # nwvc + numWritesPerpVecComp # nwvp + """ + + # TODO load sub-vector + vwa = kernel["GlobalLoadVectorWidthA"] + vwb = kernel["GlobalLoadVectorWidthB"] + + self.numItersPLR = kernel["PrefetchLocalRead"]%kernel["LoopIters"] + self.numVgprBuffer = kernel["LoopIters"] if kernel["PrefetchLocalRead"] > kernel["LoopIters"] else kernel["PrefetchLocalRead"] + # merge N iteration's read into 1 iteration if can't coalesce read + # ex, A can coalesce read, B can't + # MergeRead 0: ds_readAx1 ds_readBx1 mfma | ds_readAx1 ds_readBx1 mfma | => ds_readAx2 ds_readBx1 mfma | ds_readBx1 mfma | + # MergeRead 1: ds_readAx1 ds_readBx1 mfma | ds_readAx1 ds_readAx1 mfma | => ds_readAx2 ds_readBx1 ds_readBx1 mfma | mfma | + MergeRead = 0 + if not kernel["ProblemType"]["TLUA"] or MergeRead or kernel["allowLRVWforTLUandMI"]: + if (not kernel["ProblemType"]["TLUA"]) and kernel["DirectToVgprA"]: + # DirectToVgpr + TLU=False case, ignore LocalReadVectorWidth and use GlobalLoadVectorWidth instead. + self.lrvwA = vwa + else: + self.lrvwA = kernel["LocalReadVectorWidth"] + else: + if kernel["EnableMatrixInstruction"]: + self.lrvwA = kernel["MIInputPerThread"] + else: + self.lrvwA = 1 + if not kernel["ProblemType"]["TLUB"] or MergeRead or kernel["allowLRVWforTLUandMI"]: + if (not kernel["ProblemType"]["TLUB"]) and kernel["DirectToVgprB"]: + # DirectToVgpr + TLU=False case, ignore LocalReadVectorWidth and use GlobalLoadVectorWidth instead. + self.lrvwB = vwb + else: + self.lrvwB = kernel["LocalReadVectorWidth"] + else: + if kernel["EnableMatrixInstruction"]: + self.lrvwB = kernel["MIInputPerThread"] + else: + self.lrvwB = 1 + + # DirectToVgprB + VW > 1 case, set lrvwB = VW + # DirectToVgprB case, global load data directly goes to Vgpr. + # If VW=2, it means lrwvB is 2. + if kernel["DirectToVgprB"] and kernel["VectorWidth"] > 1: + self.lrvwB = kernel["VectorWidth"] + # DirectToVgpr + TLU=False case + # set lrvw = VW + self.vgprValuDouble = False + #if kernel["DirectToVgprA"] and kernel["PrefetchLocalRead"] > 1 and (not kernel["ProblemType"]["TLUA"]) and kernel["VectorWidth"] > 1: + if kernel["DirectToVgprA"] and (not kernel["ProblemType"]["TLUA"]) and (not kernel["ProblemType"]["TLUB"]) or \ + kernel["DirectToVgprB"] and (not kernel["ProblemType"]["TLUB"]) and (not kernel["ProblemType"]["TLUA"]): + self.lrvwA = max(self.lrvwA, self.lrvwB) + self.lrvwB = self.lrvwA + if kernel["DepthU"] // kernel["MatrixInstK"] <= 2 and self.lrvwA > 1: + # need to double vgprValu to avoid local read overwritting vgprValu registers + self.vgprValuDouble = True + + # Wider LocalRead + if kernel["EnableMatrixInstruction"]: + self.numReadsIterCoalescedA = self.lrvwA // kernel["MIInputPerThread"] + self.numReadsIterCoalescedB = self.lrvwB // kernel["MIInputPerThread"] + if kernel["allowLRVWforTLUandMI"]: + self.numReadsIterCoalescedA = 1 + self.numReadsIterCoalescedB = 1 + else: + self.numReadsIterCoalescedA = 1 + self.numReadsIterCoalescedB = 1 + self.numIterPerCoalescedReadA = max(1,self.numReadsIterCoalescedA//kernel["InnerUnroll"]) + self.numIterPerCoalescedReadB = max(1,self.numReadsIterCoalescedB//kernel["InnerUnroll"]) + + if kernel["ScheduleIterAlg"] == 3 or kernel["ScheduleIterAlg"] == 2: + self.numMfmaPerIter = kernel["MIWaveTile"][0] * kernel["MIWaveTile"][1] * kernel["InnerUnroll"] + if kernel["ProblemType"]["DataType"].isComplex(): self.numMfmaPerIter *= 4 + + # NamedTuple is immutable + class intermediateTPValues(NamedTuple): + numReadsTile: int = -1 + numReadsUnroll: int = -1 + readTileDimVector: bool = False + writeTileDimComponents: bool = False + numWritesCoalVecComp: int = -1 + numWritesPerpVecComp: int = -1 + # convert tile/unroll to para/perp + numReadsCoalVecComp: int = -1 + numReadsPerpVecComp: int = -1 + + def readWriteVectors(mat, vw, kernel): + ######################################## + # read vectors or vector components + ######################################## + # Two dim: tile and unroll + if kernel["ProblemType"]["TLU%s"%mat]: # NT no transpose + numReadsTile = kernel["NumLoadsCoalesced%s"%mat] + numReadsUnroll = kernel["NumLoadsPerpendicular%s"%mat] + readTileDimVector = True # Vector + else: # TN yes transpose + numReadsTile = kernel["NumLoadsPerpendicular%s"%mat] + numReadsUnroll = kernel["NumLoadsCoalesced%s"%mat] + readTileDimVector = False # Scalar + + numReadsCoalVecComp = vw + numReadsPerpVecComp = 1 + + ######################################## + # write vectors or vector components + ######################################## + if kernel["ProblemType"]["TLU%s"%mat] != kernel["UnrollMajorLDS%s"%mat]: # NT no transpose + writeTileDimComponents = False # Vector + # writeCoal indicates writes should be done in the coal dim or else perp + numWritesCoalVecComp = vw // kernel["DepthULdsDivisor"] + numWritesPerpVecComp = 1 + else: # TN yes transpose + writeTileDimComponents = kernel["GlobalReadVectorWidth"] > 1 # Components + numWritesCoalVecComp = 1 + numWritesPerpVecComp = vw + + return intermediateTPValues(numReadsTile, numReadsUnroll, readTileDimVector, \ + writeTileDimComponents, numWritesCoalVecComp, numWritesPerpVecComp, \ + numReadsCoalVecComp, numReadsPerpVecComp) + + itP = dict() + itP["A"] = readWriteVectors("A", vwa, kernel) + itP["B"] = readWriteVectors("B", vwb, kernel) + + self.getTensorParameters(tensorParametersA, kernel, itP, True) + self.getTensorParameters(tensorParametersB, kernel, itP, False) + + tensorParametersA["PackedIndices"] = kernel["PackedC%uIndicesX"%tensorParametersA["tile01Idx"]] + tensorParametersB["PackedIndices"] = kernel["PackedC%uIndicesX"%tensorParametersB["tile01Idx"]] + + # condition(s) to enable init accvgpr opt (initialize only the last set of accvgpr instead of whole accvgpr) + self.useInitAccVgprOpt = False + # enable for the following conditions + if (kernel["DirectToVgprA"] or kernel["DirectToVgprB"]): + self.useInitAccVgprOpt = True + # force to disable for the following conditions + if self.useInitAccVgprOpt: + if kernel["PrefetchGlobalRead"] == 2: + # PGR=2 case, K > DepthU * 2 is necessary ( if not noTailLoop, need > DepthU * 3) + # (kernel["AssertSizeGreaterThan"][3] > DepthU * 2 (or 3) + minDUnum = 2 if self.noTailLoop else 3 + if not (3 in kernel["AssertSizeGreaterThan"].keys() and kernel["AssertSizeGreaterThan"][3] >= kernel["DepthU"] * minDUnum): + print2("InitAccVgprOpt is disabled because AssertSizeGreaterThan for K is not greater than DepthU * %u"%minDUnum) + self.useInitAccVgprOpt = False + if kernel["PrefetchGlobalRead"] == 1: + # PGR=1 case, K > DepthU * 1 is necessary ( if not noTailLoop, need > DepthU * 2) + # (kernel["AssertSizeGreaterThan"][3] > DepthU * 2 (or 3) + minDUnum = 1 if self.noTailLoop else 2 + if not (3 in kernel["AssertSizeGreaterThan"].keys() and kernel["AssertSizeGreaterThan"][3] >= kernel["DepthU"] * minDUnum): + print2("InitAccVgprOpt is disabled because AssertSizeGreaterThan for K is not greater than DepthU * %u"%minDUnum) + self.useInitAccVgprOpt = False + + # condition(s) to enable singleNLL opt + self.enableSingleNLLOpt = False + if self.noTailLoop: + pass + # so far, not enabled for DirectToVgpr + # Performance is better with Tensile, but does not perform better with HPL + #if kernel["DirectToVgprA"] or kernel["DirectToVgprB"]: + # self.enableSingleNLLOpt = True + + ############################################################################## + # Function Signature + ############################################################################## + @abc.abstractmethod + def functionSignature(self): + return "" + + ############################################################################## + # Allocate Resources + ############################################################################## + @abc.abstractmethod + def defineAndResources(self, kernel, tPA, tPB): + return "" + + ############################################################################## + # Check Resources + ############################################################################## + @abc.abstractmethod + def checkResources(self, mkb) -> None: + pass + + ############################################################################## + # Global Read Addresses: Work-Group + ############################################################################## + @abc.abstractmethod + def graWorkGroup(self, kernel): + return "" + + ############################################################################## + # Get Params For Tensor A/B + ############################################################################## + def getTensorParameters(self, tP, kernel, itP, tA): + cM = "A" if tA else "B" + tP["mirror"] = bool(kernel["ProblemType"]["MirrorDims%s" % ("A" if tA else "B")]) + if tA: # A + tP["tensorIdx"] = 0 # tensor index A=0, B=1 + tP["tileChar"] = self.tileCharA # tile char I0 or J1 + else: # B + tP["tensorIdx"] = 1 + tP["tileChar"] = self.tileCharB + + tP["isA"] = tA # is this tensor A + tP["isB"] = (not tA) # is this tensor B + tP["bpe"] = int(4*kernel["ProblemType"]["DataType"].numRegisters()) + tP["tensorChar"] = cM # tensor character A/B + tP["tileIdx"] = kernel["ProblemType"]["Index01%s"%cM] # is the tile dimension of A the 0th or 1th index, i.e. Aki, tileIdx=0 + tP["tile01Idx"] = 1 if tP["tileIdx"] else 0 + tP["lsc"] = "LSC%s"%cM # load size coalesced A/B, number of elements that get loaded along coalesced dimension with each load + tP["lsp"] = "LSP%s"%cM # load size perpendicular A/B, number of elements that get loaded along non-coalesced dimension with each load + tP["lvc"] = "LVC%s"%cM # "load size" in terms of number of short-vectors and not elements + tP["lvp"] = "LVP%s"%cM # "load size" in terms of number of short-vectors and not elements + tP["rtv"] = itP[cM].readTileDimVector # bool in the tile dimension, reads will read vectors + tP["wg"] = "WorkGroup%u" % (tP["tile01Idx"]) # these are storing the actual strong to lookup the number from kernel dictionary + tP["sg"] = "SubGroup%u" % (tP["tile01Idx"]) + tP["tt"] = "ThreadTile%u" % (tP["tile01Idx"]) + tP["mt"] = "MacroTile%u" % (tP["tile01Idx"]) + tP["tlu"] = kernel["ProblemType"]["TLU%s"%cM] # thread stride is less than unroll stride, i.e., not transposing matrix + tP["ia"] = kernel["ProblemType"]["IndexAssignments%s"%cM] # array of index assignments + tP["nrt"] = itP[cM].numReadsTile # number of reads along tile dimension + tP["nru"] = itP[cM].numReadsUnroll # number of reads along unroll dimension + tP["nrc"] = kernel["NumLoadsCoalesced%s"%cM] # number of reads along coalesced dimension + tP["nrcv"] = itP[cM].numReadsCoalVecComp # number of vector components along coalesced dimension + tP["nrp"] = kernel["NumLoadsPerpendicular%s"%cM] # number of reads along perpendicular dimension + tP["nrpv"] = itP[cM].numReadsPerpVecComp # number of vector components along perpendicular dimension + tP["nwcv"] = itP[cM].numWritesCoalVecComp # number of vector component writes along coalesced dimension + tP["nwpv"] = itP[cM].numWritesPerpVecComp # number of vector component writes along perpendicular dimension + tP["glvw"] = kernel["GlobalLoadVectorWidth%s"%cM] + tP["wtc"] = itP[cM].writeTileDimComponents # write vector components along tile dimension + tP["idx"] = kernel["ProblemType"]["Index%d"%tP["tensorIdx"]] # index 0 is tile dimension belonging to A. Note 'idx' may not be in tP['ia']. + tP["NonTemporal"] = kernel["NonTemporal%s"%cM] # non-temporal read type + + # KernelWriterAssembly + tP["localReadSwapByteOffset"] = 0 + tP["localWriteSwapByteOffset"] = 0 + tP["gpr"] = {} + + ############################################################################## + # Global Read Addresses: Tile Assignment A/B + ############################################################################## + @abc.abstractmethod + def graTileAssignment(self, kernel, tP): + return "" + + ############################################################################## + # Global Read Addresses: Unroll Assignment A/B + ############################################################################## + @abc.abstractmethod + def graUnrollAssignment(self, kernel, tP): + return "" + + ############################################################################## + # Global Read Addresses: Other Free Assignments + ############################################################################## + @abc.abstractmethod + def graOtherFreeAssignments(self): + return "" + + ############################################################################## + # Global Read Addresses: Other Summation Assignments + ############################################################################## + @abc.abstractmethod + def graOtherSummationAssignments(self, kernel): + return "" + + ############################################################################## + # Global Read Addresses: Tile Offsets A/B + ############################################################################## + @abc.abstractmethod + def graTileOffsets(self, kernel, tP): + return "" + + ############################################################################## + # Global Read Addresses: Unroll Offsets A/B + ############################################################################## + @abc.abstractmethod + def graUnrollOffsets(self, kernel, tP): + return "" + + ############################################################################## + # Global Read Addresses: Shift A/B + ############################################################################## + @abc.abstractmethod + def graShift(self, kernel, tP): + return "" + + ############################################################################## + # Global Read Addresses: Final Offsets A/B + ############################################################################## + @abc.abstractmethod + def graFinalOffsets(self, kernel, tP): + return "" + + ############################################################################## + # Global Read Addresses: Addresses A/B + ############################################################################## + @abc.abstractmethod + def graAddresses(self, kernel, tP): + return "" + + ############################################################################## + # Global Read Addresses: Increments A/B + # This function declares the increments + ############################################################################## + @abc.abstractmethod + def graIncrements(self, kernel, loopIdx, tP): + return "" + + ############################################################################## + # Local Write Addresses: Tile Assignment A/B + ############################################################################## + @abc.abstractmethod + def lwaTileAssignment(self, tP): + return "" + + ############################################################################## + # Local Write Addresses: Unroll Assignment A/B + ############################################################################## + @abc.abstractmethod + def lwaUnrollAssignment(self, kernel, tP): + return "" + + ############################################################################## + # Local Write Addresses: First Offset A/B + ############################################################################## + @abc.abstractmethod + def lwaFirstOffset(self, kernel, tP, uDu): + return "" + + ############################################################################## + # Local Read Addresses: Tile Assignment + ############################################################################## + @abc.abstractmethod + def lraTileAssignment(self, kernel, tPA, tPB): + return "" + + ############################################################################## + # Local Read Addresses: Final Offset A/B + ############################################################################## + @abc.abstractmethod + def lraFinalOffset(self, kernel, tP): + return "" + + ############################################################################## + # Local Read Addresses offset conversion for DTL + NLC > 1 + ############################################################################## + @abc.abstractmethod + def lraOffsetConversionForDTLandNLC(self, kernel, tP, offset_val, generateAsm=False, \ + finalVgpr=None, tmp1=None, tmp2=None): + return "" + + ############################################################################## + # Local Read Addresses: Declare Addresses A/B + ############################################################################## + @abc.abstractmethod + def lraDeclareAddresses(self, kernel, tP): + return "" + + ############################################################################## + # Recalculate local read addresses A/B + ############################################################################## + @abc.abstractmethod + def recalcLocalReadAddressesAB(self, kernel, tPA, tPB): + return "" + + ############################################################################## + # Recalculate local write addresses A/B + ############################################################################## + @abc.abstractmethod + def recalcLocalWriteAddresses(self, kernel, tP, uDu): + return "" + + ############################################################################## + # Define stagger parms that will be used in calculateStagger + ############################################################################## + @abc.abstractmethod + def declareStaggerParms(self, kernel): + return "" + + + ############################################################################## + # Calculate and apply stagger offsets and edge + ############################################################################## + @abc.abstractmethod + def calculateStagger(self, kernel, loopIdx): + return "" + + ############################################################################## + # Remove stagger offset (before tail loop) + ############################################################################## + @abc.abstractmethod + def removeStagger(self, kernel): + return "" + + ############################################################################## + # Calculate Loop Num Iter + ############################################################################## + @abc.abstractmethod + def calculateLoopNumIter(self, kernel, tPA, tPB, loopIdx): + return "" + + + ############################################################################## + # openShadowInit: + # Top of shadow init code + ############################################################################## + @abc.abstractmethod + def openShadowInit(self): + return "" + + ############################################################################## + # closeShadowInit: + # Top of shadow init code + ############################################################################## + @abc.abstractmethod + def closeShadowInit(self, kernel): + return "" + + ############################################################################## + # Initialize C + ############################################################################## + @abc.abstractmethod + def initC(self, kernel): + return "" + + ############################################################################## + # Open Loop + # loopIdx<0 : tail loop + ############################################################################## + @abc.abstractmethod + def openLoop(self, kernel, tPA, tPB, loopIdx, uDu, noLabelGen, beginLabelOnly): + return "" + + ############################################################################## + # Close Loop + ############################################################################## + @abc.abstractmethod + def closeLoop(self, kernel, tPA, tPB, loopIdx, \ + finalLoop, uDu, emitEndLabelOnly, oddLabel=False): + return "" + + ############################################################################## + # End Summation + ############################################################################## + @abc.abstractmethod + def endSummation(self, kernel, label = None): + return "" + + ############################################################################## + # At Least 1 Unroll + ############################################################################## + @abc.abstractmethod + def openSumAtLeastUnroll(self, kernel, prefetch, isOptNLL): + return "" + + @abc.abstractmethod + def closeSumAtLeastUnroll(self, kernel, tPA, tPB, prefetch, isOptNLL, isNGLL): + return "" + + ############################################################################## + # Global Read: Increment A/B + ############################################################################## + @abc.abstractmethod + def globalReadIncrementAB(self, kernel, tPA, tPB, loopIdx, prefetchIndex): + return "" + + ############################################################################## + # Global Read: Do It A/B + # mode: 0=prefetch, 1=unroll loop, 2=guardK + ############################################################################## + @abc.abstractmethod + def globalReadDo(self, kernel, mode, tP, vregSetIdx=0): + return "" + + ############################################################################## + # directToLds m0 update: Do It A/B + # mode: 0=prefetch, 1=unroll loop, 2=guardK + ############################################################################## + @abc.abstractmethod + def directToLdsM0Update(self, kernel, mode, tP, usePlaceHolder=False): + return "" + + ############################################################################## + # Local Write: Swap Offsets A/B + ############################################################################## + @abc.abstractmethod + def localWriteSwapOffsets(self, kernel, internalPointerSwap, tP): + return "" + + ############################################################################## + # Local Write: Reset Offsets A/B + ############################################################################## + @abc.abstractmethod + def localWriteResetOffsets(self, kernel, internalPointerSwap, tP): + return "" + + ############################################################################## + # Local Write in Prefetch Pass (PreLoop): Do It A/B + ############################################################################## + @abc.abstractmethod + def preLoopLocalWriteDo(self, kernel, tPA, tPB): + return "" + + ############################################################################## + # Local Write: Do It A/B + ############################################################################## + @abc.abstractmethod + def localWriteDo(self, kernel, tP, uDu): + return "" + + ############################################################################## + # Local Read: Swap Offsets A/B + ############################################################################## + @abc.abstractmethod + def localReadSwapOffsets(self, kernel, internalPointerSwap, tP): + return "" + + ############################################################################## + # Local Read: Reset Offsets A/B + ############################################################################## + @abc.abstractmethod + def localReadResetOffsets(self, kernel, tP): + return "" + + ############################################################################## + # Local Read: Init Pointers A/B + ############################################################################## + @abc.abstractmethod + def localReadInitPointers(self, kernel, tPA, tP): + return "" + + ############################################################################## + # Local Read: Increment A/B + ############################################################################## + @abc.abstractmethod + def localReadInc(self, kernel, tP): + return "" + + ############################################################################## + # Local Read: Do It A/B + ############################################################################## + @abc.abstractmethod + def localReadDo(self, kernel, bufferIdx, innerUnrollIndex, epsi, tP): + return "" + + ############################################################################## + # Shift Vector Components d0/1 + ############################################################################## + @abc.abstractmethod + def shiftVectorComponents(self, kernel, tP): + return "" + + ############################################################################## + # LocalSplitU: Local Write + ############################################################################## + @abc.abstractmethod + def localSplitULocalWrite(self, kernel): + return "" + + ############################################################################## + # LocalSplitU: Local Read + ############################################################################## + @abc.abstractmethod + def localSplitULocalRead(self, kernel): + return "" + + ############################################################################## + # LocalSplitU: Reduction + ############################################################################## + @abc.abstractmethod + def localSplitUReduction(self, kernel): + return "" + + ############################################################################## + # globalWriteWorkGroupInit: + # Perform work-group granularity init + ############################################################################## + @abc.abstractmethod + def globalWriteWorkGroupInit(self, kernel): + return "" + + ############################################################################## + # LocalSplitU: Global Write Indices + ############################################################################## + @abc.abstractmethod + def localSplitUGlobalWriteIndices(self, kernel): + return "" + + ############################################################################## + # LocalSplitU: Global Write + ############################################################################## + @abc.abstractmethod + def localSplitUGlobalWrite(self, kernel, tPA, tPB): + return "" + + ############################################################################## + # Not LocalSplitU: Global Write Indices + ############################################################################## + @abc.abstractmethod + def notLocalSplitUGlobalWriteIndices(self, kernel): + return "" + + ############################################################################## + # Not LocalSplitU: Global Write + ############################################################################## + @abc.abstractmethod + def notLocalSplitUGlobalWrite(self, kernel, tPA, tPB): + return "" + + ############################################################################## + # openOddNoLoadLoopForDTV + # generate open code for DirectToVgpr + odd exit case in noLoadLoop code + ############################################################################## + @abc.abstractmethod + def openOddNoLoadLoopForDTV(self, name): + return "" + + ############################################################################## + # closeOddNoLoadLoopForDTV + # generate close code for DirectToVgpr + odd exit case in noLoadLoop code + ############################################################################## + @abc.abstractmethod + def closeOddNoLoadLoopForDTV(self, name): + return "" + + ############################################################################## + # generateEvenEndLabeNoLoadLoopForDTV + # generate even end label for DirectToVgpr + ############################################################################## + @abc.abstractmethod + def generateEvenEndLabeNoLoadLoopForDTV(self, name): + return "" + + ############################################################################## + # generateOddEndVgprCopyForDTV + # generate odd end vgpr copy for DirectToVgpr + ############################################################################## + @abc.abstractmethod + def generateOddEndVgprCopyForDTV(self, kernel): + return "" + + ############################################################################## + # PrefetchGlobalRead2 + ############################################################################## + @abc.abstractmethod + def openPrefetchGlobalRead2(self, kernel): + return "" + + @abc.abstractmethod + def closePrefetchGlobalRead2(self): + return "" + + ############################################################################## + # Function End + ############################################################################## + @abc.abstractmethod + def functionEnd(self, addLabel=True): + return "" + + ############################################################################## + # Function Suffix + ############################################################################## + @abc.abstractmethod + def functionSuffix(self, kernel): + return "" + + ############################################################################## + # WaitCnt + ############################################################################## + @abc.abstractmethod + def wait(self, kernel, tPA, tPB, globalRead, localWrite, localRead, comment): + return "" + + ############################################################################## + # SyncThreads + ############################################################################## + @abc.abstractmethod + def syncThreads(self, kernel): + return "" + + ############################################################################## + # MapAcctoArch + ############################################################################## + @abc.abstractmethod + def MapAcctoArchRegs(self, kernel): + return "" + + ############################################################################## + # + # Entry Functions + # + ############################################################################## + + + ############################################################################## + # get kernel name + ############################################################################## + def getKernelFileBase(self, kernel): + if isCustomKernelConfig(kernel): + fileBase = kernel["CustomKernelName"] + elif globalParameters["ShortNames"]: + fileBase = Solution.getNameSerial(kernel, self.kernelSerialNaming) + else: + fileBase = self.shortenFileBase(kernel) + return fileBase + + def getKernelName(self, kernel): + kernelName = Solution.getNameMin(kernel, self.kernelMinNaming) + return kernelName + + def getKernelSource(self, kernel): + """ + Returns the source of the kernel, either C++ or assembly. + """ + + + fileString = "" + tensorParametersA = {} + tensorParametersB = {} + self.initKernel(kernel, tensorParametersA, tensorParametersB ) + self.stringIdx = 0 + (error, kb) = self.kernelBody( kernel, tensorParametersA, tensorParametersB) + fileString += str(kb) + + if error != 0: + if globalParameters["ForceGenerateKernel"]: + print ("warning: Generating kernel source resulted in error {}, but ForceGenerateKernel=1 so saving source".format(error)) + else: + raise RuntimeError("Generating kernel source resulted in error {}".format(error)) + return fileString + + def getAssemblyDirectory(self): + return Common.ensurePath(os.path.join(globalParameters["WorkingPath"], "assembly")) + + def byteArrayScriptSource(self): + return """ +#!/usr/bin/env python + +fileString = "" +fileString += "/*******************************************************************************\\n" +fileString += "* Copyright (C) 2016 Advanced Micro Devices, Inc. All rights reserved.\\n" +fileString += "*\\n" +fileString += "* Permission is hereby granted, free of charge, to any person obtaining a copy\\n" +fileString += '* of this software and associated documentation files (the \"Software\"), to deal\\n' +fileString += "* in the Software without restriction, including without limitation the rights\\n" +fileString += "* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop-\\n" +fileString += "* ies of the Software, and to permit persons to whom the Software is furnished\\n" +fileString += "* to do so, subject to the following conditions:\\n" +fileString += "*\\n" +fileString += "* The above copyright notice and this permission notice shall be included in all\\n" +fileString += "* copies or substantial portions of the Software.\\n" +fileString += "*\\n" +fileString += '* THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM-\\n' +fileString += "* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\\n" +fileString += "* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\\n" +fileString += "* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\\n" +fileString += "* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE-\\n" +fileString += "* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\\n" +fileString += "*******************************************************************************/\\n\\n" +fileString += "/**************************************************\\n" +fileString += "* This file was generated by Tensile: *\\n" +fileString += "* https://github.com/ROCmSoftwarePlatform/Tensile *\\n" +fileString += "**************************************************/\\n\\n\\n" +import os.path +fileString += '#include "Kernels.h"\\n\\n' +fileString += "/* code object byte array */\\n\\n" +codeObjectFileNames = [f for f in os.listdir(".") if (os.path.isfile(f) and f.endswith(".co"))] +for codeObjectFileName in codeObjectFileNames: + print codeObjectFileName + print "\\n" + kernelName=os.path.splitext(codeObjectFileName)[0] + codeObjectFile = open(codeObjectFileName, "r") + codeObjectByteArray = bytearray(codeObjectFile.read()) + codeObjectFile.close() +# write code object byte array for asm + fileString += "const unsigned char %s_coba[%u] = {\\n" % (kernelName, len(codeObjectByteArray)) + for byteIdx in range(0, len(codeObjectByteArray)): + byte = codeObjectByteArray[byteIdx] + fileString += "0x%02x" % byte + if byteIdx < len(codeObjectByteArray)-1: + fileString += "," + else: + fileString += "};\\n" + if byteIdx % 16 == 15: + fileString += "\\n" + text_file = open("Kernels.cpp", "w") + text_file.write("%s" % fileString) + text_file.close() +""" + + def writeByteArrayScript(self): + asmPath = self.getAssemblyDirectory() + + bytearrayFileName = os.path.join(asmPath,"insert_byte_array.py") + if not os.path.isfile(bytearrayFileName): + with open(bytearrayFileName, 'w') as bytearrayFile: + bytearrayFile.write(self.byteArrayScriptSource()) + os.chmod(bytearrayFileName, 0o777) + return bytearrayFileName + + def shortenFileBase(self, kernel): + base = self.getKernelName(kernel) + if len(base) <= globalParameters["MaxFileName"]: + return base + + import hashlib + import base64 + + pivot = globalParameters["MaxFileName"] * 3 // 4 + firstPart = base[:pivot] + secondPart = base[pivot:] + + secondHash = hashlib.sha256(secondPart.encode()).digest() + secondPart = base64.b64encode(secondHash, b'_-').decode() + + return firstPart + secondPart + + def getKernelObjectAssemblyFile(self, kernel): + asmPath = self.getAssemblyDirectory() + # write assembly file to assembly directory + kernelName = self.getKernelFileBase(kernel) + fileBase = os.path.join(asmPath, kernelName ) + assemblyFileName = "%s.s" % fileBase + + kernelSource = self.getKernelSource(kernel) + + if globalParameters["PrintLevel"] >= 2: + print("write_assemblyFilename %s" % assemblyFileName) + + with open(assemblyFileName, 'w') as assemblyFile: + assemblyFile.write(kernelSource) + + return assemblyFileName + + def getAssembledKernelObjectFile(self, kernel): + assemblyFileName = self.getKernelObjectAssemblyFile(kernel) + + base, ext = os.path.splitext(assemblyFileName) + objectFileName = base + '.o' + + args = self.getCompileArgs(assemblyFileName, objectFileName) + if globalParameters["PrintCodeCommands"]: + print (' '.join(args), " && ") + + subprocess.check_call(args, cwd=self.getAssemblyDirectory()) + + return objectFileName + + def getSingleCodeObjectFile(self, kernel): + objectFileName = self.getAssembledKernelObjectFile(kernel) + + base, ext = os.path.splitext(objectFileName) + coFileName = base + '.co' + + args = self.getLinkCodeObjectArgs([objectFileName], coFileName) + if globalParameters["PrintCodeCommands"]: + print (' '.join(args)) + + subprocess.check_call(args, cwd=self.getAssemblyDirectory()) + + return coFileName + + ############################################################################## + def getSourceFileString(self, kernel): + """ + Returns a string suitable for placing in Kernels.cpp. This means the actual kernel source in the case + of a source kernel, or an assembled code object byte array definition in the case of an assembly kernel, + or an empty string in the case that CodeFromFiles is true. + + In the case of an assembly kernel, this function has the side effect of creating the following files: + * An assembly source file + * An object file + * A code object file + * A Python script which can create byte array variable definitions. + """ + + try: + if kernel["KernelLanguage"] == "Assembly": + # asmPath = self.getAssemblyDirectory() + # kernelName = self.getKernelName(kernel) + + if globalParameters["GenerateSourcesAndExit"]: + # only create the assembly file. + self.getKernelObjectAssemblyFile(kernel) + return (0, "") + else: + self.writeByteArrayScript() + self.getSingleCodeObjectFile(kernel) + + # I guess in this case we are making sure that the code object file exists by executing the code + # above but we aren't placing it into the source. + return (0, "") + + else: + return (0, self.getKernelSource(kernel)) + + except subprocess.CalledProcessError as exc: + print(exc) + return (-1, "") + except RuntimeError as exc: + if globalParameters["PrintSolutionRejectionReason"]: + print(exc) + return (-2, "") + + ############################################################################## + # header file string + ############################################################################## + def getHeaderFileString(self, kernel): + kernelName = self.getKernelName(kernel) + fileString = "" # CHeader + if self.language == "HIP" or self.language == "OCL": + if not globalParameters["MergeFiles"]: + fileString += CHeader + fileString += "#pragma once\n\n" + if self.language == "HIP": + fileString += "#include \n" + fileString += "#include \n" + fileString += "#include \n" + fileString += "\n" + else: + fileString += "#include \n" + if self.language == "OCL": + fileString += "extern const char * const %s_src;\n" % kernelName + else: + # Asm should not reach here + fileString += self.functionSignature() + fileString += ";\n" + else: + if not globalParameters["MergeFiles"] or globalParameters["NumMergedFiles"] > 1: + fileString += "#pragma once\n\n" + if not globalParameters["CodeFromFiles"]: + fileString += "extern const unsigned char %s_coba[]; // code object byte array\n" % kernelName + + return fileString + + ############################################################################## + # flip Vreg set for DirectToVgpr in global read + ############################################################################## + def replaceSet(module: Item, srcStr, dst): + if isinstance(module, Module): + for item in module.items(): + replaceSet(item, srcStr, dst) + elif isinstance(item, GlobalReadInstruction): + if isinstance(item, FLATReadInstruction): + item.dst.replaceRegName(srcStr, dst) + elif isinstance(item, MUBUFReadInstruction): + item.dst.replaceRegName(srcStr, dst) + elif isinstance(item, GlobalWriteInstruction): + if isinstance(item, FLATStoreInstruction): + item.srcData.replaceRegName(srcStr, dst) + elif isinstance(item, MUBUFStoreInstruction): + item.srcData.replaceRegName(srcStr, dst) + elif isinstance(module, SWaitCnt): + assert(isinstance(dst, int)) + if module.vmcnt == srcStr: + module.vmcnt = dst + if module.lgkmcnt == srcStr: + module.lgkmcnt = dst + if module.vscnt == srcStr: + module.vscnt = dst + + def flipVregSetForDirectToVgprInGlobalRead(self, kernel, item): + # need to swap VGPR register set for odd code + baseName = "G2LA" if kernel["DirectToVgprA"] else "G2LB" # only one of them is enabled + set0 = baseName + "0" + set1 = baseName + "1" + itemStr = str(item) + if set0 in itemStr: + # replace set0 with set1 + replaceSet(item, set0, set1) + elif set1 in itemStr: + # replace set1 with set0 + replaceSet(item, set1, set0) + return item + + ############################################################################## + # waitcnt code for DirectToVgpr + ############################################################################## + def getWaitcntCodeForDirectToVgpr(self, kernel, localWriteEndIter, u, firstIter, beforeBarrier=False, NLLlast=False, oddLast=False): + inst = TextBlock(slash50("No need to wait.")) + # generate wait only if BufferLoad is True (this logic does not work with FlatLoad) + if (kernel["DirectToVgprA"] or kernel["DirectToVgprB"]) and kernel["BufferLoad"]: + pgr2 = kernel["PrefetchGlobalRead"] == 2 + numGlobalReadA = kernel["NumLoadsPerpendicularA"] * kernel["NumLoadsCoalescedA"] + numGlobalReadB = kernel["NumLoadsPerpendicularB"] * kernel["NumLoadsCoalescedB"] + numGlobalRead = numGlobalReadA if kernel["DirectToVgprA"] else numGlobalReadB + numGlobalReadAll = numGlobalReadA + numGlobalReadB + numGlobalStoreC = 0 + numReadsIterCoalesced = self.numReadsIterCoalescedA if kernel["DirectToVgprA"] else self.numReadsIterCoalescedB + waitComment = "global read wait for DirectToVgpr" + # delay DirectToVgpr global read (from previous iteration) which is not referred yet + numRegsIn1set = (numGlobalRead // kernel["LoopIters"]) * numReadsIterCoalesced + numSet = (u + numReadsIterCoalesced) // numReadsIterCoalesced + numSetMod = (u + numReadsIterCoalesced) % numReadsIterCoalesced + if numSetMod > 0: + # if mod > 0, wait is already done by mod == 0 case and no need to wait for same set of global read + return inst + needToWait = numGlobalRead - numSet * numRegsIn1set + # No global load A, B in no load loop. Reset numGlobalReadAll and numGlobalRead + numGlobalReadAll = 0 + numGlobalRead = 0 + if pgr2: + # PGR=2 case, add numGlobalReadAll for second set of prefetch + needToWait += numGlobalReadAll + if u > 0: + # count number of global read for i < u + count = 0 + for i in range(u): + globalReadStr = ' '.join([str(x) for x in self.perIterGlobalReadCode[i].flatitems()]) + count += globalReadStr.count("_buffer_load") + # PGR=2 case, global read is in LocalWriteCode + localWriteStr = ' '.join([str(x) for x in self.perIterLocalWriteCode[i].flatitems()]) + count += localWriteStr.count("_buffer_load") + needToWait += count + if u == localWriteEndIter + 1 and beforeBarrier: + # beforeBarrier case, reduce the amount of non-Vgpr global read + needToWait -= (numGlobalReadAll - numGlobalRead) + # adjustment for oddLast + # oddLast case, ignore all of above and set 0 + if oddLast: + needToWait = 0 + inst = SWaitCnt(vmcnt=needToWait, comment=waitComment) + return inst diff --git a/tensilelite/Tensile/KernelWriterActivationEnumHeader.py b/tensilelite/Tensile/KernelWriterActivationEnumHeader.py new file mode 100644 index 0000000000..11010cb362 --- /dev/null +++ b/tensilelite/Tensile/KernelWriterActivationEnumHeader.py @@ -0,0 +1,66 @@ +################################################################################ +# Copyright 2020-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from copy import deepcopy + +from .Common import globalParameters, CHeader +from .Activation import ActivationType +from .KernelWriterBase import KernelWriterBase + +class KernelWriterActivationEnumHeader(KernelWriterBase): + + def __init__(self, state): + super().__init__() + self.state["ProblemType"] = deepcopy(state["ProblemType"]) + + # derive parameter + self.language = "HIP" + self.kernelName = self.getKernelName() + + def keys(self): + return self.getKernelName() + + def getKernelName(self): + return "TensileActivationEnum_%s"%self.state["ProblemType"]["ActivationComputeDataType"].toChar() + + + def getSourceFileString(self): + fileString = "// This is a dummy file." + return (0, fileString) + + def getHeaderFileString(self): + fileString = "" # CHeader + if not globalParameters["MergeFiles"]: + fileString += CHeader + fileString += "#pragma once\n\n" + + activationCDataType = self.state["ProblemType"]["ActivationComputeDataType"] + enumName = "ActivationType_%s"%activationCDataType.toChar() + fileString += "namespace Tensile {\n" + fileString += "enum class %s : uint32_t\n"%enumName + fileString += "{\n" + enumList = ActivationType.getEnumStrList(activationCDataType) + for idx, enumStr in enumerate(enumList): + fileString += " %s = %s,\n"%(ActivationType(enumStr).toEnum(), ActivationType.getEnumIndex(enumStr)) + fileString += "};\n" + fileString += "} // End of namespace Tensile\n" + + return fileString diff --git a/tensilelite/Tensile/KernelWriterActivationFunction.py b/tensilelite/Tensile/KernelWriterActivationFunction.py new file mode 100644 index 0000000000..90dcb64538 --- /dev/null +++ b/tensilelite/Tensile/KernelWriterActivationFunction.py @@ -0,0 +1,66 @@ +################################################################################ +# Copyright 2020-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from copy import deepcopy + +from .TensileInstructions import TensileInstructions +from .Common import globalParameters, CHeader +from .Activation import ActivationInline +from .KernelWriterBase import KernelWriterBase + +class KernelWriterActivationFunction(KernelWriterBase): + + def __init__(self, state): + super().__init__() + self.state["ProblemType"] = deepcopy(state["ProblemType"]) + self.state["Kernel"] = state["Kernel"] + self._tf = TensileInstructions() + + # derive parameter + self.language = "HIP" + self.kernelName = self.getKernelName() + + def keys(self): + return self.getKernelName() + + def getKernelName(self): + return "TensileActivation_%s_%s"%(self.state["ProblemType"]["ActivationComputeDataType"].toChar(), \ + self.state["ProblemType"]["ActivationType"]) + + + def getSourceFileString(self): + fileString = "// This is a dummy file." + return (0, fileString) + + def getHeaderFileString(self): + activationCDataType = self.state["ProblemType"]["ActivationComputeDataType"] + fileString = "" # CHeader + if not globalParameters["MergeFiles"]: + fileString += CHeader + fileString += "#pragma once\n\n" + fileString += "#include \"TensileActivationEnum_%s.h\"\n"%activationCDataType.toChar() + fileString += "\n" + + self._tf.setKernelInfo(tuple(self.state["Kernel"]["ISA"]), self.state["Kernel"]["WavefrontSize"]) + activation = ActivationInline(activationCDataType) + fileString += activation.generateInlineAssemblyFunction(self.state["ProblemType"]["ActivationType"]) + + return fileString diff --git a/tensilelite/Tensile/KernelWriterActivationOnly.py b/tensilelite/Tensile/KernelWriterActivationOnly.py new file mode 100644 index 0000000000..7512c9ef83 --- /dev/null +++ b/tensilelite/Tensile/KernelWriterActivationOnly.py @@ -0,0 +1,266 @@ +################################################################################ +# Copyright 2020-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from copy import deepcopy + +from .Common import globalParameters, CHeader +from .KernelWriterBase import KernelWriterBase + +class KernelWriterActivationOnly(KernelWriterBase): + + def __init__(self, state): + super().__init__() + + self.state["ProblemType"] = deepcopy(state["ProblemType"]) + self.state["_GlobalAccumulation"] = state["_GlobalAccumulation"] + + # derive parameter + self.language = "HIP" + self.kernelName = self.getKernelName() + + # determine chars for fast access + self.indexChars = [] + for i in range(0, len(globalParameters["IndexChars"])): + self.indexChars.append(globalParameters["IndexChars"][i]) + self.indexChars[self.state["ProblemType"]["Index0"]] = "0" + self.indexChars[self.state["ProblemType"]["Index0"]] + self.indexChars[self.state["ProblemType"]["Index1"]] = "1" + self.indexChars[self.state["ProblemType"]["Index1"]] + self.tileChar0 = self.indexChars[self.state["ProblemType"]["Index0"]] + self.tileChar1 = self.indexChars[self.state["ProblemType"]["Index1"]] + + + def functionSignature(self): + kStr = "" + + # self.state name + kStr += self.endLine + kStr += "extern \"C\"" + self.endLine + kStr += "__global__ " + kStr += "void %s" % ( self.kernelName ) + kStr += "(" + self.endLine + + # pointers + ptrStr = self.state["ProblemType"]["DestDataType"].toDevice(self.language) + isStridedBuffer = self.state["ProblemType"]["StridedBatched"] + ptrStr += "" if isStridedBuffer else "*" + batch = "" if isStridedBuffer else "Batch" + kStr += " " + ptrStr + " * " + batch + "D," + self.endLine + + ptrStr = self.state["ProblemType"]["DestDataType"].toDevice(self.language) + isStridedBuffer = self.state["ProblemType"]["StridedBatched"] + ptrStr += "" if isStridedBuffer else "*" + batch = "" if isStridedBuffer else "Batch" + + activationCDataType = self.state["ProblemType"]["ActivationComputeDataType"] + enumName = "Tensile::ActivationType_%s"%activationCDataType.toChar() + if self.state["ProblemType"]["ActivationType"] != 'none': + if self.state["ProblemType"]["ActivationType"] == 'all': + kStr += " %s const activationType,%s" % (enumName, self.endLine) + for name in self.state["ProblemType"]["ActivationType"].getAdditionalArgStringList(): + kStr += " %s const %s,%s" % (activationCDataType.toDevice(self.language), name, self.endLine) + + # strides + firstStrideCD = 1 + if self.state["ProblemType"]["UseInitialStridesCD"]: + firstStrideCD = 0 + lastStrideC = self.state["ProblemType"]["NumIndicesC"] + for i in range(firstStrideCD, lastStrideC): + kStr += " unsigned int const strideD%s,%s" % (self.indexChars[i], self.endLine) + + # sizes + for i in range(0, self.state["ProblemType"]["NumIndicesC"]): + kStr += " unsigned int const size%s,%s" % (self.indexChars[i], self.endLine) + + # offset + kStr += " unsigned int offsetD)%s" % self.endLine + + return kStr + + + ############################################################################## + # Kernel Body Beta-Only + ############################################################################## + def kernelBody(self): + problemType = self.state["ProblemType"] + + kStr = "" + kStr += "{%s" % self.endLine + + ######################################## + # defined initial strides + firstStride = 0 + if problemType["UseInitialStridesCD"]: + # no strides #defined + lastStrideC = 0 + assert 0 # need to fix beta-clear routine to pass initial stride parms + else: + # #define initial stride + kStr += "/* hard-coded initial strides */%s" % self.endLine + lastStrideC = 1 + for i in range(firstStride, lastStrideC): + kStr += "#define strideD" + self.indexChars[i] + " 1" + self.endLine + + ######################################## + # GLOBAL_D() + kStr += "#define GLOBAL_D(IDX%s" % self.indexChars[0] + for i in range(1, problemType["NumIndicesC"]): + kStr += ", IDX%s" % self.indexChars[i] + indexChar = self.indexChars[0] + kStr += ") (( (IDX%s)*strideD%s" % (indexChar, indexChar) + for i in range(1, problemType["NumIndicesC"]): + indexChar = self.indexChars[i] + kStr += " + (IDX%s)*strideD%s" % (indexChar, indexChar) + kStr += " ))" + self.endLine + + ######################################## + # multi buffers GSU: Accumulate all GSU buffer + indexChar = self.indexChars[0] + kStr += " uint64_t id = %s(0);%s" % (self.getGlobalIdStr, self.endLine) + kStr += " if (id >= (size%s" % self.indexChars[0] + for i in range(1, problemType["NumIndicesC"]): + kStr += "*size%s" % self.indexChars[i] + kStr += "))%s" % self.endLine + kStr += " return;%s" % self.endLine + + kStr += self.endLine + kStr += " uint64_t id0" + for i in range(1, problemType["NumIndicesC"]): + kStr += ", id%d" % i + kStr += ";%s" % self.endLine + + for i in range(0, problemType["NumIndicesC"]): + kStr += " id%d = id %% size%s;%s" % (i, self.indexChars[i], self.endLine) + kStr += " id = id / size%s;%s" % (self.indexChars[i], self.endLine) + + nonTileFreeIndices = [] + + # apply batch + if not self.state["ProblemType"]["StridedBatched"]: + nonTileFreeIndices = list(range(0, self.state["ProblemType"]["NumIndicesC"])) + nonTileFreeIndices.remove(self.state["ProblemType"]["Index0"]) + nonTileFreeIndices.remove(self.state["ProblemType"]["Index1"]) + + kStr += self.endLine + kStr += " uint64_t wg = 0" + batchStride = "1" + for i in nonTileFreeIndices: + kStr += " + id%d * %s " % (i, batchStride) + batchStride += " * size%s" % self.indexChars[i] + kStr += ";" + self.endLine + + ptrStr = self.state["ProblemType"]["DestDataType"].toDevice(self.language) + kStr += " " + ptrStr + " * D = BatchD[wg];" + self.endLine + + # apply offset + kStr += self.endLine + kStr += " D = D + offsetD;" + self.endLine + + + kStr += self.endLine + ######################################## + # D index + kStr += " %s idxD = GLOBAL_D( (%s)" % (self.uint64Str, self.uint64Str) + for i in range(problemType["NumIndicesC"]): + tmpStr = '' + if i in nonTileFreeIndices: + tmpStr = '0' + else: + tmpStr = 'id%d' % i + kStr += ', ' if i else '' + kStr += tmpStr + kStr += ");%s" % (self.endLine) + + ######################################## + # Activation + typeStr = self.state["ProblemType"]["DestDataType"].toDevice(self.language) + typeActivationStr = self.state["ProblemType"]["ComputeDataType"].toDevice(self.language) if self.state["ProblemType"]["ActivationHPA"] else \ + self.state["ProblemType"]["DestDataType"].toDevice(self.language) + if self.state["ProblemType"]["ActivationType"] != 'none': + names = "" + if self.state["ProblemType"]["ActivationType"] == 'all': + names += ", activationType" + for name in self.state["ProblemType"]["ActivationType"].getAdditionalArgStringList(): + names += (", " + name) + rvalueStr = "activation((%s)D[idxD]%s)" % (typeActivationStr, names) + + if self.state["ProblemType"]["DestDataType"].isInt8() and self.state["ProblemType"]["HighPrecisionAccumulate"]: + rvalueStr = "min(127, max(-128, %s))" % rvalueStr + + kStr += " D[idxD] = (%s)%s;%s" % (typeStr, rvalueStr, self.endLine) + + ######################################## + # end + kStr += "}%s" % self.endLine + for i in range(firstStride, lastStrideC): + kStr += "#undef strideD" + self.indexChars[i] + self.endLine + kStr += "#undef GLOBAL_D%s" % (self.endLine) + + return kStr + + + def getKernelName(self): + indexChars = globalParameters["IndexChars"] + # C dimensions + name = "D" + for i in range(0, self.state["ProblemType"]["NumIndicesC"]): + name += indexChars[i].lower() + name += "_" + name += self.state["ProblemType"]["DestDataType"].toChar() + if self.state["ProblemType"]["ActivationType"] == 'all': + name += "_%s"%"A" + elif self.state["ProblemType"]["ActivationType"] != 'none': + name += "_%s"%str(self.state["ProblemType"]["ActivationType"]).upper() + name += ("h" if self.state["ProblemType"]["ActivationHPA"] else "") + + return name + + + def getSourceFileString(self): + fileString = "" + if not globalParameters["MergeFiles"]: + fileString += "\n" + fileString += "#include \"%s.h\"\n" % self.kernelName + fileString += "\n" + + fileString += self.functionSignature() + fileString += self.kernelBody() + + return (0, fileString) + + def getHeaderFileString(self): + fileString = "" # CHeader + if not globalParameters["MergeFiles"]: + fileString += CHeader + fileString += "#pragma once\n\n" + fileString += "\n" + fileString += "#include \n\n" + fileString += "#include \n" + fileString += "#include \n" + fileString += "\n" + activationCDataType = self.state["ProblemType"]["ActivationComputeDataType"] + if self.state["ProblemType"]["ActivationType"] == 'all': + fileString += "#include \"TensileActivation_%s_%s.h\"\n"%(activationCDataType.toChar(), \ + self.state["ProblemType"]["ActivationType"]) + fileString += "\n" + + fileString += self.functionSignature() + fileString += ";\n" + + return fileString diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py new file mode 100644 index 0000000000..87d6f5b0d0 --- /dev/null +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -0,0 +1,8278 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .TensileInstructions import getGfxName, KernelBody, Label, Macro, Module, RegSet, SrdUpperValue, \ + StructuredModule, TextBlock, ValueEndif, ValueIf, ValueSet, SignatureBase, \ + MUBUFModifiers, RegisterContainer, InstType, SelectBit, SGetPositivePCOffset, \ + SLongBranchPositive, SCLongBranchScc0, SCLongBranchScc1, \ + SBranchIfZero, SBranchIfNotZero, SMulInt64to32, DSInit, \ + ArgumentLoader, Assert, bomb, Dump, vectorStaticDivideAndRemainder, \ + vectorStaticDivide, vectorStaticRemainder, \ + scalarStaticDivideAndRemainder, sMagicDiv, staticMultiply, \ + scalarStaticMultiply, MacroVMagicDiv, MacroVDynamicScalarDiv, \ + RegisterPool, allocTmpGpr, RegisterPoolResource, Holder, \ + vgpr, sgpr, accvgpr, mgpr, LabelManager, log2, ceilDivide, DataType, \ + dataTypeToMfmaInstTypePair, getAsmCompileArgs, getAsmLinkCodeObjectArgs +from .TensileInstructions.Instructions import * +from .TensilePass import getActivationFunctionModuleName, getActivationBranchModuleName +from .Common import globalParameters, print2, printExit, printWarning, roundUp +from .Component import Component +from .KernelWriter import KernelWriter +from .SolutionStructs import isPackedIndex +from .AsmMemoryInstruction import MemoryInstruction +from .AsmStoreState import StoreState +from .Activation import ActivationType, ActivationModule + +from dataclasses import dataclass, field +from math import ceil, log +from copy import deepcopy +from typing import NamedTuple +import collections + +################################################################################ +# Assembly Kernel +################################################################################ +@dataclass +class VgprStruct: + coord0: int = field(init=False) + coord1: int = field(init=False) + + # StoreRemapVectorWidth + storeRemapLW: int = field(init=False) + storeRemapLR: int = field(init=False) + storeRemapCoord0: int = field(init=False) + storeRemapCoord1: int = field(init=False) + storeRemapOffsetCoord1: int = field(init=False) + # BufferStore + cinRowPtr: int = field(init=False) + coutRowPtr: int = field(init=False) + + # FlatStore + addrD: int = field(init=False) + addrC: int = field(init=False) + addrBias: int = field(init=False) + + def __post_init__(self): + self.coord0 = -1 + self.coord1 = -1 + self.storeRemapLW = -1 + self.storeRemapLR = -1 + self.storeRemapCoord0 = -1 + self.storeRemapCoord1 = -1 + self.storeRemapOffsetCoord1 = -1 + self.cinRowPtr = -1 + self.coutRowPtr = -1 + self.addrD = -1 + self.addrC = -1 + self.addrBias = -1 + +class KernelWriterAssembly(KernelWriter): + + ############################################################################## + # Init + ############################################################################## + def __init__( self, kernelMinNaming, kernelSerialNaming ): + super(KernelWriterAssembly, self).__init__( \ + kernelMinNaming, kernelSerialNaming) + self.do = {} + self.do["PreLoop"] = True + self.do["GlobalReadA"] = True + self.do["GlobalReadB"] = True + self.do["GlobalInc"] = True + self.do["LocalWrite"] = True + self.do["LocalReadA"] = True + self.do["LocalReadB"] = True + self.do["Wait"] = True + self.do["Sync"] = True + self.do["MAC"] = True + self.do["PostLoop"] = True + self.do["ApplyAlpha"] = True + self.do["GlobalWrite"] = True + self.do["EdgeWrite"] = True + self.do["KeepDirectToLdsAlloc"] = False # If true, keep regs used for LDS alloc even if not used + + # Various debug flags and modes + self.db = {} + self.db["EnableAsserts"] = globalParameters["EnableAsserts"] # Enable assertion codegen. Requires 2 SGPR. + self.db["DebugKernelMaxItems"] = 16 # Capture first N(=16) print values, ignore subsequent. If -1, debug writing is faster but writing more than 16 values is undefined. + + # Chicken bit to add conservative synchronization at strategic points: + # 0x01 = waitcnt + barrier after vector load + # 0x02 = waitcnt at self.wait() for globalRead + # 0x04 = waitcnt at self.wait() for localWrite + # 0x08 = waitcnt at self.wait() for localRead + # 0x10 = waitcnt after summation iteration, this can catch lingering ds or vm activity from summation loop + # 0x20 = waitcnt before each write batch + # 0x40 = waitcnt after each write batch + self.db["ConservativeWaitCnt"] = 0x00 + + self.db["InitLds"] = False # Initialize LDS at start of kernel + self.initLdsValue = 0xFFFFFFFF # Value to use for LDS Init, if enabled + + # InitSgpr and InitVgpr can initialize at various points: + # 0x1: Init at kernel start + # 0x2: Init at end of summation loop (after tail too) - this is just before store loop + self.db["InitSgpr"] = 0x0 # init SGPRs + self.initSgprValue = 0x0 # Value to use for Sgpr Init, if enabled + + self.db["InitVgpr"] = 0x0 # init VGPRs + self.initVgprValue = 0xFFFFFFFF # Value to use for Vgpr Init, if enabled + + # Debug and Check flags: + # Check A and B values loaded from memory to ensure they are 1 + # Requires DataInitTypeAB=1. + # Only works if the problem uses full tiles (no edges) + # Mismatches will assert (generate GPUVM fault) + self.db["CheckValue1A"] = globalParameters["EnableDebugA"] + self.db["CheckValue1B"] = globalParameters["EnableDebugB"] + + # Check value in C matrix. + # Caveats: + # - Only works for single, or Half/BF with HPA. + # - Checks after alpha calc for each element. Later elements (in the TT) will not yet have applied their alpha. + # - Only works if matrix is integral multiple of macro-tile (no edges) - check is dumb so doesn't know + # which work-items are outside the valid edge. + # - Does not work in OptNoLoadLoop + self.db["CheckValueC"] = globalParameters["EnableDebugC"] + # value expected if CheckValueC is set. Use '.' for FP. + # For example could be 16.0 if U=8 and alpha=2 + self.db["ValueCExpectedValue"] = globalParameters["ExpectedValueC"] + + # Force an expected value for all C outputs. + # May be useful for checking store path + # See same caveats as CheckValueC + self.db["ForceExpectedValue"] = globalParameters["ForceCExpectedValue"] + + # Force VSerial value into the output, this will + # not match reference but can be useful to see which work-items are + # storing which values + # See same caveats as CheckValueC + self.db["ForceVSerial"] = False + + # can't do both of these since they both override output + assert (not (self.db["ForceExpectedValue"] and self.db["ForceVSerial"])) + + self.db["ForceInputValueA"] = False + self.db["ForceInputValueB"] = False + self.db["ForceValueA"] = 1.0 + self.db["ForceValueB"] = 1.0 + + self.db["CheckStoreC"] = -1 # -1 disables, reload and verify output data. Specify expected constant value. + #self.db["CheckStoreC"] = 1024.0 # possible value + + self.db["ForceEdgeStores"] = 0 # 1=force use of edge store path for all tiles, 2=add assert in non-edge stores + self.db["AssertNoEdge"] = 0 # Add assert in edge store code so crashes if executed + + # print vgpr register pool checkins and checkouts + self.db["PrintRP"] = 0 + self.db["AssertOnSgprOverflow"] = False + self.db["PrintStoreRegisterDb"] = False + + self.dumpData = Dump("DebugKernelItems", "AddressDbg", self.db["DebugKernelMaxItems"], \ + globalParameters["DebugKernel"]) + + self.maxVgprs = 256 + # max allowed is 112 out of 112 , 6 is used by hardware 4 SGPRs are wasted + self.maxSgprs = 102 + self.maxOccupancy = 10 + + self.labels = LabelManager() + self.inTailLoop = False + + @property + def laneSGPRCount(self) -> int: + """ How many SGPRs does it take to have one bit per lane? """ + if self.kernel["WavefrontSize"] == 64: + return 2 + else: + return 1 + + def getVgprOccupancy(self, numThreads, vgprs, unifiedVgprRegs=False): + multiplier = int(ceil(max(numThreads, 256) / 256.0)) # example: wg=512 multiplier=2, 1024=4 + maxOccupancy = self.maxOccupancy//multiplier + + vgprAllocateAligned = 4 if not unifiedVgprRegs else 8 + totalVgprs = self.maxVgprs if not unifiedVgprRegs else self.maxVgprs*2 + vgprsAligned = int(ceil(vgprs/vgprAllocateAligned))*vgprAllocateAligned + vgprsAligned *= multiplier + + if vgprsAligned > totalVgprs: return 0 + elif vgprsAligned < 1: return maxOccupancy + occupancy = min(totalVgprs//vgprsAligned, maxOccupancy) + + #print("vgprs = ", vgprs, "vgprsAligned = ", vgprsAligned, "unifiedVgprRegs = " ,unifiedVgprRegs, "Occupancy = ", occupancy) + + return occupancy + + ######################################## + def getOccupancy(self, numThreads, vgprs, ldsSize, accvgprs=0, unifiedVgprRegs=False): + + ldsLimitedOccupancy = self.getLdsLimitedOccupancy(ldsSize) + + if not unifiedVgprRegs: + vgprLimitedOccupancy = self.getVgprOccupancy(numThreads, vgprs, unifiedVgprRegs) + accvgprLimitedOccupancy = self.getVgprOccupancy(numThreads, accvgprs, unifiedVgprRegs) + else: + vgprLimitedOccupancy = self.getVgprOccupancy(numThreads, vgprs+accvgprs, unifiedVgprRegs) + accvgprLimitedOccupancy = vgprLimitedOccupancy + + return min(ldsLimitedOccupancy, vgprLimitedOccupancy, accvgprLimitedOccupancy) + + # TODO: also consider sgpr + def getMaxRegsForOccupancy(self, numThreads, vgprs, ldsSize, accvgprs=0, unifiedVgprRegs=False): + lastVgprs = vgprs + considerAccVgprs = 0 if not unifiedVgprRegs else accvgprs + totalVgprs = self.maxVgprs if not unifiedVgprRegs else self.maxVgprs*2 + + initOccupancy = self.getOccupancy(numThreads, vgprs, ldsSize, accvgprs, unifiedVgprRegs) + if initOccupancy == 0: return lastVgprs + + while (vgprs + considerAccVgprs) < totalVgprs and vgprs < self.maxVgprs: + vgprs += 1 + if self.getVgprOccupancy(numThreads, vgprs + considerAccVgprs, unifiedVgprRegs) >= initOccupancy: + lastVgprs = vgprs + next + else: + break + + return lastVgprs + + @staticmethod + def getLdsLimitedOccupancy(ldsSize): + maxLds = 65536 + # As ldsSize gets large, rounding might push us slightly higher than maxLds. + # Clamp at maxLds + ldsSize = min(ldsSize + 255, maxLds) & 0x1ff00 # 256-byte granularity + + ldsLimitedOccupancy = maxLds//ldsSize + return ldsLimitedOccupancy + + @staticmethod + def getLdsSize(kernel): + ldsSize = kernel["LdsNumElements"] * kernel["ProblemType"]["DataType"].numBytes() + return ldsSize + + ######################################## + def sizeRef(self, idx): + """ + Return sgpr() or const with the specified size + See above definitions for how these are mapped to Free or Sum sizes + based on the problem definition. + """ + idxChar= globalParameters["IndexChars"][idx] + return sgpr("Size%s"%idxChar) + + def loopChar(self, kernel, loopIdx): + loopDim = kernel["ProblemType"]["IndicesSummation"][loopIdx] + return globalParameters["IndexChars"][loopDim] + + def loopSizeRef(self, kernel, loopIdx): + loopDim = kernel["ProblemType"]["IndicesSummation"][loopIdx] + return self.sizeRef(loopDim) + + def loopCounterName(self, kernel, loopIdx): + return "LoopCounter%s"%(self.loopChar(kernel, loopIdx)) + + def loopCounter(self, kernel, loopIdx): + """ + Return loopCounter for loopIdx wrapped in "SGPR" syntax + loop idx is 0...unrollIdx + """ + return sgpr(self.loopCounterName(kernel,loopIdx)) + + def checkLastIter(self, kernel, comment="at last iteration?"): + """ Return last iteration of unroll loop. """ + return SCmpEQU32(src0=self.loopCounter(kernel, self.unrollIdx), src1=0, comment=comment) + + def isConstUnitStride(self, stride): + if isinstance(stride, RegisterContainer): + return False + return stride.startswith("const") + + ######################################## + def strideRef(self, tc, dim): + """ + Return sgpr with specified stride or define starting with const if constant. + dim is index 0...max indices and is in global index space. + """ + problemType = self.kernel["ProblemType"] + if tc in ['A','B']: + if not problemType["UseInitialStridesAB"] and \ + dim == problemType["IndexAssignments%s"%tc][0]: + return ("constStride%s%s"%(tc,self.indexChars[dim])) + else: + return sgpr("Stride%s%s"%(tc,self.indexChars[dim])) + elif tc in ['D','C']: + if not problemType["UseInitialStridesCD"] and dim == 0: + return ("constStride%s%s"%(tc,self.indexChars[dim])) + else: + return sgpr("Stride%s%s"%(tc,self.indexChars[dim])) + else: + raise ValueError("unexpected tensorChar='%s' in stride function"%tc) + + ############################################################################## + # Find Memory Instruction For Width and Stride + ############################################################################## + def findMemoryInstructionForWidthStride(self, width, strides, combine, \ + instructions): + for i in range(0, len(instructions)): + instruction = instructions[i] + numAddresses = instruction.numAddresses + numOffsets = instruction.numOffsets + offsetMultiplier = instruction.offsetMultiplier + blockWidth = instruction.blockWidth + valid = True + if width < blockWidth: + valid = False + if combine: # try to combine ops + if numOffsets > 0: # if inst combines using offsets + for stride in strides: + if stride % offsetMultiplier != 0: + valid = False + else: # don't try to combine ops + if numOffsets > 1 or numAddresses > 1: + valid = False + if valid: + return i + else: + continue + + printWarning("Could not find valid memory instruction for width=%f" % width) + return len(instructions) + + ############################################################################## + # Select Memory Instruction + # when selecting instruction, need to support stride in both dims + ############################################################################## + def selectMemoryInstruction(self, + operation, # ReadGlobal, WriteLocal, ReadLocal + width, # num registers 1 chunk + write2, # Para, Perp, None + para2, # NumLoadsPara >= 2 + perp2, # NumLoadsPerp >= 2 + strides ): + + #instructions = self.memoryArchitecture[operation] + instructions = self.memoryInstructions[operation] + # try to combine + if (write2 == "Coalesced" and para2) \ + or (write2 == "Perpendicular" and perp2): + instructionIdx = self.findMemoryInstructionForWidthStride( \ + width, strides, True, instructions) + # don't or can't combine + else: + instructionIdx = self.findMemoryInstructionForWidthStride( \ + width, strides, False, instructions) + + if instructionIdx < len(instructions): # found + return instructionIdx + else: + raise RuntimeError("Could not find valid memory instruction for operation=%s, width=%f, kernel=%s" %(operation, width, self.kernelName)) + + def initGlobalReadMemoryInstruction(self, instructions, tP, bpr): + # globalRead instruction; no flat_load2_* + globalReadWidth = float(tP["nrcv"]*tP["bpe"])/bpr + globalRead2Coalesced = tP["nrc"] > 1 + globalRead2Perpendicular = tP["nrp"] > 1 + globalReadInstructionIdx = self.selectMemoryInstruction("GlobalRead", globalReadWidth, \ + False, \ + globalRead2Coalesced, globalRead2Perpendicular, [] ) + + tP["globalReadInstruction"] = instructions["GlobalRead"][globalReadInstructionIdx] + + def initLocalWriteMemoryInstruction(self, instructions, kernel, tP, bpr): + ######################################## + # localWrite instruction + # for local, tile->para, unroll->perp + # self.localWriteWidth = 1 if tP["wtc"] else kernel["VectorWidth"] + # wtc = writeTileDimComponents + localWriteWidth = tP["nwcv"]*tP["bpe"]//bpr + if localWriteWidth < 1: + localWriteWidth = (1.0*tP["nwcv"]*tP["bpe"])/bpr + localWrite2Coalesced = tP["nrc"]>1 or tP["wtc"] + localWrite2Perpendicular = tP["nrp"]>1 + # localWrite stride tile + if tP["tlu"]: + if tP["wtc"]: + localWriteStrideTile = 1 + else: + localWriteStrideTile = kernel[tP["lsc"]] + else: + localWriteStrideTile = kernel[tP["lsp"]] + localWriteStrideTile = localWriteStrideTile*tP["bpe"]//bpr + # localWrite stride unroll + if tP["tlu"]: + localWriteStrideUnroll = kernel[tP["lsc"]]*kernel[tP["mt"]] + else: + if tP["wtc"]: + localWriteStrideUnroll = 1*kernel[tP["mt"]] + else: + localWriteStrideUnroll = kernel[tP["lsc"]]*kernel[tP["mt"]] + localWriteStrideUnroll = \ + (localWriteStrideUnroll*tP["bpe"])//bpr + localWriteInstructionIdx = self.selectMemoryInstruction("LocalWrite", localWriteWidth, \ + False, \ + localWrite2Coalesced, localWrite2Perpendicular, + [localWriteStrideTile, localWriteStrideUnroll] ) + + tP["localWrite2Coalesced"] = localWrite2Coalesced + tP["localWrite2Perpendicular"] = localWrite2Perpendicular + tP["localWriteStrideTile"] = localWriteStrideTile + tP["localWriteStrideUnroll"] = localWriteStrideUnroll + tP["localWriteInstruction"] = instructions["LocalWrite"][localWriteInstructionIdx] + + def initLocalReadMemoryInstruction(self, instructions, kernel, tP, bpr, lrvw): + + tChar = "A" if tP["isA"] else "B" + localReadWidth = (kernel["VectorWidth"] * tP["bpe"]) // bpr + if kernel["EnableMatrixInstruction"]: + if tP["tlu"] and kernel["allowLRVWforTLUandMI"]: + localReadWidth = (lrvw * tP["bpe"]) // bpr + else: + localReadWidth = tP["bpe"] / bpr + if kernel["UnrollMajorLDS%s"%tChar]: + localReadWidth = (lrvw * tP["bpe"]) // bpr + # for directToLds x2/x4 support + if kernel["DirectToLds%s"%tChar]: + localReadWidth = 1 # for fp64 its f32 + + #localReadStridePerpendicular = 0 + localRead2Perpendicular = False + localReadStrideCoalesced = \ + kernel[tP["tt"]] * tP["bpe"]//bpr + localRead2Coalesced = kernel[tP["tt"]]//kernel["VectorWidth"] > 1 + localReadInstructionIdx = self.selectMemoryInstruction("LocalRead", localReadWidth, \ + False, \ + localRead2Coalesced, localRead2Perpendicular, + [localReadStrideCoalesced] ) + tP["localRead2Coalesced"] = localRead2Coalesced + tP["localRead2Perpendicular"] = localRead2Perpendicular + tP["localReadStrideCoalesced"] = localReadStrideCoalesced + tP["localReadInstruction"] = instructions["LocalRead"][localReadInstructionIdx] + + def allocTmpSgpr(self, num: int, alignment=None, tag=None): + def overflowListener(e): + self.overflowedResources = 2 + if self.db["AssertOnSgprOverflow"]: + raise e + + tmpSgpr = allocTmpGpr(self.sgprPool, num, self.maxSgprs, alignment, tag, overflowListener) + return tmpSgpr + + def defineSgpr(self, name, numSgprs, align=1): + if numSgprs == 0: return + + sgprIdx = self.sgprPool.checkOutAligned(numSgprs, align, tag=name, preventOverflow=0) + #self.sgprIdx = roundUpToNearestMultiple(self.sgprIdx,align) + #print (name, "->", self.sgprIdx, "+", numSgprs) + self.sgprs[name] = sgprIdx + + return sgprIdx + + def undefineSgpr(self, name): + self.sgprPool.checkIn(self.sgprs[name]) + # later references will result in compile-time error (with odd 'error: expected relocatable expression') + # and 'Kernel ... not found in any loaded module' + # TODO: temporarily disable undef as it seems to have issues + return ValueSet(name=name, value="UNDEF", format = -1) + + def defineVariableSgprs(self, kernel): + #------------------------ + # Registers defined below this point are not available in the post-loop + # Post-loop is after tail loop exits, ie the store code. + # (we reclaim them to use as temps, typically for execmasks) + # Mostly impacts flat kernels and GSU edge since these need SGPR + # for conditionals + # self.lastPostLoopSgpr = self.sgprPool.size() + + if self.use64bShadowLimit: + # If need more SGPR could overlap this with the Tensor2dSize regs + self.defineSgpr("ShadowLimitA", 2, 2) + self.defineSgpr("ShadowLimitB", 2, 2) + + if self.staggerU: + self.defineSgpr("StaggerUIter", 1) # stagger loop iterations, used for various iter counts in the code + self.defineSgpr("WrapUA", 2) # Bytes to add to SrdA to reset address from N-1 iter to AddressA + self.defineSgpr("WrapUB", 2) # Bytes to add to SrdB to reset address from N-1 iter to AddressB + + self.defineSgpr("GlobalReadIncsA", self.numSgprGlobalReadIncsA) + self.defineSgpr("GlobalReadIncsB", self.numSgprGlobalReadIncsB) + + if kernel["LocalWriteUseSgprA"]: + self.defineSgpr("LocalWriteAddrA", 1) + if kernel["LocalWriteUseSgprB"]: + self.defineSgpr("LocalWriteAddrB", 1) + + if kernel["_UseSgprForGRO"]: + needFirstSgprOffset = kernel["DirectToLdsA"] and kernel["UseInstOffsetForGRO"] + numberOfSgpr = self.numGlobalReadOffsetsA if needFirstSgprOffset else (self.numGlobalReadOffsetsA-1) + self.defineSgpr("ScalarGlobalReadOffsetA", numberOfSgpr) + + needFirstSgprOffset = kernel["DirectToLdsB"] and kernel["UseInstOffsetForGRO"] + numberOfSgpr = self.numGlobalReadOffsetsB if needFirstSgprOffset else (self.numGlobalReadOffsetsB-1) + self.defineSgpr("ScalarGlobalReadOffsetB", numberOfSgpr) + + # debug flag to allocate dummy / unused sgpr + # useful when comparing code that adds new kernel arguments to see what + # was actually changed + numDummySgpr= 0 + for i in range(numDummySgpr): + self.defineSgpr("DummySgpr%d"%i, 1) + + if self.sgprPool.size() >= self.maxSgprs: + print ("warning: Number of defined SGPRS (%d) overflowed max SGPRS (%d)." \ + % (self.sgprPool.size(), self.maxSgprs)) + + ############################################################################## + # Init Kernel + ############################################################################## + def initKernel(self, kernel, tPA, tPB ): + super(KernelWriterAssembly, self).initKernel(kernel, tPA, tPB) + + self.kernel = kernel + + # init these here in case some kernel pieces are disabled for performance exploration: + tPA["localReadOffset"] = 0 + tPB["localReadOffset"] = 0 + + self.vgprs = VgprStruct() + self.sgprs = collections.OrderedDict() + + self.labels = LabelManager() + + self.LdsOOB = 0xF00000 + + #--- + # Internal optimization and debug controls. + # These have a default which is almost always faster so don't make a full-blown YAML parm + # But have a control here so we can disable for debugging and also easily tell + # which parts of the code were changed to support the new mode. + self.globalReadIncsUseVgpr = False if kernel["BufferLoad"] else True + + # If True, GRO are expressed as offsets from the beginning of the macro-tile, and the SRD + # is set to the beginning of the macro-tile. + # If False, GRO are expressed as offsets from the beginning of the lowest 2 dimensions + # in the tensor. + # True can allow Buffer-Based logic to have significantly higher range and handle larger tensors + # groOffsetInMacroTile doesn't work with pointer-shift because it sets the SRD to point to the + # start of the macro-tile - if we overhang by small number of elements (0, shift the start of the SRD left by specified #elements (not bytes) + # Gives pointer shift some room to move left, even into the previous macro-tile + # This slightly reduces the range of the GRO since they have to include the offset + # Pointer shift still cannot be used with very small matrices < GRVW + self.srdShiftLeft = {} + self.srdShiftLeft["A"] = kernel["GlobalLoadVectorWidthA"] + self.srdShiftLeft["B"] = kernel["GlobalLoadVectorWidthB"] + + self.checkGRO = False + # checkGRO requires useSgprForGRO=0 so that code allocates and uses + # the VGPRs that are used for the GRO offset checking + assert not (kernel["_UseSgprForGRO"] and self.checkGRO) + + # Debug mode to explore combining VGPRs. + # Saves VGPRs but doesn't generate correct answer + self.combineLocalAddresses = 0 + + self.unifiedVgprRegs = False + if globalParameters["ArchCaps"][self.version]["ArchAccUnifiedRegs"]: + self.unifiedVgprRegs = True + + if kernel["EnableMatrixInstruction"]: + if (kernel["ProblemType"]["DataType"].MIOutputTypeNameAbbrev() == 'f64') and (not self.asmCaps["HasMFMA_f64"]): + raise RuntimeError("FP64 MatrixInstruction not supported for {0}".format(self.version)) + elif not self.asmCaps["HasMFMA"]: + raise RuntimeError("MatrixInstruction not supported for {0}".format(self.version)) + + if kernel["MFMA_BF16_1K"] and not self.asmCaps["HasMFMA_bf16_1k"]: + raise RuntimeError("BF16_1k MatrixInstruction not supported for {0}".format(self.version)) + + if kernel["ProblemType"]["Fp16AltImpl"] and not self.asmCaps["HasMFMA_bf16_1k"]: + raise RuntimeError("Fp16AltImpl not supported for {0}".format(self.version)) + + if not globalParameters["AsmCaps"][self.version]["HasDirectToLds"]: + kernel["DirectToLdsA"] = False + kernel["DirectToLdsB"] = False + kernel["LocalWriteUseSgprA"] = False # Requires DirectToLdsA + kernel["LocalWriteUseSgprB"] = False # Requires DirectToLdsB + + # The inst HasAtomicAdd is using is not compatible with int32. + self.useAtomicAdd = (self.asmCaps["HasAtomicAdd"] and kernel["ProblemType"]["ComputeDataType"].isSingle()) and \ + (kernel["_GlobalAccumulation"] == 'SingleBuffer') + + if self.asmCaps["v_fma_mix_f32"]: + self.mixinst = VFmaMixF32 + elif self.asmCaps["v_mad_mix_f32"]: + self.mixinst = VMadMixF32 + else: + self.mixinst = "NOT_SUPPORTED" + + self.overflowedResources = 0 # if true, comment out whole kernel + + self.kernelName = self.getKernelName(kernel) + self.inTailLoop = False + self.serializedStore = False + self.codeAccVgprRead = None + self.codeMulAlpha = None + + # registers per element + self.bpr = 4 # all registers are 32bit + + # default setup + # AB=DataType / Cexternal=DestDataType / Cinternal=Accumulation (MAC or MFMA) + self.bpeAB = int(self.bpr * kernel["ProblemType"]["DataType"].numRegisters()) + + # Cexternal = the "current" kernel output type, + # - default: the "current" kernel is a non-GSU-kernel, + # Cexternal (= DestDataType) and is the final gemm result + # + # - For GSU: the "current" kernel is a GSU-kernel, + # this kernel returns a temp buffer with same type as Cinternal. + # Later, another kernel will accumulate this buffer + # and convert the final result to Cexternal (= DestDataType) as the gemm result + self.bpeCexternal = int(self.bpr * kernel["ProblemType"]["DestDataType"].numRegisters()) + + # already covers: dgemm, cgemm, zgemm, sgemm + # : hgemm + !HPA ([H/H/H] compute = internal = f16) + # : hgemm + HPA ([H/H/S] or [H/S/S] compute = internal = f32) + # : bfgemm + HPA ([B/B/S] or [H/S/S] compute = internal = f32) + # : int8x4-gemm (internal = i32) + self.bpeCinternal = int(self.bpr * kernel["ProblemType"]["ComputeDataType"].numRegisters()) + + #jgolds Need to check device for support + # HPA not allowed in dgemm, cgemm, zgemm, sgemm + if kernel["ProblemType"]["HighPrecisionAccumulate"] and \ + not (kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16() or \ + kernel["ProblemType"]["DataType"].isInt8x4() or kernel["ProblemType"]["DataType"].isInt8()): + print("HighPrecisionAccumulate only valid when DataType is half, bf16, Int8x4, Int8. Forcing HPA to False") + kernel["ProblemType"]["HighPrecisionAccumulate"] = False + + self.bpeCexternal = self.bpeCinternal if kernel["_GlobalAccumulation"] else self.bpeCexternal + + assert self.bpeAB == tPA["bpe"] + assert self.bpeAB == tPB["bpe"] + # registers per global address + self.rpga = 2 # 64-bit + # registers per local address + self.rpla = 1 # 32-bit + # registers per global 32-bit offset (some intructions only support 32-bit offset) + self.rpgo = 1 # 32-bit + + #######################################L + # Available Memory Instructions + ######################################## + + # name, numAddresses, numOffsets, offsetMultiplier, blockWidth, formatting): + ######################################## + # Local Read + _ds_load_b128 = MemoryInstruction(DSLoadB128, 1, 1, 4, 4) + _ds_load2_b64 = MemoryInstruction(DSLoad2B64, 1, 2, 2, 2) + _ds_load_b64 = MemoryInstruction(DSLoadB64, 1, 1, 2, 2) + _ds_load2_b32 = MemoryInstruction(DSLoad2B32, 1, 2, 1, 1) + _ds_load_b32 = MemoryInstruction(DSLoadB32, 1, 1, 1, 1) + _ds_load_u16 = MemoryInstruction(DSLoadU16, 1, 1, 1, 0.5) + _ds_load_u8 = MemoryInstruction(DSLoadU8, 1, 1, 1, 0.25) + ######################################## + # Local Write + _ds_store_b128 = MemoryInstruction(DSStoreB128, 1, 1, 4, 4) + _ds_store2_b64 = MemoryInstruction(DSStore2B64, 1, 2, 2, 2) + _ds_store_b64 = MemoryInstruction(DSStoreB64, 1, 1, 2, 2) + _ds_store2_b32 = MemoryInstruction(DSStore2B32, 1, 2, 1, 1) + _ds_store_b32 = MemoryInstruction(DSStoreB32, 1, 1, 1, 1) + _ds_store_b16 = MemoryInstruction(DSStoreB16, 1, 1, 1, 0.5) + _ds_store_b8 = MemoryInstruction(DSStoreB8, 1, 1, 1, 0.25) + ######################################## + # Global Read + _flat_load_b128 = MemoryInstruction(FlatLoadB128, 1, 0, 0, 4) + _flat_load_b64 = MemoryInstruction(FlatLoadB64, 1, 0, 0, 2) + _flat_load_b32 = MemoryInstruction(FlatLoadB32, 1, 0, 0, 1) + + _buffer_load_b128 = MemoryInstruction(BufferLoadB128, 1, 0, 0, 4) + _buffer_load_b64 = MemoryInstruction(BufferLoadB64, 1, 0, 0, 2) + _buffer_load_b32 = MemoryInstruction(BufferLoadB32, 1, 0, 0, 1) + # generate half directly w/o using the format string to handle hi/lo correctly + _buffer_load_d16_b16 = MemoryInstruction(BufferLoadD16B16, 1, 0, 0, 0.5) + # generate byte directly w/o using the format string to handle hi/lo correctly + _buffer_load_d16_u8 = MemoryInstruction(BufferLoadD16U8, 1, 0, 0, 0.25) + + self.buff_load_inst_offset_max = 4096 + + ######################################## + # Global Write + _flat_store_b128 = MemoryInstruction(FlatStoreB128, 1, 0, 0, 4) + _flat_store_b64 = MemoryInstruction(FlatStoreB64, 1, 0, 0, 2) + _flat_store_b32 = MemoryInstruction(FlatStoreB32, 1, 0, 0, 1) + + ######################################## + # Available Memory Instructions per Architecture + # gfx701 "Hawaii" + # gfx801 "Carrizo" + # gfx802 "Tonga" + # gfx803 "Fiji" + # gfx900 + ######################################## + if (kernel["BufferLoad"]): + chosen_load_b128 = _buffer_load_b128 + chosen_load_b64 = _buffer_load_b64 + chosen_load_b32 = _buffer_load_b32 + chosen_load_b16 = _buffer_load_d16_b16 + chosen_load_b8 = _buffer_load_d16_u8 + else: + chosen_load_b128 = _flat_load_b128 + chosen_load_b64 = _flat_load_b64 + chosen_load_b32 = _flat_load_b32 + chosen_load_b16 = _flat_load_b32 # not supported + chosen_load_b8 = _flat_load_b32 # not supported + + chosen_store_b128 = _flat_store_b128 + chosen_store_b64 = _flat_store_b64 + chosen_store_b32 = _flat_store_b32 + + self.memoryInstructions = { + "GlobalRead" : [ chosen_load_b128, chosen_load_b64, chosen_load_b32, + chosen_load_b16, chosen_load_b8 ], + "GlobalWrite": [ chosen_store_b128, chosen_store_b64, chosen_store_b32 ], + "LocalRead" : [ _ds_load_b128, _ds_load2_b64, _ds_load_b64, + _ds_load2_b32, _ds_load_b32, _ds_load_u16, _ds_load_u8 ], + "LocalWrite" : [ _ds_store_b128, _ds_store2_b64, _ds_store_b64, _ds_store2_b32, + _ds_store_b32, _ds_store_b16, _ds_store_b8 ] + } + + #################################### + # choose memory instructions + #################################### + + ######################################## + + instructions = self.memoryInstructions + self.initGlobalReadMemoryInstruction(instructions, tPA, self.bpr) + self.initGlobalReadMemoryInstruction(instructions, tPB, self.bpr) + self.initLocalWriteMemoryInstruction(instructions, kernel, tPA, self.bpr) + self.initLocalWriteMemoryInstruction(instructions, kernel, tPB, self.bpr) + self.initLocalReadMemoryInstruction(instructions, kernel, tPA, self.bpr, self.lrvwA) + self.initLocalReadMemoryInstruction(instructions, kernel, tPB, self.bpr, self.lrvwB) + + # global reads per instruction + tPA["nrcvpi"] = int((tPA["globalReadInstruction"].totalWidth*self.bpr)/tPA["bpe"]) + tPB["nrcvpi"] = int((tPB["globalReadInstruction"].totalWidth*self.bpr)/tPB["bpe"]) + tPA["nwcvpi"] = int((tPA["localWriteInstruction"].totalWidth*self.bpr)/tPA["bpe"]) + tPB["nwcvpi"] = int((tPB["localWriteInstruction"].totalWidth*self.bpr)/tPB["bpe"]) + #################################### + # VGPR Allocation + #################################### + + #################################### + # num vgprs: valu + #jgolds bpeCinternal because we are allocating accumulation registers here + self.numVgprValuC = (kernel["ThreadTile0"]*kernel["ThreadTile1"]*self.bpeCinternal)//self.bpr + + PLR = kernel["PrefetchLocalRead"] if kernel["PrefetchLocalRead"] < kernel["LoopIters"] else kernel["LoopIters"] - 1 + valuBlocks = (1+PLR) * kernel["InnerUnroll"] + # double the number of VgprValu if self.vgprValuDouble is true + if self.vgprValuDouble: + valuBlocks *= 2 + if kernel["EnableMatrixInstruction"]: + self.numVgprValuAPerBlock = kernel["MIWaveTileA"] * kernel["MIInputPerThread"] * tPA["bpe"] // self.bpr + self.numVgprValuBPerBlock = kernel["MIWaveTileB"] * kernel["MIInputPerThread"] * tPB["bpe"] // self.bpr + else: + printExit("TensileLite does not support non MFMA.") + + # change numVgprValuAPerBlock to 0 for A if DirectToVgpr is enabled + if kernel["DirectToVgprA"]: + self.numVgprValuAPerBlock = 0 + self.numVgprValuA = self.numVgprValuAPerBlock * valuBlocks + # change numVgprValuBPerBlock to 0 for B if DirectToVgpr is enabled + if kernel["DirectToVgprB"]: + self.numVgprValuBPerBlock = 0 + self.numVgprValuB = self.numVgprValuBPerBlock * valuBlocks + + #################################### + # num vgprs: global -> local elements + self.numVgprG2LA = 0 + if not kernel["DirectToLdsA"] or self.do["KeepDirectToLdsAlloc"]: + self.numVgprG2LA = roundUp((kernel["NumLoadsCoalescedA"] * kernel["NumLoadsPerpendicularA"] * \ + kernel["GlobalLoadVectorWidthA"] * tPA["bpe"]) / (float)(self.bpr)) + # using _ds_store_b8: need one more vgpr space to do lshr + if tPA["localWriteInstruction"].blockWidth == 0.25: + self.numVgprG2LA = self.numVgprG2LA * 2 + # double numVgprG2LA if DirectToVgpr is enabled + if kernel["DirectToVgprA"]: + self.numVgprG2LA = self.numVgprG2LA * 2 + + self.numVgprG2LB = 0 + if not kernel["DirectToLdsB"] or self.do["KeepDirectToLdsAlloc"]: + self.numVgprG2LB = roundUp((kernel["NumLoadsCoalescedB"] * kernel["NumLoadsPerpendicularB"] * \ + kernel["GlobalLoadVectorWidthB"] * tPB["bpe"]) / (float)(self.bpr)) + # using _ds_store_b8: need one more vgpr space to do lshr + if tPB["localWriteInstruction"].blockWidth == 0.25: + self.numVgprG2LB = self.numVgprG2LB * 2 + # double numVgprG2LB if DirectToVgpr is enabled + if kernel["DirectToVgprB"]: + self.numVgprG2LB = self.numVgprG2LB * 2 + + #################################### + # num vgprs: local read addresses + self.numVgprLocalReadAddressesA = 1 * self.rpla + self.numVgprLocalReadAddressesB = 1 * self.rpla + # do not allocate local read address register if DirectToVgpr is enabled + if kernel["DirectToVgprA"]: + self.numVgprLocalReadAddressesA = 0 + if kernel["DirectToVgprB"]: + self.numVgprLocalReadAddressesB = 0 + + #################################### + # num vgprs: local write addresses + #numLocalWritesA = kernel["NumLoadsCoalescedA"] \ + # * nlp * self.numWriteVectorComponentsA + #numLocalWriteInstructionsA = numLocalWritesA \ + # / tPA["localWriteInstruction"][self.instructionIdxNumOffsets] + self.numVgprLocalWriteAddressesA = 0 if kernel["LocalWriteUseSgprA"] else 1 * self.rpla + + #numLocalWritesB = kernel["NumLoadsCoalescedB"] \ + # * nlp * self.numWriteVectorComponentsB + #numLocalWriteInstructionsB = numLocalWritesB \ + # / tPB["localWriteInstruction"][self.instructionIdxNumOffsets] + self.numVgprLocalWriteAddressesB = 0 if kernel["LocalWriteUseSgprB"] else 1 * self.rpla + + # do not allocate local write address register if DirectToVgpr is enabled + if kernel["DirectToVgprA"]: + self.numVgprLocalWriteAddressesA = 0 + if kernel["DirectToVgprB"]: + self.numVgprLocalWriteAddressesB = 0 + + #################################### + # num vgprs: global read addresses + numGlobalReadsA = kernel["NumLoadsCoalescedA"] \ + * kernel["NumLoadsPerpendicularA"] * kernel["GlobalLoadVectorWidthA"] + numGlobalReadInstructionsA = (numGlobalReadsA * tPA["bpe"])//\ + (tPA["globalReadInstruction"].blockWidth * 4) + + if kernel["BufferLoad"]: + self.numGlobalReadOffsetsA = roundUp(numGlobalReadInstructionsA * self.rpgo) + else: + numVgprGlobalReadAddressesA = numGlobalReadInstructionsA * self.rpga + + numGlobalReadsB = kernel["NumLoadsCoalescedB"] \ + * kernel["NumLoadsPerpendicularB"] * kernel["GlobalLoadVectorWidthB"] + numGlobalReadInstructionsB = (numGlobalReadsB * tPB["bpe"])// \ + (tPB["globalReadInstruction"].blockWidth * 4) + if kernel["BufferLoad"]: + self.numGlobalReadOffsetsB = roundUp(numGlobalReadInstructionsB * self.rpgo) + else: + numVgprGlobalReadAddressesB = numGlobalReadInstructionsB * self.rpga + if self.globalReadIncsUseVgpr: + numVgprGlobalReadIncsA = kernel["ProblemType"]["NumIndicesSummation"] \ + * self.rpga + numVgprGlobalReadIncsB = kernel["ProblemType"]["NumIndicesSummation"] \ + * self.rpga + else: + numVgprGlobalReadIncsA = 0 + numVgprGlobalReadIncsB = 0 + + numVgprAddressDbg = self.rpga if globalParameters["DebugKernel"] else 0 + + #################################### + # num vgprs: c write address + # 1 address where to write first value + # 1 tmp address where to write current value + + #################################### + # VGPR Assignment + #################################### + vgprIdx = 0 + self.totalAgprs = 0 + self.startVgprValuC = vgprIdx; vgprIdx += self.numVgprValuC + + if kernel["EnableMatrixInstruction"]: + # MI kernels can overlap C-tile w/ AB-tile up until writeback. Illustrated below: + # |<-------------- valuC -------------->| + # |------------|-----------|xx|---------| + # lastValuAB ^ ^ ^ ^ + # lastVgprForReads ^ ^ ^ + # startVgprReuse ^ ^ + # lastValuC ^ + # TODO a bit tricky. Better to manage all GPRs solely through RegisterPool + self.serializedStore = True + + ######################################## + # AGPR Allocation + ######################################## + if not kernel["MIArchVgpr"]: + self.totalAgprs = self.numVgprValuC + vgprIdx = 0 + self.numVgprValuC = 0 + + self.startaccValuC0 = None + self.startaccValuC1 = None + + # TODO: alignment hack, figure out a better solution + vgprIdx = ((vgprIdx+1)//2)*2 + # Avoid bank conflict between VgprA and VgprC + if (self.version[0] == 10) and ((vgprIdx % 4) == (self.startVgprValuC % 4)): + vgprIdx += 1 + self.startVgprValuA = vgprIdx; vgprIdx += self.numVgprValuA + self.startVgprG2LA = None + if not kernel["DirectToLdsA"] or self.do["KeepDirectToLdsAlloc"]: + # if PGR = True, PAP could be possibly enabled, we move G2LA later to prevent it from being reclaimed + # otherwise, put G2L here since it can overlap valu + if not kernel["PrefetchGlobalRead"] and not kernel.enabledSplitLDS: # g2l can overlap valu + self.startVgprG2LA = self.startVgprValuA + vgprIdx = self.startVgprValuA \ + + max(self.numVgprValuAPerBlock*valuBlocks, self.numVgprG2LA) + + # TODO: alignment hack, figure out a better solution + vgprIdx = ((vgprIdx+1)//2)*2 + self.startVgprValuB = vgprIdx; vgprIdx += self.numVgprValuB + self.startVgprG2LB = None + if not kernel["DirectToLdsB"] or self.do["KeepDirectToLdsAlloc"]: + # if PGR = True, PAP could be possibly enabled, we move G2LB later to prevent it from being reclaimed + # otherwise, put G2L here since it can overlap valu + if not kernel["PrefetchGlobalRead"] and not kernel.enabledSplitLDS: # g2l can overlap valu + self.startVgprG2LB = self.startVgprValuB + vgprIdx = self.startVgprValuB \ + + max(self.numVgprValuBPerBlock*valuBlocks, self.numVgprG2LB) + + # Registers allocated above this point can be used as temps during setup + # Registers above here are reserved in initC, near the end of the setup + # code + self.lastValuAB = vgprIdx + #---------------------------------- + # Point at last VGPR that can be reclaimed for use in the summation loop + # If more VGPRs are added here be aware of the register reclaim code in + # endSummation - registers that should be preserved after lastVgprForReads + # + # For PAP: decide the reclaim case + # if we're not doing PAP, then the GlobalRead, LocalWrite, LocalRead, VgprG2L can be reclaimed + # (and we'll extend the "lastVgprForReads" value later) + # otherwise if we have PAP, they can't be reclaimed so we simply use the current vgprIdx + self.lastVgprForReads = vgprIdx + #---------------------------------- + + if not kernel["LocalWriteUseSgprA"]: + if self.combineLocalAddresses: + self.startVgprLocalWriteAddressesA = self.startVgprLocalReadAddressesA + else: + self.startVgprLocalWriteAddressesA = vgprIdx + vgprIdx += self.numVgprLocalWriteAddressesA + + if not kernel["LocalWriteUseSgprB"]: + if self.combineLocalAddresses: + self.startVgprLocalWriteAddressesB = self.startVgprLocalReadAddressesA + else: + self.startVgprLocalWriteAddressesB = vgprIdx + vgprIdx += self.numVgprLocalWriteAddressesB + + # BufferLoad: + # Uses a resource descriptor (SRD) which is stored in 4 SGPRs and thus shared by all work-items. + # Each work-item also uses a unique 32-bit offset into vgprGlobalReadOffset. These offsets are set when + # the tile is initialized and stay constant through the execution of the kernel. + # The base address in the SRD is updated when the algorithm moves to a new tile + # BufferLoad disables the gptGlobalReadAddr used in flat addressing. + if kernel["BufferLoad"]: + self.startVgprGlobalReadOffsetA = vgprIdx + vgprIdx += 1 if kernel["_UseSgprForGRO"] else self.numGlobalReadOffsetsA + self.startVgprGlobalReadOffsetB = vgprIdx + vgprIdx += 1 if kernel["_UseSgprForGRO"] else self.numGlobalReadOffsetsB + else: + # TODO: alignment hack, figure out a better solution + vgprIdx = ((vgprIdx+1)//2)*2 + self.startVgprGlobalReadAddressesA = vgprIdx + vgprIdx += numVgprGlobalReadAddressesA + self.startVgprGlobalReadAddressesB = vgprIdx + vgprIdx += numVgprGlobalReadAddressesB + + self.startVgprGlobalReadIncsA = vgprIdx + vgprIdx += numVgprGlobalReadIncsA + self.startVgprGlobalReadIncsB = vgprIdx + vgprIdx += numVgprGlobalReadIncsB + #----------- + + if self.startVgprG2LA is None: + # TODO: alignment hack, figure out a better solution + vgprIdx = ((vgprIdx+1)//2)*2 + self.startVgprG2LA = vgprIdx; vgprIdx += self.numVgprG2LA + + if self.startVgprG2LB is None: + # TODO: alignment hack, figure out a better solution + vgprIdx = ((vgprIdx+1)//2)*2 + self.startVgprG2LB = vgprIdx; vgprIdx += self.numVgprG2LB + + # GlobalRead, LocalWrite, LocalRead, G2L can be reclaimed, extend the "lastVgprForReads" value + self.lastVgprForReads = vgprIdx + #----------- + + self.startVgprLocalReadAddressesA = vgprIdx + vgprIdx += self.numVgprLocalReadAddressesA + if self.combineLocalAddresses: + self.startVgprLocalReadAddressesB = self.startVgprLocalReadAddressesA + else: + self.startVgprLocalReadAddressesB = vgprIdx + vgprIdx += self.numVgprLocalReadAddressesB + + self.startVgprAddressDbg = vgprIdx + vgprIdx += numVgprAddressDbg + + # for zgemm + (SCIU or MIAV) case, allocate 4 vgpr for alpha calculation (cannot use tmp vgpr in unroll loop or write batch) + if kernel["ProblemType"]["DataType"].isDoubleComplex() and kernel["MIArchVgpr"]: + # need proper alignment + vgprIdx = ((vgprIdx+2 - 1)//2)*2 + self.startVgprAlphaTmp = vgprIdx + vgprIdx += 4 + + # TODO: Serial is always the first/last register in the pool so the store + # code doesn't have to deal with fragmentation + self.startVgprSerial = vgprIdx + vgprIdx += 1 # for vgpr serial id + + # tmp vgprs + #minVgprTmp += 4 + #if globalParameters["DebugKernel"]: + # minVgprTmp += 2 + #vgprIdx += minVgprTmp + #print2("%3u vgprs <- %s" % (vgprIdx, self.kernelName) ) + self.startVgprReuse = vgprIdx # for register reuse; + + self.totalVgprs = max(vgprIdx, self.numVgprValuC) + if self.totalVgprs < kernel["MinVgprNumber"] or self.totalVgprs > kernel["MaxVgprNumber"]: + raise RuntimeError("Generating asm kernel error: total vgpr: %u not in [%u, %u].\n" % (self.totalVgprs, kernel["MinVgprNumber"], kernel["MaxVgprNumber"])) + + ######################################## + # SGPR Allocation + ######################################## + + #################################### + # num sgprs: initial kernel state + self.sgprPool = RegisterPool(0, 's', defaultPreventOverflow=True, printRP=0) + numSgprAddressD = self.rpga # til end + numSgprAddressC = self.rpga # til end + numSgprAddressA = self.rpga # til read offsets + numSgprAddressB = self.rpga # til read offsets + # would not less than 1 reg, + # since even if ComputeType = H, we still pass the arg as a 32-bit (concate two 16-bit) + numSgprAlpha = max(1,int(self.bpeCinternal/4)) + numSgprBeta = max(1,int(self.bpeCinternal/4)) if kernel["ProblemType"]["UseBeta"] else 0 + self.numSgprStridesD = kernel["ProblemType"]["NumIndicesC"] + self.numSgprStridesC = kernel["ProblemType"]["NumIndicesC"] + self.numSgprStridesA = len(kernel["ProblemType"]["IndexAssignmentsA"]) + self.numSgprStridesB = len(kernel["ProblemType"]["IndexAssignmentsB"]) + if not kernel["ProblemType"]["UseInitialStridesCD"]: + self.numSgprStridesD -= 1 + self.numSgprStridesC -= 1 + if not kernel["ProblemType"]["UseInitialStridesAB"]: + self.numSgprStridesA -= 1 + self.numSgprStridesB -= 1 + self.numSgprSizesSum = kernel["ProblemType"]["NumIndicesSummation"] + self.numSgprSizesFree = kernel["ProblemType"]["NumIndicesC"] + self.numSgprOffsetD = 1 + self.numSgprOffsetC = 1 + self.numSgprOffsetA = 1 + self.numSgprOffsetB = 1 + self.numActivationTypeArgSize = 0 # Will change to 1 if activationType == All + self.numActivationArgSize = max(1, int(kernel["ProblemType"]["DestDataType"].numRegisters())) + self.numactivationArgTotalSize = self.numActivationArgSize * kernel["ProblemType"]["ActivationType"].getAdditionalArgNum() + self.numSgprAddressDbg = self.rpga if globalParameters["DebugKernel"] else 0 + + #################################### + # num sgprs: global read increments + if self.globalReadIncsUseVgpr: + self.numSgprGlobalReadIncsA = 0 + self.numSgprGlobalReadIncsB = 0 + else: + self.numSgprGlobalReadIncsA = kernel["ProblemType"]["NumIndicesSummation"] * self.rpgo + self.numSgprGlobalReadIncsB = kernel["ProblemType"]["NumIndicesSummation"] * self.rpgo + + ######################################## + # SGPR Assignment according to AMDGPU-ABI + ######################################## + self.defineSgpr("KernArgAddress", self.rpga) + assert(self.sgprs["KernArgAddress"] == 0) # kernarg is passed to kernel as SGPR0 + + if kernel["WorkGroupMapping"]>=0 : + self.defineSgpr("WorkGroup0", 1) + self.defineSgpr("WorkGroup1", 1) + else: + self.defineSgpr("WorkGroup1", 1) + self.defineSgpr("WorkGroup0", 1) + + wg=2 + + for idx in kernel["ProblemType"]["IndicesBatch"]: + if not isPackedIndex(kernel,idx): + self.defineSgpr("WorkGroup%u"%wg, 1) + wg+=1 + + # SGPR above are user SGPR which are set by GPU hardware when the kernel is launched + self.firstInitSgpr = self.sgprPool.size() + + # To avoid corrupting tmp sgprs that may be used around the assert, + # reserve some sgprs to save/restore the execmask + if self.db["EnableAsserts"]: + self.defineSgpr("SaveExecMask", 2, 2) + self.asmAssert = Assert(self.laneSGPRCount, kernel["WavefrontSize"], self.db["EnableAsserts"]) + + self.defineSgpr("GSUSumIdx", 2 if kernel["GlobalSplitU"] > 1 else 0) + + # for packed batches without stride restrictions need to do something different here + assert sorted(kernel["PackedC0IdxChars"]+kernel["PackedC1IdxChars"]) == \ + sorted(set(kernel["PackedC0IdxChars"]+kernel["PackedC1IdxChars"])) + for idxChar in kernel["PackedC0IdxChars"][:-1]: + if kernel["MagicDivAlg"]==2: + self.defineSgpr("MagicAbitSize%s"%idxChar, 1) + for idxChar in kernel["PackedC1IdxChars"][:-1]: + if kernel["MagicDivAlg"]==2: + self.defineSgpr("MagicAbitSize%s"%idxChar, 1) + + # product of all packed dims in the 0 or 1 dimensions: + if len(kernel["PackedC0IndicesX"]) > 1: + self.defineSgpr("PackedSize0", 1) + if len(kernel["PackedC1IndicesX"]) > 1: + self.defineSgpr("PackedSize1", 1) + + # contractions with multiple summations will use multiple LoopCounters, if PSD=0 + for i in range(kernel["ProblemType"]["NumIndicesSummation"]): + self.defineSgpr(self.loopCounterName(kernel,i), 1) + + self.defineSgpr("OrigLoopCounter", 1) + + if globalParameters["DebugKernel"]: + self.defineSgpr("AddressDbg", self.numSgprAddressDbg) + self.defineSgpr("DebugKernelItems", 1) + + if kernel["BufferLoad"]: + # resource descriptor (SRD) A and B, must be aligned on 4-SGPR boundary + self.defineSgpr("SrdA", 4, 4) + self.defineSgpr("SrdB", 4, 4) + if kernel["BufferStore"]: + self.defineSgpr("SrdD", 4, 4) + self.defineSgpr("SrdC", 4, 4) + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + self.defineSgpr("SrdBias", 4, 4) + + ################################### + # Get kernel argument start here + self.defineSgpr("Tensor2dSizeA", 2,4) + # fill empty Sgpr slot caused by Sgpr alignment, + # because we need following defineSgpr use continuous sgpr + SgprSlot = [] + currentSize = self.sgprPool.size() + while (1): + tempSgpr = self.sgprPool.checkOut(1,"fill empty slot temporarily",preventOverflow=0) + if tempSgpr >= currentSize: + self.sgprPool.checkIn(tempSgpr) + break + SgprSlot.append(tempSgpr) + self.defineSgpr("Tensor2dSizeB", 2, 2) + self.argAddressOffset = 6 * 4 # 8 bytes C, A, B + + self.defineSgpr("AddressD", numSgprAddressD) + self.defineSgpr("AddressC", numSgprAddressC) + self.defineSgpr("AddressA", numSgprAddressA) + self.defineSgpr("AddressB", numSgprAddressB) + self.defineSgpr("Alpha", numSgprAlpha, numSgprAlpha) + if kernel["ProblemType"]["UseBeta"]: + self.defineSgpr("Beta", numSgprBeta, numSgprBeta) + numSgprAddressBias = 0 + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + numSgprAddressBias = numSgprAddressA + self.defineSgpr("AddressBias", numSgprAddressBias) # Same as mat A + if ((kernel["ProblemType"]["ActivationType"] != 'none') and (kernel["GlobalSplitU"] == 1) \ + and kernel["ActivationFused"]): + for name in kernel["ProblemType"]["ActivationType"].getAdditionalArgStringList(): + self.defineSgpr(name, self.numActivationArgSize, self.numActivationArgSize) + if kernel["ProblemType"]["ActivationType"] == 'all': + self.numActivationTypeArgSize = 1 + self.defineSgpr("ActivationType", self.numActivationTypeArgSize) + self.defineSgpr("StridesD", self.numSgprStridesD) + self.defineSgpr("StridesC", self.numSgprStridesC) + self.defineSgpr("StridesA", self.numSgprStridesA) + self.defineSgpr("StridesB", self.numSgprStridesB) + self.defineSgpr("SizesFree", self.numSgprSizesFree) + self.defineSgpr("SizesSum", self.numSgprSizesSum) + + # for packed batches without stride restrictions need to do something different here + assert sorted(kernel["PackedC0IdxChars"]+kernel["PackedC1IdxChars"]) == \ + sorted(set(kernel["PackedC0IdxChars"]+kernel["PackedC1IdxChars"])) + for idxChar in kernel["PackedC0IdxChars"][:-1]: + self.defineSgpr("MagicNumberSize%s"%idxChar, 1) + self.defineSgpr("MagicShiftSize%s"%idxChar, 1) + for idxChar in kernel["PackedC1IdxChars"][:-1]: + self.defineSgpr("MagicNumberSize%s"%idxChar, 1) + self.defineSgpr("MagicShiftSize%s"%idxChar, 1) + self.defineSgpr("OrigStaggerUIter", 1) # Original stagger register. Only needed for Persistent + self.defineSgpr("NumWorkGroups0", 1) + self.defineSgpr("NumWorkGroups1", 1) + + #------------------------ + # Registers defined below this point are not available in the post-loop + # Post-loop is after tail loop exits, ie the store code. + # (we reclaim them to use as temps, typically for execmasks) + # Mostly impacts flat kernels and GSU edge since these need SGPR + # for conditionals + self.lastPostLoopSgpr = self.sgprPool.size() + self.defineSgpr("NumFullBlocks", 1) # Magic number to use for div by (NumWorkGroups1 % WGM) + self.defineSgpr("WgmRemainder1", 1) # Magic number to use for div by (NumWorkGroups1 % WGM) + self.defineSgpr("MagicNumberWgmRemainder1", 1) # Magic number to use for div by (NumWorkGroups1 % WGM) + + self.defineSgpr("OffsetD", self.numSgprOffsetD) + self.defineSgpr("OffsetC", self.numSgprOffsetC) + self.defineSgpr("OffsetA", self.numSgprOffsetA) + self.defineSgpr("OffsetB", self.numSgprOffsetB) + + self.numSgprToLoad = 2 + 2 + numSgprAddressD + numSgprAddressC + numSgprAddressA + numSgprAddressB + numSgprAddressBias + numSgprAlpha + \ + (numSgprBeta if kernel["ProblemType"]["UseBeta"] else 0) + self.numSgprStridesD + self.numSgprStridesC + self.numSgprStridesA + \ + self.numSgprStridesB + self.numSgprSizesFree + self.numSgprSizesSum + \ + len(kernel["PackedC0IdxChars"][:-1])*2 + len(kernel["PackedC1IdxChars"][:-1])*2 + \ + 1 + \ + 2 + \ + 3 + \ + self.numSgprOffsetD + self.numSgprOffsetC + self.numSgprOffsetA + self.numSgprOffsetB + if ((kernel["ProblemType"]["ActivationType"] != 'none') and (kernel["GlobalSplitU"] == 1) \ + and kernel["ActivationFused"]): + self.numSgprToLoad += self.numActivationTypeArgSize + self.numactivationArgTotalSize + + self.argOffsetOffset = (self.numSgprToLoad + 2 - (self.numSgprOffsetD + self.numSgprOffsetC + self.numSgprOffsetA + self.numSgprOffsetB)) * 4 + + # Get kernel argument end here + ################################### + + # put unused Sgpr back to SgprPool + while SgprSlot: + tempSgpr = SgprSlot.pop(0) + self.sgprPool.checkIn(tempSgpr) + if not self.staggerU: + self.undefineSgpr("OrigStaggerUIter") # Original stagger register. Only needed for Persistent + + ######################################## + # Register Pools + ######################################## + #print "TotalVgprs", self.totalVgprs + self.vgprPool = RegisterPool(self.totalVgprs, 'v', defaultPreventOverflow=False, + printRP=self.db["PrintRP"]) + #print self.vgprPool.state() + self.savedVgprPool = None + self.savedSgprPool = None + + # C regs are not used during initialization so mark them as available - + # we will claim then just before the start of the unroll loop: + self.vgprPool.add(self.startVgprValuA, \ + self.lastValuAB - self.startVgprValuA, "ValuAB") # Add as available + + self.vgprPool.add(self.startVgprValuC, \ + self.numVgprValuC, "ValuC-Block") # Add as available + #print self.vgprPool.state() + ## accumulator Buffer for storeCinUnroll feature + self.agprPool = RegisterPool(self.totalAgprs, 'a', defaultPreventOverflow=False, printRP=0) + # C regs are not used during initialization so mark them as available - + # we will claim then just before the start of the unroll loop: + numAccvgprs = self.totalAgprs + self.agprPool.add(0, numAccvgprs, "ValuC-Block") + + ######################################## + # reads Per Iteration + ######################################## + if kernel["EnableMatrixInstruction"]: + # setting numReadPerVector to 0 for DirectToVgpr makes performance a little worse. + # so, keep this part unchanged. + #self.numReadPerVectorA = 0 if kernel["DirectToVgprA"] else tPA["bpe"] * self.lrvwA // int(tPA["localReadInstruction"].blockWidth * 4) + #self.numReadPerVectorB = 0 if kernel["DirectToVgprB"] else tPB["bpe"] * self.lrvwB // int(tPB["localReadInstruction"].blockWidth * 4) + self.numReadPerVectorA = tPA["bpe"] * self.lrvwA // int(tPA["localReadInstruction"].blockWidth * 4) + self.numReadPerVectorB = tPB["bpe"] * self.lrvwB // int(tPB["localReadInstruction"].blockWidth * 4) + numA = kernel["InnerUnroll"]*(kernel["MIWaveTile"][0] * self.numReadPerVectorA) // tPA["localReadInstruction"].numOffsets + numB = kernel["InnerUnroll"]*(kernel["MIWaveTile"][1] * self.numReadPerVectorB) // tPB["localReadInstruction"].numOffsets + # wider localread has 2 mode + # 1. using larger IU to coalesced localread, only half of local reads in 1 iteration + # 2. using larger PLR to read more iterations, same number local reads in 1 iteration + if kernel["InnerUnroll"] >= self.numReadsIterCoalescedA: + numA //= self.numReadsIterCoalescedA + if kernel["allowLRVWforTLUandMI"]: + numA //= self.lrvwA + if kernel["InnerUnroll"] >= self.numReadsIterCoalescedB: + numB //= self.numReadsIterCoalescedB + if kernel["allowLRVWforTLUandMI"]: + numB //= self.lrvwB + else: + printExit("TensileLite does not support non MFMA.") + self.numReadsPerIterA = numA + self.numReadsPerIterB = numB + self.localReadDoCntA = 0 + self.localReadDoCntB = 0 + + if kernel["EnableMatrixInstruction"]: + self.miLatency = kernel["MatrixInstM"] // 2 + miIssueLatency = 2 + # give 1 quad-cycle buffer to prevend bubble from sync + miLatencyBuffer = 1 + self.miLatencyLeft = max(self.miLatency - miLatencyBuffer - miIssueLatency,0) + + # shift vectors determined later + + canCheckValueC = (kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16()) and \ + kernel["ProblemType"]["HighPrecisionAccumulate"] + canCheckValueC = canCheckValueC or kernel["ProblemType"]["DataType"].isSingle() + canCheckValueC = canCheckValueC or (kernel["ProblemType"]["DataType"].isInt8() and kernel["ProblemType"]["HighPrecisionAccumulate"]) + assert not self.db["CheckValueC"] or canCheckValueC + + if self.db["InitLds"] : print ("\n***WARNING: InitLds enabled, may impact performance\n") + if self.db["InitSgpr"] : print ("\n***WARNING: InitSgpr enabled, may impact performance\n") + if self.db["InitVgpr"] : print ("\n***WARNING: InitVgpr enabled, may impact performance\n") + if self.db["ConservativeWaitCnt"] : print ("\n***WARNING: ConservativeWaitCnt enabled, may impact performance\n") + if self.do["KeepDirectToLdsAlloc"] : print ("\n***WARNING: KeepDirectToLdsAlloc enabled, may impact performance\n") + if self.db["CheckValue1A"] : print ("\n***WARNING: CheckValue1A enabled, may impact performance\n") + if self.db["CheckValue1B"] : print ("\n***WARNING: CheckValue1B enabled, may impact performance\n") + if self.db["CheckValueC"] : print ("\n***WARNING: CheckValueC enabled, may impact performance\n") + if self.db["ForceExpectedValue"] : print ("\n***WARNING: ForceExpectedValue enabled, may impact functionality\n") + if self.db["ForceVSerial"] : print ("\n***WARNING: ForceVSerial enabled, will impact functionality\n") + if self.db["ForceInputValueA"] : print ("\n***WARNING: ForceInputValueA enabled, may impact functionality\n") + if self.db["ForceInputValueB"] : print ("\n***WARNING: ForceInputValueB enabled, may impact functionality\n") + if self.db["CheckStoreC"] >=0 : print ("\n***WARNING: CheckStoreC enabled, may impact performance\n") + if self.db["ForceEdgeStores"] : print ("\n***WARNING: ForceEdgeStores enabled, may impact performance\n") + if self.db["AssertNoEdge"] : print ("\n***WARNING: AssertNoEdge enabled, may impact functionality and performance\n") + if self.db["PrintRP"] : print ("\n***WARNING: PrintRP enabled, may generate verbose output\n") + + ############################################################################## + def functionSignature(self) -> SignatureBase: + """ + Function Signature + called after rest of code + """ + signatureClass = Component.Signature.find(self) + signature = signatureClass(self) + return signature + + def macroAndSet(self, kernel, tPA, tPB) -> Module: + module = Module("MacroNSet") + module.add(MacroVMagicDiv(kernel["MagicDivAlg"])) + + ######################################## + # VGPR Macros + ######################################## + module.addComment2("VGPR Assignments") + module.addComment0("ValuC range: [%u-%u), %s"%(self.startVgprValuC, self.startVgprValuC+self.numVgprValuC, \ + "serializedStore enabled" if self.serializedStore else "")) + module.add(RegSet("v", "vgprValuC", self.startVgprValuC)) + + module.addComment0("ValuA/B Xn=PLR buffer idx, In=InnerUnroll idx") + # PLR index: from X0 to X (at most) -> VGPRs will be duplicated LoopIters times (at most) + # eg, if LoopIters = 4, there would be at most 4*VGPRs + # PLR = kernel["PrefetchLocalRead"] if kernel["PrefetchLocalRead"] < kernel["LoopIters"] else kernel["LoopIters"] - 1 + PLR = min(kernel["PrefetchLocalRead"], kernel["LoopIters"]-1) + numBi = PLR+1 + # double the number of VgprValue if self.vgprValuDouble is true + if self.vgprValuDouble: + numBi *= 2 + ri = 0 + if self.numVgprValuA > 0: # Do not generate vgprValuA if numVgprValuA is 0 + for bi in range(0,numBi): # buffer indices + for iui in range(0, kernel["InnerUnroll"]): + module.add(RegSet("v", "vgprValuA_X%u_I%u"%(bi,iui), self.startVgprValuA+ri)) + ri += self.numVgprValuAPerBlock + if not kernel["DirectToLdsA"] or self.do["KeepDirectToLdsAlloc"]: + module.add(RegSet("v", "vgprG2LA", self.startVgprG2LA)) + if kernel["DirectToVgprA"]: + # additional definition G2LA0, G2LA1 for swapping register sets + module.add(RegSet("v", "vgprG2LA0", self.startVgprG2LA)) + module.add(RegSet("v", "vgprG2LA1", self.startVgprG2LA + self.numVgprG2LA//2)) + + ri = 0 + if self.numVgprValuB > 0: # Do not generate vgprValuB if numVgprValuB is 0 + for bi in range(0,numBi): # buffer indices + for iui in range(0, kernel["InnerUnroll"]): + module.add(RegSet("v", "vgprValuB_X%u_I%u"%(bi,iui), self.startVgprValuB+ri)) + ri += self.numVgprValuBPerBlock + if not kernel["DirectToLdsB"] or self.do["KeepDirectToLdsAlloc"]: + module.add(RegSet("v", "vgprG2LB", self.startVgprG2LB)) + if kernel["DirectToVgprB"]: + # additional definition G2LB0, G2LB1 for swapping register sets + module.add(RegSet("v", "vgprG2LB0", self.startVgprG2LB)) + module.add(RegSet("v", "vgprG2LB1", self.startVgprG2LB + self.numVgprG2LB//2)) + if not kernel["LocalWriteUseSgprA"] and self.numVgprLocalWriteAddressesA > 0: + module.add(RegSet("v", "vgprLocalWriteAddrA", \ + self.startVgprLocalWriteAddressesA)) + if self.numVgprLocalWriteAddressesA > 1: + module.add(RegSet("v", "vgprLocalWriteAddrOverhangA", \ + self.startVgprLocalWriteAddressesA+1)) + if not kernel["LocalWriteUseSgprB"] and self.numVgprLocalWriteAddressesB > 0: + module.add(RegSet("v", "vgprLocalWriteAddrB", \ + self.startVgprLocalWriteAddressesB)) + if self.numVgprLocalWriteAddressesB > 1: + module.add(RegSet("v", "vgprLocalWriteAddrOverhangB", \ + self.startVgprLocalWriteAddressesB+1)) + if kernel["BufferLoad"]: + module.add(RegSet("v", "vgprGlobalReadOffsetA", \ + self.startVgprGlobalReadOffsetA)) + module.add(RegSet("v", "vgprGlobalReadOffsetB", \ + self.startVgprGlobalReadOffsetB)) + else: + module.add(RegSet("v", "vgprGlobalReadAddrA", \ + self.startVgprGlobalReadAddressesA)) + module.add(RegSet("v", "vgprGlobalReadAddrB", \ + self.startVgprGlobalReadAddressesB)) + + if self.globalReadIncsUseVgpr: + module.add(RegSet("v", "vgprGlobalReadIncsA", \ + self.startVgprGlobalReadIncsA)) + module.add(RegSet("vgprGlobalReadIncsB", "v", \ + self.startVgprGlobalReadIncsB)) + if self.numVgprLocalReadAddressesA > 0: + module.add(RegSet("v", "vgprLocalReadAddrA", \ + self.startVgprLocalReadAddressesA)) + if self.numVgprLocalReadAddressesB > 0: + module.add(RegSet("v", "vgprLocalReadAddrB", \ + self.startVgprLocalReadAddressesB)) + + if kernel["ProblemType"]["DataType"].isDoubleComplex() and kernel["MIArchVgpr"]: + module.add(RegSet("v", "vgprAlphaTmp", \ + self.startVgprAlphaTmp)) + + module.add(RegSet("v", "vgprSerial", self.startVgprSerial)) + + if globalParameters["DebugKernel"]: + module.add(RegSet("v", "vgprAddressDbg", \ + self.startVgprAddressDbg)) + #module.addComment0("Occu: %u waves/simd" % self.numWavesPerSimd ) + # module.addComment0("Num VGPR=%u"%self.vgprPool.size()) + # module.addComment0("Num AccVGPR=%u"%self.agprPool.size()) + + ######################################## + # SGPR Macros + ######################################## + module.addComment2("SGPR Assignments") + + # Emit declarations for all sgprs allocated with defineSgpr + # in the order they were declared + for skey in self.sgprs: + module.add(RegSet("s", "sgpr"+skey, self.sgprs[skey])) + # module.addComment0("max SGPR=%u"%self.sgprPool.size()) + + module.addSpaceLine() + module.addComment0("Size Assignments") + problemType = kernel["ProblemType"] + for idx in range(max(problemType["IndexAssignmentsA"] + problemType["IndexAssignmentsB"])+1): + idxChar= globalParameters["IndexChars"][idx] + if idx in problemType["IndicesFree"] or idx in problemType["IndicesBatch"]: + idxType="Free" + elif idx in problemType["IndicesSummation"]: + idxType="Sum" + idx = idx - problemType["NumIndicesC"] + else: + raise ValueError("unexpected index type in size assignments") + + module.add(RegSet("s", "sgprSize%s"%(idxChar), \ + "sgprSizes%s"%idxType, idx)) + + module.addSpaceLine() + module.addComment0("Stride Assignments") + for tc in ('D','C'): + for idx in range(0, problemType["NumIndicesC"]): + i = idx + idxChar= self.indexChars[idx] + if i == 0 and not kernel["ProblemType"]["UseInitialStridesCD"]: + module.add(ValueSet("constStride%s%s"%(tc,idxChar), 1)) + else: + if not kernel["ProblemType"]["UseInitialStridesCD"]: + i = i-1 + module.add(RegSet("s", "sgprStride%s%s"%(tc,idxChar), \ + "sgprStrides%s"%tc, i)) + + for tc in ('A','B'): + for i, idx in enumerate(problemType["IndexAssignments%s"%tc]): + idxChar= self.indexChars[idx] + if i == 0 and not kernel["ProblemType"]["UseInitialStridesAB"]: + module.add(ValueSet("constStride%s%s"%(tc,idxChar), 1)) + else: + if not kernel["ProblemType"]["UseInitialStridesAB"]: + i = i-1 + module.add(RegSet("s", "sgprStride%s%s"%(tc,idxChar), \ + "sgprStrides%s"%tc, i)) + + module.addSpaceLine() + module.add(ValueSet("MT0", kernel["MacroTile0"])) + module.add(ValueSet("MT1", kernel["MacroTile1"])) + module.add(ValueSet("DepthU", kernel["DepthU"])) + module.add(ValueSet("GSU", kernel["GlobalSplitU"])) + module.add(ValueSet("BpeA", tPA["bpe"])) + module.add(ValueSet("BpeALog2", log2(tPA["bpe"]))) + module.add(ValueSet("BpeB", tPB["bpe"])) + module.add(ValueSet("BpeBLog2", log2(tPB["bpe"]))) + module.addComment0("Number of elements to shift-left SRD") + module.add(ValueSet("SrdShiftLeftA", self.srdShiftLeft['A'])) + module.add(ValueSet("SrdShiftLeftB", self.srdShiftLeft['B'])) + + if kernel["BufferLoad"] or kernel["BufferStore"]: + module.addComment0("2GB limit - set offsets to -1 to exceed this and clamp") + module.add(ValueSet("BufferLimit", 0xffffffff, format=1)) + #TODO-64 : This is max 32-bit negative value, the tail loop + # does incrementally step through the GRO and increment GRO + # which are initialized with this value + module.add(ValueSet("BufferOOB", 0x80000000, format=1)) + + srdUpperValue = SrdUpperValue(self.version) + module.addComment2("Bits 127:96 of SRD.\n" + srdUpperValue.desc()) + module.add(ValueSet("Srd127_96", srdUpperValue.getValue(), format=1)) + + ######################################## + # Global Offsets + ######################################## + # justOffset32 means we should only write the 32-bit offset + # This is used in Buffer addressing modes. + # Flat addressing modes expect the GLOBAL_OFFSET to initialize a full 64-bit address + for (tc, indices, justOffset32, tP) in [ \ + ("C", list(range(0, kernel["ProblemType"]["NumIndicesC"])), kernel["BufferStore"], None), \ + ("A", kernel["ProblemType"]["IndexAssignmentsA"], kernel["BufferLoad"], tPA), \ + ("B", kernel["ProblemType"]["IndexAssignmentsB"], kernel["BufferLoad"], tPB) ]: + + # BufferStore does not use this macro so don't generate it: + if tc == "C" and kernel["BufferStore"]: + continue + + module.addComment1("Global Offset %s"%tc) + numDim = len(indices) + idxChars = [] + for i in indices: + idxChars.append(self.indexChars[i]) + + # macro declaration + calcDims = [] # dimensions which are participating in the address calc (ignores other summation) + mirrorSumDims = [] + macroArgs = [] + for i in range(0, numDim): + if tc == 'C': + useInitialStrides = kernel["ProblemType"]["UseInitialStridesCD"] + idxChar = self.indexChars[i] + else: + useInitialStrides = kernel["ProblemType"]["UseInitialStridesAB"] + idxChar = self.indexChars[tP['ia'][i]] + + # tile index or unroll vgpr or summation + # other summation (other than unroll) are included in the GLOBAL_OFFSET macro but not used in address calc + if tc in ('A','C') and indices[i] == kernel["ProblemType"]["Index0"] \ + or tc in ('B','C') and indices[i] == kernel["ProblemType"]["Index1"] \ + or indices[i] == kernel["ProblemType"]["IndexUnroll"]: + macroArgs.append("vgprOffset%s:req" % idxChars[i]) + calcDims.append(i) + elif indices[i] in kernel["ProblemType"]["IndicesSummation"]: + # other summation index (not unroll) + if tc in ('A', 'B') and indices[i] in kernel["ProblemType"]["MirrorDims%s" % tc]: + mirrorSumDims.append(i) + continue + else: + # other batch or free index + if isPackedIndex(kernel, indices[i]): + calcDims.append(i) + macroArgs.append("vgprOffset%s:req" % idxChars[i]) + elif not justOffset32: # buffer/justOffset32 scalars are included in SRD not the offset, so skip here + calcDims.append(i) + macroArgs.append("sgprOffset%s:req" % idxChars[i]) + macro = Macro("GLOBAL_OFFSET_%s" % tc, "vgprAddr:req", *macroArgs, "vgprTmp:req") + + # Each index may be skipped, scaled by stride, or unscaled + # If destLo is unset, no accumulation is necessary. + + # if the first index (i==0) is unscaled (UseInitialStrides), + # it can be combined at the next update or moved at end + # (if there is no next update) + + pendingOffset = None # offset pending for accumulation + offsetIsVgpr = False # True if the source is VGPR ; False if SGPR + destLo = None + + # true for first addr calc. In this case, we can directly write addr + # rather than accumulating through a tmp + writeDirectToAddr = 1 + + # mirror other summation indices + for i in mirrorSumDims: + if writeDirectToAddr: + dest = "v[\\vgprAddr+0]" + needAdd = 0 # don't need add since writing address directly. + writeDirectToAddr = 0 + else: + dest = "v[\\vgprTmp+0]" + needAdd = 1 + macro.add(VSubU32(dst=dest, \ + src0=sgpr("Size%s"%globalParameters["IndexChars"][indices[i]]), \ + src1=1, \ + comment="mirror %s%s 1"%(tc, globalParameters["IndexChars"][indices[i]]))) + macro.add(VMulLOU32(dst=dest, \ + src0=dest, \ + src1=self.strideRef(tc, indices[i]), \ + comment="mirror %s%s 2"%(tc, globalParameters["IndexChars"][indices[i]]))) + + if needAdd: + writeDirectToAddr = 0 # safety net, once we write address can't directly overwrite it later + destLo = "v[\\vgprAddr+0]" + destHi = "v[\\vgprAddr+1]" + + srcLo = pendingOffset if pendingOffset else destLo + srcHi = 0 if pendingOffset else destHi + macro.add(VAddCOU32(dst=destLo, \ + dst1=VCC(), \ + src0=srcLo, \ + src1="v[\\vgprTmp+0]", \ + comment="accumulate %s lower"%idxChar)) + + for i in calcDims: + # should have eliminated these above + idx = indices[i] + isMirrorIdx = tc in ('A', 'B') and idx in kernel["ProblemType"]["MirrorDims%s" % tc] + assert not (idx in kernel["ProblemType"]["IndicesSummation"] and idx != kernel["ProblemType"]["IndexUnroll"]) + + if indices[i] == kernel["ProblemType"]["Index0"] \ + or indices[i] == kernel["ProblemType"]["Index1"] \ + or indices[i] == kernel["ProblemType"]["IndexUnroll"]: + offsetIsVgpr = True + # other c index sgpr (free or batch) + elif indices[i] < kernel["ProblemType"]["NumIndicesC"]: + if isPackedIndex(kernel, indices[i]): + offsetIsVgpr = True + else: + offsetIsVgpr = False + else: + assert(0) # no other type allowed + + # offset is VGPR or SGPR string to use for the offset + if offsetIsVgpr: + offset = "v[\\vgprOffset%s]" % idxChars[i] + else: + offset = "s[\\sgprOffset%s]" % idxChars[i] + + # macro.addComment0("dim%s pendingOffset=%s offset=%s offsetIsVgpr=%s" \ + # % (self.indexChars[indices[i]], pendingOffset, offset, offsetIsVgpr)) + + needAdd = 0 + # should be indices[i]?? + if i==0 and not useInitialStrides: + # slide into next address calc - can do addr = pendingOffset + nextAddrCalc + pendingOffset = offset + writeDirectToAddr = 0 + else: + # tile index or unroll vgpr + if offsetIsVgpr: + if writeDirectToAddr: + destLo = "v[\\vgprAddr+0]" + destHi = "v[\\vgprAddr+1]" + needAdd = 0 # don't need add since writing address directly. + writeDirectToAddr = 0 + else: + destLo = "v[\\vgprTmp+0]" + destHi = "v[\\vgprTmp+1]" + needAdd = 1 + if isMirrorIdx: + macro.add(VSubI32( + dst="v[\\vgprTmp+0]", + src0=sgpr("Size%s"%globalParameters["IndexChars"][idx]), \ + src1=offset, \ + comment="mirror %s%s 1"%(tc, globalParameters["IndexChars"][indices[i]]))) + macro.add(VSubI32(\ + dst="v[\\vgprTmp+0]", + src0="v[\\vgprTmp+0]", \ + src1=1, \ + comment="mirror %s%s 2"%(tc, globalParameters["IndexChars"][indices[i]]))) + offset = "v[\\vgprTmp+0]" + + # offset * stride + macro.add(VMulLOU32(dst=destLo, + src0=self.strideRef(tc, indices[i]), \ + src1=offset, \ + comment="mul d%u lower"%i)) + if not justOffset32: + macro.add(VMulHIU32(dst=destHi, + src0=self.strideRef(tc, indices[i]), \ + src1=offset, \ + comment="mul d%u upper"%i)) + else: # offset is SGPR: + assert not isMirrorIdx + if not justOffset32: + # buffer mode (aka justOffset32) does scalars into SRD not offset + macro.add(VMovB32(dst="v[\\vgprTmp+2]", src="s[\\sgprOffset%s]"%idxChars[i], \ + comment="sgprOffset -> vgprTmp+2")) + # offset * stride + macro.add(VMulLOU32(dst="v[\\vgprTmp+0]", \ + src0=self.strideRef(tc, indices[i]), src1="v[\\vgprTmp+2]", \ + comment="other stride mul d%u lower"%i)) + macro.add(VMulHIU32(dst="v[\\vgprTmp+1]", \ + src0=self.strideRef(tc, indices[i]), src1="v[\\vgprTmp+2]", \ + comment="mul d%u upper"%i)) + needAdd = 1 + + if needAdd: + writeDirectToAddr = 0 # safety net, once we write address can't directly overwrite it later + destLo = "v[\\vgprAddr+0]" + destHi = "v[\\vgprAddr+1]" + # addr += offset * stride (lo) : accumulate just-computed address term into addr + + srcLo = pendingOffset if pendingOffset else destLo + srcHi = 0 if pendingOffset else destHi + macro.add(VAddCOU32(dst=destLo, dst1=VCC(), \ + src0=srcLo, src1="v[\\vgprTmp+0]", \ + comment="accumulate %s lower"%idxChar)) + + # addr += offset * stride (hi) + if not justOffset32: + macro.add(VAddCCOU32(dst="v[\\vgprAddr+1]", dst1=VCC(), \ + src0="v[\\vgprTmp+1]", src1=srcHi, src2=VCC(), \ + comment="accumulate %s upper"%idxChar)) + pendingOffset = None + + # pendingOffset but never got a chance to apply it, + # need to just add an explicit move or add: + # this can happen for small-order tensors + if pendingOffset != None: + destLo = "v[\\vgprAddr+0]" + if writeDirectToAddr: + macro.add(VMovB32(dst=destLo, src=offset, comment="setup d0 lower")) + if not justOffset32: + macro.add(VMovB32(dst="v[\\vgprAddr+1]", src=hex(0), comment="d0 upper")) + else: + macro.add(VAddCOU32(dst=destLo, dst1=VCC(), \ + src0=destLo, src1=pendingOffset, \ + comment="accumulate final pendingOffset")) + + + if tP != None and kernel["BufferLoad"] and self.srdShiftLeft[tc]: + macro.add(VAddU32(dst="v[\\vgprAddr+0]", \ + src0=hex(self.srdShiftLeft[tc]), \ + src1="v[\\vgprAddr+0]", \ + comment="add prepad for pointer shift")) + + # addr *= bytes/element + if justOffset32: + macro.add(staticMultiply("v[\\vgprAddr+0]", "v[\\vgprAddr+0]", self.bpeAB, None, "offset *= bytes/element")) + else: + macro.add(VLShiftLeftB64(dst="v[\\vgprAddr+0:\\vgprAddr+1]", \ + shiftHex=hex(log2(self.bpeAB)), \ + src="v[\\vgprAddr+0:\\vgprAddr+1]", \ + comment="offset *= bytes/element")) + module.add(macro) + + module.add(MacroVDynamicScalarDiv(kernel["WavefrontSize"])) + + module.setNoOpt(True) + return module + + def checkResources(self, mkb: KernelBody): + # register allocation + totalVgprs = self.vgprPool.size() + totalAgprs = self.agprPool.size() + totalSgprs = self.sgprPool.size() + + mkb.setGprs(totalVgprs=totalVgprs, totalAgprs=totalAgprs, totalSgprs=totalSgprs) + module = mkb.body + + if self.overflowedResources: + if self.overflowedResources == 1: + msg = "too many vgprs" + elif self.overflowedResources == 2: + msg = "too many sgprs" + elif self.overflowedResources == 3: + msg = "half store requires at least two elements per batch" + elif self.overflowedResources == 4: + msg = "Occupancy limit" + elif self.overflowedResources == 5: + msg = "reading and writing LDS at same time require 2 LDS buffer" + elif self.overflowedResources == 6: + msg = "SIA2 better with occupancy 2" + else: + msg = "unknown" + + if globalParameters["PrintSolutionRejectionReason"]: + printWarning("%s overflowed resources. errorCode=%d, msg=\"%s\", vgprs=%u, sgprs=%u" \ + % (self.kernelName, self.overflowedResources, msg, \ + self.vgprPool.size(), self.sgprPool.size())) + module.add(SEndpgm(comment="overflowed resources"), 0) + module.add(ValueIf(value=0), 1) + + ############################################################################## + # code phrase for load batched address from array of buffer pointer + ############################################################################## + def loadBatchedAddress(self, kernel, Batch, tmpSgprResource: RegisterPoolResource): + tmpSgpr = tmpSgprResource.idx + laneSC = tmpSgprResource.size + module = Module("loadBatchedAddress %s" % Batch) + module.addSpaceLine() + + # handle Batch C/D + if not kernel["_GlobalAccumulation"]: + for idx in kernel["ProblemType"]["IndicesBatch"]: + if not isPackedIndex(kernel,idx): + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=sgpr(Batch), src1=0x8, comment="offset of global buffer address")) + module.add(SLoadB64(dst=sgpr("AddressD", 2), base=sgpr("AddressD",2), soffset=sgpr(tmpSgpr), comment="load global buffer D address")) + + endCheckLabel = Label(self.labels.getName(f"label_skip_c_buffer_deref_{Batch}"), "") + module.add(SBranchIfZero("Beta", kernel["ProblemType"]["ComputeDataType"], tmpSgpr, laneSC, endCheckLabel, \ + kernel['WavefrontSize'])) + + for idx in kernel["ProblemType"]["IndicesBatch"]: + if not isPackedIndex(kernel,idx): + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=sgpr(Batch), src1=0x8, comment="offset of global buffer address")) + module.add(SLoadB64(dst=sgpr("AddressC", 2), base=sgpr("AddressC",2), soffset=sgpr(tmpSgpr), comment="load global buffer C address")) + + module.add(endCheckLabel) + + #handle Batch A/B + endCheckLabel = Label(self.labels.getName(f"label_skip_ab_buffer_deref_{Batch}"), "") + module.add(SMovB32(dst=sgpr(tmpSgpr), src=hex(1), comment="check summation size")) + for i in range(0, self.numSgprSizesSum): + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=sgpr("SizesSum+%u"%(i)), src1=sgpr(tmpSgpr), comment="check summation size")) + module.add(SCmpEQU32(src0=sgpr(tmpSgpr), src1=hex(0), comment="skip buffer deref is size of summation is 0")) + module.add(SCBranchSCC1(labelName=endCheckLabel.getLabelName(), comment="skip buffer deref is size of summation is 0")) + module.add(SBranchIfZero("Alpha", kernel["ProblemType"]["ComputeDataType"], tmpSgpr, laneSC, endCheckLabel, \ + kernel['WavefrontSize'])) + + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=sgpr(Batch), src1=0x8, comment="offset of global buffer address")) + for idx in kernel["ProblemType"]["IndicesBatch"]: + if not isPackedIndex(kernel,idx): + module.add(SLoadB64(dst=sgpr("AddressA", 2), base=sgpr("AddressA",2), soffset=sgpr(tmpSgpr), comment="load global buffer A address")) + module.add(SLoadB64(dst=sgpr("AddressB", 2), base=sgpr("AddressB",2), soffset=sgpr(tmpSgpr), comment="load global buffer B address")) + + module.add(endCheckLabel) + + return module + + ############################################################################## + def defineAndResources(self, kernel, tPA, tPB): + module = Module("allocateResources") + unsetD = self.undefineSgpr("OffsetD") + unsetC = self.undefineSgpr("OffsetC") + unsetA = self.undefineSgpr("OffsetA") + unsetB = self.undefineSgpr("OffsetB") + self.defineVariableSgprs(kernel) + module.add(self.macroAndSet(kernel, tPA, tPB)) + + module.addComment2("Allocate Resources") + if kernel["StorePriorityOpt"]: + module.add(SSetPrior(prior=3, comment="optimization store")) + + if self.do["PreLoop"]: + if self.db["InitSgpr"] & 0x1: + module.addComment1("Init SGPRs") + for i in range(self.firstInitSgpr, self.sgprPool.size()): + module.add(SMovB32(dst=sgpr(i), src=hex(self.initSgprValue), comment="InitSgpr&0x1")) + module.addSpaceLine() + + if self.db["InitVgpr"] & 0x1: + module.addComment1("Init VGPRs") + for i in range(1, self.totalVgprs): + module.add(VMovB32(dst=vgpr(i), src=hex(self.initVgprValue), comment="InitVgpr&0x1")) + module.addSpaceLine() + + # set m0 + module.add(SMovB32(dst=mgpr(0), src=hex(kernel["LdsNumElements"] * self.bpeAB), + comment="LDS clamp at %u bytes"%(kernel["LdsNumElements"] * self.bpeAB))) + + # set Serial id vgpr + module.add(VMovB32(dst=vgpr("Serial"), src=vgpr(0), comment="thread serial id")) + + if self.kernel["WavefrontSize"] == 32: + module.add(SMovB32(dst=VCC(setHi=True), src=0, comment="Ensure hi bits are zero")) + + ######################################## + # load kernel args + module.addComment1("Load Kernel Args") + self.kernArgOffset = 0 + argLoader = ArgumentLoader() + if globalParameters["DebugKernel"]: + module.add(argLoader.loadSingleKernArg("AddressDbg", "KernArgAddress")) + module.add(argLoader.loadSingleKernArg("AddressDbg+1", "KernArgAddress")) + + argLoader.loadSingleKernArg("Tensor2dSizeC+0", "KernArgAddress", False) + argLoader.loadSingleKernArg("Tensor2dSizeC+1", "KernArgAddress", False) + + load = self.numSgprToLoad + sgprStart = self.sgprs["Tensor2dSizeA"] + module.addModuleAsFlatItems(argLoader.loadAllKernArg(sgprStart, "KernArgAddress", load)) + if kernel.enabledSetPrioSplitLDS: + module.add(SSetPrior(prior=1, comment="prioritize init code so as to issue load sooner")) + module.add(SWaitCnt(lgkmcnt=0, comment="wait for %u bytes of kern args" % argLoader.getOffset() )) + + if not kernel["ProblemType"]["StridedBatched"]: + with self.allocTmpSgpr(self.laneSGPRCount) as tmpSgpr: + module.add(self.loadBatchedAddress(kernel, "WorkGroup2", tmpSgpr)) + module.add(SWaitCnt(lgkmcnt=0, comment="wait global buffer address ready")) + else: + module.add(ValueIf(0)) + + # add offset to buffer + def addOffset2Buffer(imod, mat, value): + imod.add(SLShiftLeftB32(dst=sgpr("Offset%s"%mat), src=sgpr("Offset%s"%mat), shiftHex=hex(value), comment="elements offset to bytes offset")) + imod.add(SAddU32(dst=sgpr("Address%s+0"%mat), src0=sgpr("Address%s+0"%mat), src1=sgpr("Offset%s"%mat), comment="add offset to buffer address")) + imod.add(SAddCU32(dst=sgpr("Address%s+1"%mat), src0=sgpr("Address%s+1"%mat), src1=0, comment="add offset to buffer address")) + + if not kernel["_GlobalAccumulation"]: + addOffset2Buffer(module, "D", log2(self.bpeCexternal)) + addOffset2Buffer(module, "C", log2(self.bpeCexternal)) + addOffset2Buffer(module, "A", log2(self.bpeAB)) + addOffset2Buffer(module, "B", log2(self.bpeAB)) + + # self.groOffsetInMacroTile == 1 case, subtract pre-pad here + if self.groOffsetInMacroTile: + prePad = self.srdShiftLeft["A"] * tPA["bpe"] # leave room in case we have to pointer shift + module.add(SSubU32(dst=sgpr("AddressA+0"), src0=sgpr("AddressA+0"), src1=prePad, comment="pre-pad to make room for possible pointer shift")) + module.add(SSubBU32(dst=sgpr("AddressA+1"), src0=sgpr("AddressA+1"), src1=0, comment="pre-pad to make room for possible pointer shift")) + prePad = self.srdShiftLeft["B"] * tPB["bpe"] # leave room in case we have to pointer shift + module.add(SSubU32(dst=sgpr("AddressB+0"), src0=sgpr("AddressB+0"), src1=prePad, comment="pre-pad to make room for possible pointer shift")) + module.add(SSubBU32(dst=sgpr("AddressB+1"), src0=sgpr("AddressB+1"), src1=0, comment="pre-pad to make room for possible pointer shift")) + + # undefine Offset sgpr + module.addSpaceLine() + module.add(unsetD) + module.add(unsetC) + module.add(unsetA) + module.add(unsetB) + + # Check alpha == 0, is done before kernel body + # so if alpha/beta=Half, they haven't been converted to f32 + # This means we can use ComputeDataType as AlphaType (even +"HPA") + if self.do["ApplyAlpha"]: + module.addComment1("Short circuit condition if Alpha == 0, then sumDims=0") + endCheckLabel = Label("AlphaNonZero", "") + module.add(SBranchIfNotZero("Alpha", kernel["ProblemType"]["ComputeDataType"], endCheckLabel)) + + # Conditional set summation dimensions to 0 on SCC==1 + for i in range(0, self.numSgprSizesSum): + module.add(SMovB32(dst=sgpr("SizesSum+%u"%(i)), src=hex(0), comment="Set summation dim=0 if Alpha == 0")) + + # Jump here if alpha is non-zero + module.add(endCheckLabel) + + if kernel["MagicDivAlg"]==2: + for idxChar in sorted(set(kernel["PackedC0IdxChars"][:-1] + kernel["PackedC1IdxChars"][:-1])): + module.add(SLShiftRightB32(dst=sgpr("MagicAbitSize%s"%idxChar), src=sgpr("MagicShiftSize%s"%idxChar), shiftHex=31, comment="extract abit")) + module.add(SAndB32(dst=sgpr("MagicShiftSize%s"%idxChar), src0=sgpr("MagicShiftSize%s"%idxChar), src1=hex(0x7fffffff), comment="remove abit")) + + ######################################## + # Debug Buffer + if globalParameters["DebugKernel"]: + module.addComment1("Debug Buffer") + + # nwg0 FIXME use NumWorkGroups0 + nwg0 = self.vgprPool.checkOut(1) + tmpVgpr = self.vgprPool.checkOutAligned(2, 2) + tmpVgprRes = RegisterPoolResource(tmpVgpr, 2) + module.addComment("// nwg0 = (size%s + MT%s - 1) / MT%s;%s" \ + % (self.tileChar0, self.tileChar0, self.tileChar0, self.endLine)) + module.add(VMovB32(dst=vgpr(tmpVgpr), src=hex(kernel["MacroTile0"]-1), comment="MT0-1")) + module.add(VAddCOU32(dst=vgpr(nwg0), dst1=VCC(), src0=sgpr("SizesFree+0"), \ + src1=vgpr(tmpVgpr), comment="%s = size0+MT0-1"%vgpr(nwg0))) + module.add(vectorStaticDivide(nwg0, nwg0, kernel["MacroTile0"], tmpVgprRes)) + self.vgprPool.checkIn(tmpVgpr) + self.nipt = 16 # num integers per thread + v = self.vgprPool.checkOut(3) + module.add(VMovB32(dst=vgpr(v), src=sgpr("WorkGroup0"), comment="%s=wg0"%vgpr(v) )) + module.add(VMovB32(dst=vgpr(v+1), src=sgpr("WorkGroup1"), comment="%s=wg1"%vgpr(v+1) )) + module.add(VMulLOU32(dst=vgpr(v+1), src0=vgpr(v+1), src1=vgpr(nwg0), \ + comment="%s=wg1*nwg0"%vgpr(v+1) )) + module.add(VAddCOU32(dst=vgpr(v), dst1=VCC(), src0=vgpr(v), src1=vgpr(v+1), \ + comment="%s=wg1*nwg0+wg0"%vgpr(v) )) + with self.allocTmpSgpr(1) as tmpSgprInfo: + module.add(staticMultiply(vgpr(v), vgpr(v), kernel["NumThreads"], tmpSgprInfo)) + module.add(VAddCOU32(dst=vgpr(v), dst1=VCC(), src0=vgpr(v), src1=vgpr("Serial"), \ + comment="%s=tid+NT*(wg1*nwg0+wg0)=serial"%vgpr(v) )) + module.add(VMulLOU32(dst=vgpr(v), src0=hex(self.nipt*4), src1=vgpr(v), \ + comment="%s=serial*nipt*4"%vgpr(v) )) + module.add(VMovB32(dst=vgpr(v+1), src=0)) + module.add(VAddCOU32(dst=vgpr("AddressDbg"), dst1=VCC(), src0=sgpr("AddressDbg"), src1=vgpr(v), \ + comment="%s=AddrD* + serial*nipt*4"%vgpr("AddressDbg") )) + module.add(VMovB32(dst=vgpr(v+2), src=sgpr("AddressDbg+1"), comment="%s=AddressD1"%vgpr(v+2) )) + module.add(VAddCCOU32(dst=vgpr("AddressDbg+1"), dst1=VCC(), src0=vgpr(v+2), \ + src1=vgpr(v+1), src2=VCC(), comment="%s=AddrD* + serial*nipt*4"%vgpr("AddressDbg") )) + module.add(SMovB32(dst=sgpr("DebugKernelItems"), src=0)) + self.vgprPool.checkIn(v) + self.vgprPool.checkIn(nwg0) + + if self.db["InitLds"]: + tmp = RegisterPoolResource(idx=self.vgprPool.checkOut(2), size=2) + numBytesPerElement = kernel["ProblemType"]["DataType"].numBytes() + module.add(DSInit(tmp, kernel["NumThreads"], kernel["LdsNumElements"], \ + numBytesPerElement, self.initLdsValue)) + self.vgprPool.checkIn(tmp.idx) + + return module + + def extractPackedCoord1ToRowStart(self, kernel, packedC1, packedCoordVgpr, storeChar): + # calculate packed rowStart vgpr + # vgprTmp assignments: + # - tmp+0 is the incoming packed coordinate 1, used on replay too + # - tmp+1 is DIV output + # - tmp+2 is scratch + # - tmp+3 holds thread rowStart free1 offset + module = Module("extractPackedCoord1ToRowStart") + tmpV0 = self.vgprPool.checkOut(4) + tmpV1 = tmpV0 + 1 + tmpV2 = tmpV0 + 2 + tmpV3 = tmpV0 + 3 + + module.add(VMovB32(dst=vgpr(tmpV0), src=vgpr(packedCoordVgpr), comment="copy coord1 then unpack")) + for i,idx in enumerate(packedC1[:-1]): + idxChar= globalParameters["IndexChars"][idx] + module.addComment0("extract %s"%self.sizeRef(idx)) + module.add(MacroInstruction(name="V_MAGIC_DIV", \ + args=[tmpV1, vgpr(tmpV0), sgpr("MagicNumberSize%s"%idxChar), \ + sgpr("MagicShiftSize%s"%idxChar), (sgpr("MagicAbitSize%s"%idxChar) if kernel["MagicDivAlg"]==2 else "0")])) + module.add(VMulLOU32(dst=vgpr(tmpV2), src0=vgpr(tmpV1), src1=self.sizeRef(idx), comment="remainder part 1")) + module.add(VSubU32(dst=vgpr(tmpV2), src0=vgpr(tmpV0), src1=vgpr(tmpV2), comment="remainder part 2")) + if i==0: + module.add(VMulLOU32(dst=vgpr(tmpV3), src0=vgpr(tmpV2), \ + src1=self.strideRef(storeChar, idx), comment="addrCalc <- scaled extracted dim")) + else: + module.add(VMulLOU32(dst=vgpr(tmpV2), src0=vgpr(tmpV2), \ + src1=self.strideRef(storeChar, idx), comment="scale extracted dim")) + module.add(VAddU32(dst=vgpr(tmpV3), src0=vgpr(tmpV3), \ + src1=vgpr(tmpV2), comment="addrCalc += scaled extracted dim ")) + + if i < len(packedC1)-2: + module.add(VMovB32(dst=vgpr(tmpV0), src=vgpr(tmpV1), \ + comment="Copy remaining bits for next divide")) + + module.addComment0("extract final %s"%self.sizeRef(packedC1[-1])) + module.add(VMulLOU32(dst=vgpr(tmpV2), src0=vgpr(tmpV1), \ + src1=self.strideRef(storeChar, packedC1[-1]), comment="scale final extracted dim")) + module.add(VAddU32(dst=vgpr(self.vgprs.coutRowPtr), src0=vgpr(tmpV3), \ + src1=vgpr(tmpV2), comment="rowStart += scaled extracted dim ")) + + self.vgprPool.checkIn(tmpV0) + return module + + ############################################################################## + # Global Read Addresses: WorkGroup + ############################################################################## + def graWorkGroup(self, kernel): + module = Module("graWorkGroup") + module.addComment0("graWorkGroup mapping") + if kernel["GlobalSplitU"] > 1: + module.addComment("GSU-not-WGMapRR :nwg1 = (size%s + MT%s - 1) / MT%s;" \ + % (self.tileChar1, self.tileChar1, self.tileChar1)) + + # gsuSumIdx = wg1 % GSU + # wg1 = wg1 / GSU + with self.allocTmpSgpr(3) as tmpSgprInfo:# needs 3 + tmpSgpr = tmpSgprInfo.idx + divisor = tmpSgpr+2 + module.add(SMovB32(dst=sgpr(divisor), src=sgpr("WorkGroup1"), \ + comment="copying for divisor")) + + module.add(scalarStaticDivideAndRemainder("WorkGroup1", "GSUSumIdx", \ + divisor, kernel["GlobalSplitU"], tmpSgprInfo, 1)) + + ######################################## + # Blocked rows or columns + absWgm = abs(kernel["WorkGroupMapping"]) + if abs(kernel["WorkGroupMapping"]) > 1: + smallNumMagicShift = 31 + magicNumberWgm = ((1< 1: + module.add(vectorStaticDivide(wReg, dividendReg, waveWidth, tmpVgprRes)) + module.add(vectorStaticRemainder(tmpVgpr, wReg, wReg, num1DWaves, tmpVgprRes, tmpSgprInfo)) + module.add(staticMultiply(vgpr(wReg), vgpr(wReg), strideWave, tmpSgprInfo)) + module.add(VAddU32(dst=vgpr(rReg), src0=vgpr(wReg), src1=vgpr(rReg))) + # need division for qReg + module.add(vectorStaticDivide(qReg, qReg, kernel["MatrixInstN"], tmpVgprRes)) + lrvwOther = self.lrvwB if tP["isA"] else self.lrvwA # The other side of lrvw + if lrvwOther == 2 and not kernel["allowLRVWforTLUandMI"] and tP["tlu"]: + # DirectToVgpr + LocalReadVectorWidth=2 case, multiply qReg by 2 + module.add(staticMultiply(vgpr(qReg), vgpr(qReg), lrvwOther, tmpSgprInfo)) + # release register + self.vgprPool.checkIn(wReg) + else: + module.add(vectorStaticDivideAndRemainder(qReg, rReg, dividendReg, divisor, tmpVgprRes)) + + + if kernel["WaveSeparateGlobalRead%s"%tc]: + with self.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + module.add(VReadfirstlaneB32(dst=sgpr(tmpSgpr), src=vgpr("Serial"), comment="WaveIdxWavefrontWidth")) + module.add(SLShiftRightB32(dst=sgpr(tmpSgpr), src=sgpr(tmpSgpr), shiftHex=hex(log2(kernel["WavefrontSize"])), comment="WaveId")) + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=(kernel[tP["lsp"]] * tP["nrp"]), \ + comment="Global Read Wave: each wave loads continuous lsp(%u)*nrp(%u) columns" % (kernel[tP["lsp"]], tP["nrp"]))) + module.add(VAddU32(dst=vgpr(qReg), src0=sgpr(tmpSgpr), src1=vgpr(qReg), \ + comment="Global Read Wave: add back to column index")) + self.vgprPool.checkIn(dividendReg) + self.vgprPool.checkIn(dummy) + + with self.allocTmpSgpr(1) as tmpSgprInfo: + if tP["glvw"] > 1: + if tP["tlu"]: + module.addComment0("gro-tile *= glvw") + module.add(staticMultiply(vgpr(tReg), vgpr(tReg), tP["glvw"], tmpSgprInfo)) + else: + module.addComment0("gro-unroll *= glvw") + module.add(staticMultiply(vgpr(uReg), vgpr(uReg), tP["glvw"], tmpSgprInfo)) + if forceSwap: + # in this case, need to multiply vw to gro-tile + module.addComment0("gro-tile *= vw") + module.add(staticMultiply(vgpr(tReg), vgpr(tReg), kernel["VectorWidth"], tmpSgprInfo)) + + if not self.groOffsetInMacroTile: + # Buffer Load will set the SRD to start of the MacroTile + # So don't add the static wg-related component here - save for later. + module.add(staticMultiply(vgpr(tmpVgpr), sgpr(tP["wg"]), kernel[tP["mt"]], tmpSgprInfo)) # workgroup + module.add(VAddCOU32(dst=vgpr(tReg2), dst1=VCC(), src0=vgpr(tmpVgpr), \ + src1=vgpr(tReg), comment="gro%s-tile = serial%s%s*VW + (wg%s*MT%s)" \ + % (tc, tOpStr, divisorName, tc, tc) )) + + if kernel["GlobalSplitU"] > 1: + uReg2 = self.vgprPool.checkOut(1, "uReg2", self.preventVgprOverflowDuringNewTile) + module.add(VMovB32(dst=vgpr(uReg2), src=vgpr(uReg), comment="copy for GlobalSplitU")) + tP["gpr"]["uReg2"] = uReg2 + tP["gpr"]["lwoT"] = tReg + tP["gpr"]["tReg"] = tReg2 + tP["gpr"]["uReg"] = uReg + self.vgprPool.checkIn(tmpVgpr) + + return Module("graTileAssignment (Empty)") if self.dontAppendCode else module + + ############################################################################## + # Global Read Addresses: Unroll Assignment + ############################################################################## + def graUnrollAssignment(self, kernel, tP): + module = Module("graUnrollAssignment") + # note groOffsetInMacroTile rolls these into SRD so don't change here: + if not self.groOffsetInMacroTile and kernel["GlobalSplitU"] > 1: + gsuOffset = self.vgprPool.checkOut(1, "gsuOffset", self.preventVgprOverflowDuringNewTile) + module.add(VMovB32(dst=vgpr(gsuOffset), src=sgpr("GSUSumIdx"), comment="=gsuSumIdx")) + + with self.allocTmpSgpr(1) as tmpSgprInfo: + # graUnrollAssignment += gsuSumIdx*DepthU + module.add(staticMultiply(vgpr(gsuOffset), vgpr(gsuOffset), kernel["DepthU"], tmpSgprInfo)) + + module.add(VAddCOU32(dst=vgpr(tP["gpr"]["uReg"]), dst1=VCC(), \ + src0=vgpr(gsuOffset), src1=vgpr(tP["gpr"]["uReg"]), \ + comment="graUnrollAssignment += gsuOffset")) + self.vgprPool.checkIn(gsuOffset) + else: + module.addComment0(vgpr(tP["gpr"]["uReg"])) + + return Module("graUnrollAssignment (Empty)") if self.dontAppendCode else module + + ############################################################################## + # Global Read Addresses: Other Free Assignments + ############################################################################## + def graOtherFreeAssignments(self): + module = Module("graOtherFreeAssignments") + module.addComment0(sgpr("WorkGroup2")) + return module + + ############################################################################## + # Global Read Addresses: Other Summation Assignments + ############################################################################## + def graOtherSummationAssignments(self, kernel): + module = Module("graOtherSummationAssignments") + for i in range(0,kernel["ProblemType"]["NumIndicesSummation"]-1): + index = i + module.add(ValueSet(name="globalReadOffsetA%s" % self.indexChars[index], value=0)) + module.add(ValueSet(name="globalReadOffsetB%s" % self.indexChars[index], value=0)) + return module + + ############################################################################## + # Global Read Addresses: Tile Offsets A/B + ############################################################################## + def graTileOffsets(self, kernel, tP): + module = Module("graTileOffsets") + tc = tP["tensorChar"] + tP["vgprPackedOffsets"] = None + tP["vgprTileOffsetsCheckOut"] = False + tP["numVgprTileOffsets"] = 0 + if kernel["_UseSgprForGRO"]: + # Let the vgprTileOffsets checkin handle tReg later since these are same vgpr + tP["vgprTileOffsets"] = tP["gpr"]["tReg"] + else: + numTileOffsets = tP["nrt"] + tP["vgprTileOffsets"] = self.vgprPool.checkOut(numTileOffsets, "vgprTileOffsets", self.preventVgprOverflowDuringNewTile) + tP["vgprTileOffsetsCheckOut"] = True + v = tP["vgprTileOffsets"] + numExtraPackedOffsetsPerTile = len(tP["PackedIndices"])-1 + if numExtraPackedOffsetsPerTile: + tP["vgprPackedOffsets"] = self.vgprPool.checkOut(numExtraPackedOffsetsPerTile * numTileOffsets, "vgprPackedOffsets", self.preventVgprOverflowDuringNewTile) + strideIdx = tP["lsc"] if tP["tlu"] else tP["lsp"] + stride = kernel[strideIdx] + # adjustment for DirectToVgpr + tlu=False + VW > 1 case + strideInterleave = False + if kernel["DirectToVgpr%c"%tc] and (not tP["tlu"]) and kernel["VectorWidth"] > 1: + strideInterleave = True + stride = stride * kernel["VectorWidth"] - (kernel["VectorWidth"] - 1) + + + module.add(VMovB32(dst=vgpr(v), src=vgpr(tP["gpr"]["tReg"]), comment="gro%s%s_%u"%(tP["tensorChar"], tP["tileChar"], 0) )) + for l in range(1, tP["nrt"]): + strideValue = stride + if strideInterleave and (l & 1) != 0: + strideValue = 1 + module.add(VAddCOU32(dst=vgpr(v+l), dst1=VCC(), src0=strideValue, \ + src1=vgpr(v+l-1), comment="gro%s%s_%u += %s"%(tP["tensorChar"], tP["tileChar"], l, strideIdx) )) + if numExtraPackedOffsetsPerTile: + tmpV = self.vgprPool.checkOutAligned(2,2,"packTmp", self.preventVgprOverflowDuringNewTile) + + for l in range(0, tP["nrt"]): + lastGroVgpr = vgpr(v+l) + lastGroIdx = tP["PackedIndices"][0] + module.addSpaceLine() + for p in range(0, numExtraPackedOffsetsPerTile): + groIdx = tP["PackedIndices"][p+1] + groChar = globalParameters["IndexChars"][tP["PackedIndices"][p+1]] + groVgpr = vgpr(tP["vgprPackedOffsets"] + l*numExtraPackedOffsetsPerTile + p) + pChar = globalParameters["IndexChars"][tP["PackedIndices"][p]] + module.add(MacroInstruction(name="V_MAGIC_DIV", \ + args=[tmpV, lastGroVgpr, sgpr("MagicNumberSize%s"%pChar), \ + sgpr("MagicShiftSize%s"%pChar), (sgpr("MagicAbitSize%s"%pChar) if kernel["MagicDivAlg"]==2 else "0")] )) + module.add(VMovB32(dst=groVgpr, src=vgpr(tmpV), comment="extract gro%s%s_%u (%s)"%(tc,groChar,l,groVgpr))) + module.add(VMulLOU32(dst=vgpr(tmpV), src0=groVgpr, src1=sgpr("SizesFree+%u"%lastGroIdx), comment="remainder part 1")) + module.add(VSubU32(dst=lastGroVgpr, src0=lastGroVgpr, src1=vgpr(tmpV), \ + comment="remove extracted bits from gro%s%s_%u (%s)"%(tc, globalParameters["IndexChars"][lastGroIdx], l, lastGroVgpr))) + lastGroVgpr = groVgpr + lastGroIdx = groIdx + self.vgprPool.checkIn(tmpV) + + # groOffsetInMacroTile uses same register for both of these, don't free it here: + if tP["gpr"]["lwoT"] != tP["gpr"]["tReg"] : + self.vgprPool.checkIn(tP["gpr"]["tReg"]) + tP["gpr"]["tReg"] = None + return Module("graTileOffsets (Empty)") if self.dontAppendCode else module + + ############################################################################## + # Global Read Addresses: Unroll Offsets A/B + ############################################################################## + def graUnrollOffsets(self, kernel, tP): + module = Module("graUnrollOffsets") + tc = tP["tensorChar"] + if kernel["_UseSgprForGRO"]: + tP["gpr"]["unrollOffsets"] = tP["gpr"]["uReg"] + else: + numUnrollOffsets = tP["nru"] + tP["gpr"]["unrollOffsets"] = self.vgprPool.checkOut(numUnrollOffsets, "unrollOffsets", self.preventVgprOverflowDuringNewTile) + v = tP["gpr"]["unrollOffsets"] + strideIdx = (tP["lsp"] if tP["tlu"] else tP["lsc"]) + stride = kernel[strideIdx] + prevStride = 0 + totalStride = 0 + lrvwOther = self.lrvwB if tP["isA"] else self.lrvwA # The other side of lrvw + tluOther = kernel["ProblemType"]["TLUB"] if tP["isA"] else kernel["ProblemType"]["TLUA"] # The other side of tlu + module.add(VMovB32(dst=vgpr(v), src=vgpr(tP["gpr"]["uReg"]), comment="gro%s%s_%u"%(tP["tensorChar"], self.unrollChar, 0))) + for l in range(1, tP["nru"]): + totalStride += stride + if tP["tlu"] and kernel["DirectToVgpr%s"%tc] and lrvwOther == 2 and not tluOther: + # DirectToVgpr + LocalReadVectorWidth=2 case, stride * 2 is added every 2. Add 1 in odd l case + totalStride = stride * (l - (l % 2)) + (l % 2) + currStride = totalStride - prevStride + prevStride = totalStride + module.add(VAddCOU32(dst=vgpr(v+l), dst1=VCC(), src0=currStride, \ + src1=vgpr(v+l-1), comment="gro%s%s_%u + %s"%(tP["tensorChar"], self.unrollChar, l, strideIdx))) + #self.vgprPool.checkIn(tP["gpr"]["uReg"]) + return Module("graUnrollOffsets (Empty)") if self.dontAppendCode else module + + ############################################################################## + # Global Read Addresses: Shift A/B + # See if the load (including vw) will extend past the 'free' dim of the + # tensor. If so clip to the last legal value which is inside the array + ############################################################################## + def graShift(self, kernel, tP): + # graShift requires a vgpr for each address component (so each component + # can be examined and shifted if necessary) - therefore does not work + # with UseSgprForGRO. + assert(not kernel["_UseSgprForGRO"]) + + module = Module("graShift") + #tc = tP["tensorChar"] + # edge value + margin = tP["glvw"] if tP["rtv"] else 1 + edge = self.vgprPool.checkOut(1, "edge", self.preventVgprOverflowDuringNewTile) + + with self.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + if self.groOffsetInMacroTile: + # Subtract the static component from SizesFree: + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=sgpr(tP["wg"]), src1=kernel[tP["mt"]], comment="WorkGroup[01] * MT")) + module.add(SSubU32(dst=sgpr(tmpSgpr), src0=self.sizeRef(tP["idx"]), src1=sgpr(tmpSgpr), \ + comment="edge = Size%s - WG*MT"%(tP["tileChar"]))) + # use math here to use unsigned (to increase range) + # - add srdShiftLeft to tmpSgpr - ensure it is always positive + # - below add srdShiftLeft to a tmp copy of the offset used for the compare + # edge = (Size - WG*MT) - margin = the last valid load position that won't cause OOB + # offset = the current load position for this thread + # so if offset is larger than edge, we go back to the edge position + module.add(SSubU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=margin, comment="edge -= margin(%u)"%(margin))) + module.add(VMovB32(dst=vgpr(edge), src=sgpr(tmpSgpr), comment="edge vgpr = Size%s- WG*MT - margin(%u)"%(tP["tileChar"], margin) )) + #shiftedEdge = self.vgprPool.checkOut(1, "shiftedEdge", self.preventVgprOverflowDuringNewTile) + #module.add(VAddCOU32(dst=vgpr(shiftedEdge), dst1=VCC(), src0=vgpr(edge), src1=self.srdShiftLeft[tc], + # comment="shiftedEdge = edge + srdShiftLeft({})".format(self.srdShiftLeft[tc]))) + else: + module.add(SSubU32(dst=sgpr(tmpSgpr), src0=self.sizeRef(tP["idx"]), src1=margin, \ + comment="edge = Size%s-%u"%(tP["tileChar"], margin) )) + module.add(VMovB32(dst=vgpr(edge), src=sgpr(tmpSgpr), \ + comment="edge vgpr = Size%s-%u"%(tP["tileChar"], margin) )) + + # shift offsets + vSrc = tP["vgprTileOffsets"] + vDst = tP["vgprTileOffsets"] + with self.allocTmpSgpr(self.laneSGPRCount) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + for l in range(0, tP["nrt"]): + # compare + cmpCommentText = "offset < edge" + if self.groOffsetInMacroTile: + module.add(VMinI32(dst=vgpr(vDst+l), src0=vgpr(edge), src1=vgpr(vSrc+l), + comment="offset = (%s) ? offset(v%u) : edge(v%u)"%(cmpCommentText, vSrc+l, edge))) + else: + module.add(VCmpLtU32(dst=sgpr(tmpSgpr,self.laneSGPRCount), src0=vgpr(vSrc+l), src1=vgpr(edge), + comment="shiftedOffset < shiftedEdge")) + # shift + module.add(VCndMaskB32(dst=vgpr(vDst+l), src0=vgpr(edge), src1=vgpr(vSrc+l), src2=sgpr(tmpSgpr,self.laneSGPRCount), + comment="offset = (%s) ? offset(v%u) : edge(v%u)"%(cmpCommentText, vSrc+l, edge))) + self.vgprPool.checkIn(edge) + + return module + + ############################################################################## + # Global Read Addresses: Final Offsets A/B + ############################################################################## + def graFinalOffsets(self, kernel, tP): + module = Module("graFinalOffsets") + tc = tP["tensorChar"] + tmp = self.vgprPool.checkOut(3, "tmp", self.preventVgprOverflowDuringNewTile) + graIdx = 0 + swapPerpPara = (((tc=="A" and kernel["DirectToVgprA"]) or (tc=="B" and kernel["DirectToVgprB"])) \ + and (not tP["tlu"]) and tP["nrp"] > 1) + + if not swapPerpPara: + for perp in range(0, tP["nrp"]): + for sPerp in range(0, tP["nrpv"]): + for para in range(0, tP["nrc"]): + for sPara in range(0, tP["nrcv"]//tP["nrcvpi"]): + # single loop + singleModule, graIdx = self.graFinalOffsetsSingleLoop(kernel, tP, tc, tmp, graIdx, perp, sPerp, para, sPara) + module.add(singleModule) + else: + # swap para and perp + for para in range(0, tP["nrc"]): + for sPara in range(0, tP["nrcv"]//tP["nrcvpi"]): + for perp in range(0, tP["nrp"]): + for sPerp in range(0, tP["nrpv"]): + # single loop + singleStr, graIdx = self.graFinalOffsetsSingleLoop(kernel, tP, tc, tmp, graIdx, perp, sPerp, para, sPara) + module.add(singleModule) + + if tP["vgprTileOffsetsCheckOut"]: + self.vgprPool.checkIn(tP["vgprTileOffsets"]) + tP["vgprTileOffsets"] = None + tP["vgprTileOffsetsCheckOut"] = False + # _UseSgprForGRO uses same vgpr for ureg and tP["gpr"]["unrollOffsets"] so + # let checkin(ureg) do the checkin + # vgprTileOffsets is renamed version of treg/lwo so checkin here + + if not kernel["_UseSgprForGRO"]: + self.vgprPool.checkIn(tP["gpr"]["unrollOffsets"]) + tP["gpr"]["unrollOffsets"] = None + + if tP["vgprPackedOffsets"] != None: + self.vgprPool.checkIn(tP["vgprPackedOffsets"]) + tP["vgprPackedOffsets"] = None + + self.vgprPool.checkIn(tmp) + #if tP["isB"]: + # module.add(self.getBomb(0x100)) + + return Module("Global Read Addresses: Final Offsets A/B (Empty)") if self.dontAppendCode else module + + ############################################################################## + # Global Read Addresses: Final Offsets A/B (single loop) + ############################################################################## + def graFinalOffsetsSingleLoop(self, kernel, tP, tc, tmp, graIdx, perp, sPerp, para, sPara): + module = Module("graFinalOffsetsSingleLoop") + problemType = kernel["ProblemType"] + tVW = 1 + tVS = 0 + uVW = 1 + uVS = 0 + # single loop start + + # vgpr assignments + if tP["tlu"]: + vgprTile = tP["vgprTileOffsets"] + para*tVW + sPara*tVS + vgprUnroll = tP["gpr"]["unrollOffsets"] + perp*uVW + sPerp*uVS + else: + vgprTile = tP["vgprTileOffsets"] + perp*tVW + sPara*tVS + vgprUnroll = tP["gpr"]["unrollOffsets"] + para*uVW + sPerp*uVS + + if graIdx==0 or not kernel["_UseSgprForGRO"]: + # emit global offset macro + # TODO -refactor this and macro def to pass all indices, use the ones we need + if kernel["BufferLoad"]: + bfName = "GLOBAL_OFFSET_%s" % tP["tensorChar"] + bfArgs = ["vgprGlobalReadOffset%s+%u"%(tP["tensorChar"], graIdx)] + else: + bfName = "GLOBAL_OFFSET_%s" % tP["tensorChar"] + bfArgs = ["vgprGlobalReadAddr%s+%u"%(tP["tensorChar"], graIdx)] + packedIter = 0 #iterator through ia + iaToGpr = [None] * problemType["TotalIndices"] + for i in tP["ia"]: + if i < problemType["NumIndicesC"]: + if i == tP["tileIdx"]: + iaToGpr[i] = vgprTile + bfArgs.append( "%2u" % iaToGpr[i] ) + else: + if isPackedIndex(kernel,i): + iaToGpr[i] = tP["vgprPackedOffsets"] + \ + (vgprTile-tP["vgprTileOffsets"])*(len(tP["PackedIndices"])-1) + \ + packedIter + bfArgs.append( "%2u" % (iaToGpr[i]) ) + packedIter += 1 + else: + # just a group index + if not kernel["BufferLoad"]: # buffer load adds these to SRD not the GLOBAL_OFFSET here + bfArgs.append( "sgprWorkGroup%u"%i ) + else: # summation index + if i == problemType["IndexUnroll"]: + iaToGpr[i] = vgprUnroll + bfArgs.append( "%2u" % iaToGpr[i] ) + # other summation indices are ignored + + bfArgs.append( "%u" % tmp ) + bfComment = "gRO%s_%u_%u_%u_%u" % (tP["tensorChar"], para, sPara, perp, sPerp) + module.add(MacroInstruction(name=bfName, args=bfArgs, comment=bfComment)) + + with self.allocTmpSgpr(2) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + + # modify start + if (not kernel["_UseSgprForGRO"]) and kernel["DirectToLds%s"%tc] and kernel["UseInstOffsetForGRO"]: + # add room for instruction offset + groVgpr = "GlobalReadOffset%s+%u" % (tP["tensorChar"], graIdx) + module.add(SMovB32(dst=sgpr(tmpSgpr), src=self.buff_load_inst_offset_max)) + module.add(VAddU32(dst=vgpr(groVgpr), src0=vgpr(groVgpr), src1=sgpr(tmpSgpr), comment="shift for UseInstOffsetForGRO")) + + ldsInc = (self.kernel["WavefrontSize"] if kernel["WaveSeparateGlobalRead%c"%tc] else kernel["NumThreads"]) * kernel["GlobalLoadVectorWidth%c"%tc] * tP["bpe"] + if kernel["LdsBlockSizePerPad%s"%tc] != 0: + ldsInc += (ldsInc // kernel["LdsBlockSizePerPad%s"%tc]) * kernel["LdsPad%s"%tc] * tP["bpe"] + else: + padInterval = (self.kernel["WavefrontSize"] if kernel["WaveSeparateGlobalRead%c"%tc] else kernel["NumThreads"]) * self.bpr + ldsInc += (ldsInc // padInterval) * kernel["LdsPad%s"%tc] * tP["bpe"] + + # buffer_load only support 12 bit instruction offset + # we have to increase m0 if offset is larger thant 12 bits + # so only keep 12 bit offset and subtract it on global address + # global address will add back by buffer_load instruction offset + ldsInc = (ldsInc * graIdx) % self.buff_load_inst_offset_max + if (ldsInc != 0): + module.add(SMovB32(dst=sgpr(tmpSgpr), src=ldsInc)) + module.add(VSubU32(dst=vgpr(groVgpr), src0=vgpr(groVgpr), src1=sgpr(tmpSgpr), comment="sub offset for buffer_load instoffset")) + + def computeScalarGroImpl(scalarGro): + # this needs unroll stride in some cases and free stride in others + # if we have multiple free strides - what is expected behavior? + # could just extract the first free dimension from A? + stride1 = "Stride%s%s"%(tc,self.indexChars[tP["idx"]]) + if tP["tlu"]: + tileStride = kernel[tP["lsc"]] * (para*tVW + sPara*tVS) + unrollStride = kernel[tP["lsp"]] * (perp*uVW + sPerp*uVS) + unrollSummation = [ i for i in tP["ia"] if i in problemType["IndicesSummation"] ] + strideU = "Stride%s%s"%(tc,self.indexChars[unrollSummation[-1]]) + module.add(SMulI32(dst=sgpr(scalarGro), src0=sgpr(strideU), src1=unrollStride, \ + comment="compute offset diff (scaled unrollDim)")) + if tileStride: + module.add(SAddU32(dst=sgpr(scalarGro), src0=sgpr(scalarGro), src1=tileStride, \ + comment="compute offset diff (tileDim)")) + else: + tileStride = kernel[tP["lsp"]] * (perp*tVW + sPara*tVS) + unrollStride = kernel[tP["lsc"]] * (para*uVW + sPerp*uVS) + strideF = "Stride%s%s"%(tc,self.indexChars[tP['tileIdx']]) + module.add(SMulI32(dst=sgpr(scalarGro), src0=sgpr(strideF), src1=tileStride, \ + comment="compute offset diff (scaled tileDim)")) + if unrollStride: + module.add(SAddU32(dst=sgpr(scalarGro), src0=sgpr(scalarGro), src1=unrollStride, \ + comment="compute offset diff (unrollDim)")) + + # Using offsets so GRO holds a byte offset not an element offset + # So scale here before comparison: + module.add(SLShiftLeftB32( + dst=sgpr(scalarGro), \ + src=sgpr(scalarGro), \ + shiftHex=hex(log2(tP["bpe"])), \ + comment="scalar offset *= bytes/element")) + + if kernel["DirectToLds%s"%tc] and kernel["UseInstOffsetForGRO"]: + # add room for instruction offset + module.add(SAddU32(dst=sgpr(scalarGro), src0=sgpr(scalarGro), src1=self.buff_load_inst_offset_max, comment="shift for UseInstOffsetForGRO")) + + ldsInc = (self.kernel["WavefrontSize"] if kernel["WaveSeparateGlobalRead%c"%tc] else kernel["NumThreads"]) * kernel["GlobalLoadVectorWidth%c"%tc] * tP["bpe"] + if kernel["LdsBlockSizePerPad%s"%tc] != 0: + ldsInc += (ldsInc // kernel["LdsBlockSizePerPad%s"%tc]) * kernel["LdsPad%s"%tc] * tP["bpe"] + else: + padInterval = (self.kernel["WavefrontSize"] if kernel["WaveSeparateGlobalRead%c"%tc] else kernel["NumThreads"]) * self.bpr + ldsInc += (ldsInc // padInterval) * kernel["LdsPad%s"%tc] * tP["bpe"] + + # buffer_load only support 12 bit instruction offset + # we have to increase m0 if offset is larger thant 12 bits + # so only keep 12 bit offset and subtract it on global address + # global address will add back by buffer_load instruction offset + ldsInc = (ldsInc * graIdx) % self.buff_load_inst_offset_max + if (ldsInc != 0): + module.add(SSubU32(dst=sgpr(scalarGro), src0=sgpr(scalarGro), src1=ldsInc, comment="sub offset for buffer_load instoffset")) + + if self.checkGRO: + # Debug mode to verify that the computed offsets are offset by the expected scalar + print(tc, "tileStride=", tileStride, "unrollStride=", unrollStride, \ + "stride=%s"%(stride1)) + + module.add(self.getVectorDiffAssert(vgpr("GlobalReadOffset%s+%u"%(tc,0)), \ + vgpr("GlobalReadOffset%s+%u"%(tc,graIdx)), \ + sgpr(scalarGro))) + + needFirstSgprOffset = kernel["DirectToLds%s"%tc] and kernel["UseInstOffsetForGRO"] + + if (kernel["_UseSgprForGRO"] or self.checkGRO) and (needFirstSgprOffset or graIdx > 0): + # compute offsets for scalar global read offsets: + if kernel["_UseSgprForGRO"]: + tmpIdx = graIdx if needFirstSgprOffset else graIdx-1 + scalarGro = "ScalarGlobalReadOffset%s+%u"%(tc, tmpIdx) + computeScalarGroImpl(scalarGro) + else: + # TODO: need for investagation for replacing by allocTmpSgpr + with self.allocTmpSgpr(1) as tmpSgprInfo: + scalarGro = tmpSgprInfo.idx + computeScalarGroImpl(scalarGro) + + # dump final offsets + # BufferLoad flavor: + #if tP["isA"]: + # module.add(self.dump(vgpr("GlobalReadOffset%s+%u+0"%(tP["tensorChar"], graIdx)))) + # Flat load flavor: + #module.add(dump(vgpr("GlobalReadAddr%s+%u+0"%(tP["tensorChar"], graIdx)))) + #module.add(dump(vgpr("GlobalReadAddr%s+%u+1"%(tP["tensorChar"], graIdx)))) + graIdx += self.rpgo if kernel["BufferLoad"] else self.rpga + + return module, graIdx + + ############################################################################## + # Add the constant offsets to the specified srd. + # Srd is set to point to the base of the tile. All offsets except lowest-order + # 2d dims are computed into the SRD. + # GRO are offset from the tile SRD and the first GRO will be 0 + # Only called for BufferLoad=1 (or eventually BufferStore=1) + ############################################################################## + def computeLoadSrd(self, kernel, tP, tc, indices, bpe): + module = Module("computeLoadSrd") + + with self.allocTmpSgpr(2 + 2 + 1) as tmpSgprInfo: + stmp = tmpSgprInfo.idx + tileStart = stmp+2 + wroteTileStart = False + #--- + # Compute tileStart #elements from the 2D array start + # Add tile (and unroll if GSU) component into SRD - SRD will point to beginning of the macro-tile: + if self.groOffsetInMacroTile: + # packed modes can't use this mode, and code here assumes 1 index. + assert(len(kernel["PackedC0IndicesX"])==1) + assert(len(kernel["PackedC1IndicesX"])==1) + + wroteTileStart = True + #tP['ia'][1] + + # This is guaranteed to fit in 32-bit since the WG*MT is a number of elements in some unsigned direction: + module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tileStart+0), sgpr(tileStart+1), sgpr(tP["wg"]), kernel[tP["mt"]], "WorkGroup[01] * MT")) + strideF = self.strideRef(tc, tP['tileIdx']) + if not self.isConstUnitStride(strideF): + module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tileStart), sgpr(tileStart+1), sgpr(tileStart+0), \ + strideF, "tlu=0, scaled tile-offset by stride")) + + if kernel["GlobalSplitU"] > 1: + module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(stmp+0), sgpr(stmp+1), kernel["DepthU"], sgpr("GSUSumIdx"), "gsuOffset = DepthU*bpe*GSUSumIdx")) + unrollSummation = [ i for i in tP["ia"] if i in kernel["ProblemType"]["IndicesSummation"] ] + stride = self.strideRef(tc,unrollSummation[-1]) + if tP["tlu"] and not self.isConstUnitStride(stride): + # non-transpose case, unroll is in perp dim and should be scaled by unroll Stride + module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(stmp), sgpr(stmp+1), sgpr(stmp+0), \ + stride, "tlu=1, scaled unroll-offset by stride")) + + module.add(SAddU32(dst=sgpr(tileStart+0), src0=sgpr(tileStart+0), src1=sgpr(stmp+0), comment="accum GsuOffset term to tilestart")) + module.add(SAddCU32(dst=sgpr(tileStart+1), src0=sgpr(tileStart+1), src1=sgpr(stmp+1), comment="accum GsuOffset term to tilestart")) + + # Output : tileStart[0:1] have offset in elements from the 2D start of the tile. + # if groOffsetInMacroTile=1, 2DStart + tileStart gives the the start of the macro-tile; + # This is used to compute the limit. + # Later we modify tileStart to include batch and higher-order dims and add this to SRD. + + #--- + # Compute BUFFER Limit: + prePad = self.srdShiftLeft[tc] * tP["bpe"] # leave room in case we have to pointer shift + + if not wroteTileStart: + module.add(SMovB64(dst=sgpr(tileStart, 2), src=0, comment="set default tileStart")) + + if self.use64bShadowLimit: + limitTmp0 = "ShadowLimit%s+0"%tc + limitTmp1 = "ShadowLimit%s+1"%tc + else: + limitTmp0 = stmp+0 + limitTmp1 = stmp+1 + + module.add(SSubU32(dst=sgpr(limitTmp0), src0=sgpr("Tensor2dSize%s"%tc), src1=sgpr(tileStart+0), comment="sub tileStart")) + module.add(SSubBU32(dst=sgpr(limitTmp1), src0=sgpr("Tensor2dSize%s+1"%tc), src1=sgpr(tileStart+1), comment="sub tileStart")) + + if self.use64bShadowLimit: + # Set initial buffer limit + # if the limit is >64bit, incrementSrd decrements the shadow as the SRD increments, + # and when we get within 32-bit we start to step down the SRD + # if the limit is <32bits, set it accurately here: + # Note lshl_b64 the higher-numbered SGPR has the upper 32-bits + module.add(SLShiftLeftB64(dst=sgpr("ShadowLimit%s"%tc,2), src=sgpr("ShadowLimit%s"%tc,2), \ + shiftHex=hex(log2(tP["bpe"])), comment="Set limit to use bytes")) + if prePad: + module.add(SAddU32(dst=sgpr("ShadowLimit%s+0"%tc), src0=sgpr("ShadowLimit%s+0"%tc), src1=prePad, comment="extend limit for pre-pad")) + module.add(SAddCU32(dst=sgpr("ShadowLimit%s+1"%tc), src0=sgpr("ShadowLimit%s+1"%tc), src1=0, comment="extend limit for pre-pad")) + + if kernel["DirectToLds%s"%tc] and kernel["UseInstOffsetForGRO"]: + module.add(SAddU32(dst=sgpr("ShadowLimit%s+0"%tc), src0=sgpr("ShadowLimit%s+0"%tc), src1=self.buff_load_inst_offset_max, comment="extend limit for directToLDS instruction offset")) + module.add(SAddCU32(dst=sgpr("ShadowLimit%s+1"%tc), src0=sgpr("ShadowLimit%s+1"%tc), src1=0, comment="extend limit for directToLDS instruction offset")) + + module.add(SCmpEQU32(src0=sgpr("ShadowLimit%s+1"%tc), src1=0, comment="are we within 2^32?")) + module.add(SCSelectB32(dst=sgpr("Srd%s+2"%tc), src0=sgpr("ShadowLimit%s+0"%tc), src1="BufferLimit", comment="Move shadow to real if we are within 2^32")) + else: + # put limit directly into SRD: + module.add(SLShiftLeftB32(dst=sgpr("Srd%s+2"%tc), src=sgpr(stmp+0), shiftHex=hex(log2(tP["bpe"])), comment="Set limit to use bytes")) + module.add(SAddU32(dst=sgpr("Srd%s+2"%tc), src0=sgpr("Srd%s+2"%tc), src1=prePad, comment="extend limit for pre-pad")) + + # Apply any high-order address components to the tileStart and eventually the SRD - batch idx for batched gemm + if kernel["ProblemType"]["StridedBatched"]: + numDim = len(indices) + wg=2 # TODO - refactor since only WG2 is supported and this is always batch + for i in range(1, numDim): + idx = indices[i] + if idx == kernel["ProblemType"]["Index0"] \ + or idx == kernel["ProblemType"]["Index1"] \ + or idx in kernel["ProblemType"]["IndicesSummation"] \ + or isPackedIndex(kernel, idx): + continue # these will be captured in GRO not the SRD (or other summations are always 0) + else: + assert(wg==2) # can only have one wg2 with a batch. Other dimensions should be packed into wg0/wg1 + stride = "Stride%s%s"%(tc,self.indexChars[tP['ia'][i]]) + if not wroteTileStart: + module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tileStart+0), sgpr(tileStart+1), sgpr(stride), sgpr("WorkGroup2"), "Stride*WG")) + wroteTileStart = True + else: + module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(stmp+0), sgpr(stmp+1), sgpr(stride), sgpr("WorkGroup2"), "Stride*WG")) + module.add(SAddU32(dst=sgpr(tileStart+0), src0=sgpr(tileStart+0), src1=sgpr(stmp+0), comment="accum wg term to tilestart")) + module.add(SAddCU32(dst=sgpr(tileStart+1), src0=sgpr(tileStart+1), src1=sgpr(stmp+1), comment="accum wg term to tilestart")) + wg+=1 + + # Add the tile start to the SRD + if wroteTileStart: + module.add(scalarStaticMultiply(sgpr(tileStart,2), sgpr(tileStart,2), bpe, None, "tileStart *= BPE")) + module.add(SAddU32(dst=sgpr("Srd%s+0"%tc), src0=sgpr("Address%s+0"%tc), src1=sgpr(tileStart+0), comment="SRD base = Address+ tileStart0")) + module.add(SAddCU32(dst=sgpr("Srd%s+1"%tc), src0=sgpr("Address%s+1"%tc), src1=sgpr(tileStart+1), comment="SRD base = Address+ tileStart1")) + else: + module.add(SMovB64(dst=sgpr("Srd%s"%tc, 2), src=sgpr("Address%s"%tc, 2), comment="init SRD base address")) + + # self.groOffsetInMacroTile == 1 case, pre-pad is already subtracted from AddressA/B + if prePad and self.groOffsetInMacroTile == 0: + module.add(SSubU32(dst=sgpr("Srd%s+0"%tc), src0=sgpr("Srd%s+0"%tc), src1=prePad, comment="pre-pad to make room for possible pointer shift")) + module.add(SSubBU32(dst=sgpr("Srd%s+1"%tc), src0=sgpr("Srd%s+1"%tc), src1=0, comment="pre-pad to make room for possible pointer shift")) + + if kernel["DirectToLds%s"%tc] and kernel["UseInstOffsetForGRO"]: + module.add(SSubU32(dst=sgpr("Srd%s+0"%tc), src0=sgpr("Srd%s+0"%tc), src1=self.buff_load_inst_offset_max, comment="make room for directToLDS instruction offset")) + module.add(SSubBU32(dst=sgpr("Srd%s+1"%tc), src0=sgpr("Srd%s+1"%tc), src1=0, comment="make room for directToLDS instruction offset")) + + module.add(SMovB32(dst=sgpr("Srd%s+3"%tc), src="Srd127_96", comment="Set bits 127_96 in SRD")) + + #if tP["isB"]: + # module.add(self.getCmpAssert(self.asmAssert.ne, sgpr("WorkGroup1"), 0xA)) + + return module + + ############################################################################## + # Global Read Addresses: Addresses A/B + ############################################################################## + def graAddresses(self, kernel, tP): + module = Module("graAddresses") + tc = tP["tensorChar"] + graIdx = 0 + + if kernel["BufferLoad"]: + # maxAddrSgpr = size[n] * stride[n-1] + module.addComment0("max read offset = size[n] * stride[n-1]") + + module.add(self.computeLoadSrd(kernel, tP, tc, kernel["ProblemType"]["IndexAssignments%s"%tc], tP["bpe"])) + + #module.add(self.getBomb(0x13)) # after addresses and SRD set + else: + tmp = self.vgprPool.checkOut(2, "tmp", self.preventVgprOverflowDuringNewTile) + module.add(VMovB32(dst=vgpr(tmp+0), src=sgpr("Address%s+0"%tP["tensorChar"]))) + module.add(VMovB32(dst=vgpr(tmp+1), src=sgpr("Address%s+1"%tP["tensorChar"]))) + for perp in range(0, tP["nrp"]): + for sPerp in range(0, tP["nrpv"]): + for para in range(0, tP["nrc"]): + for sPara in range(0, tP["nrcv"]//tP["nrcvpi"]): + + comment = "gRA%s_%u_%u_%u_%u = addr%s+grO%s_%u_%u_%u_%u" \ + % (tP["tensorChar"], para, sPara, perp, sPerp, \ + tP["tensorChar"], tP["tensorChar"], \ + para, sPara, perp, sPerp ) + module.add(VAddCOU32( + dst=vgpr("GlobalReadAddr%s+%u+0"%(tP["tensorChar"], graIdx)), \ + dst1=VCC(), \ + src0=vgpr("GlobalReadAddr%s+%u+0"%(tP["tensorChar"], graIdx)), \ + src1=vgpr(tmp+0), \ + comment=comment+" (lower)")) + module.add(VAddCCOU32( + dst=vgpr("GlobalReadAddr%s+%u+1"%(tP["tensorChar"], graIdx)), \ + dst1=VCC(), \ + src0=vgpr("GlobalReadAddr%s+%u+1"%(tP["tensorChar"], graIdx)), \ + src1=vgpr(tmp+1), \ + src2=VCC(), \ + comment=comment+" (upper)")) + #module.add(dump(vgpr("GlobalReadAddr%s+%u+0"%(tP["tensorChar"], graIdx)))) + #module.add(dump(vgpr("GlobalReadAddr%s+%u+1"%(tP["tensorChar"], graIdx)))) + graIdx += self.rpga + #module.add(SEndpgm()) + self.vgprPool.checkIn(tmp) + + return module + + ############################################################################## + # Global Read Addresses: Increments + # Define graIncrements, called once for each summation + ############################################################################## + def graIncrements(self, kernel, loopIdx, tP): + module = Module("graIncrements") + tc = tP["tensorChar"] + + dimIdx = kernel["ProblemType"]["IndicesSummation"][loopIdx] # dimension index + loopChar = self.indexChars[dimIdx] + + stride = self.strideRef(tc, dimIdx) + isMirrorIdx = dimIdx in kernel["ProblemType"]["MirrorDims%s"%tc] + + #print (tc, ": loopIdx=", loopIdx, "dimIdx=", dimIdx, "strideIdx=", strideIdx) + + gsu = 1 + if kernel["GlobalSplitU"] > 1: + gsu = kernel["GlobalSplitU"] + + assert(self.unrollIdx == kernel["ProblemType"]["NumIndicesSummation"]-1) + if loopIdx==self.unrollIdx: + if self.globalReadIncsUseVgpr: + with self.allocTmpSgpr(2) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + module.add(SMulI32(dst=sgpr(tmpSgpr+0), \ + src0="DepthU*%d"%(gsu*tP["bpe"]), src1=stride, \ + comment="incr%s%s = %s*DepthU*bpe (unrollIdx)"%(tc, loopChar, stride) )) + # TODO - this should be mul-H?? + module.add(SMovB32( + dst=sgpr(tmpSgpr+1), \ + src=hex(0), \ + comment="(carry)")) + module.add(VMovB32( + dst=vgpr("GlobalReadIncs%s+%u+0"%(tc, 2*loopIdx)), \ + src=sgpr(tmpSgpr+0))) + module.add(VMovB32( + dst=vgpr("GlobalReadIncs%s+%u+1"%(tc, 2*loopIdx)), \ + src=sgpr(tmpSgpr+1))) + else: # not globalReadIncsUseVgpr, ie use SGPR + + m = "DepthU*Bpe%s"%(tc) + if gsu>1: + m += "*%d"%gsu + + if isMirrorIdx: + m = "-%s"%(m) + + # multiply by stride, optimizing if unit stride + if self.isConstUnitStride(stride): + module.add(SMovB32(dst=sgpr("GlobalReadIncs%s+%u"%(tc, loopIdx)), src=m, \ + comment="incr%s (unrollIdx)"%(tc) )) + else: + module.add(SMulI32(dst=sgpr("GlobalReadIncs%s+%u"%(tc, loopIdx)), \ + src0=m, src1=stride, \ + comment="incr%s unrollIdx)"%(tc) )) + else: + # other summation + if self.globalReadIncsUseVgpr: + printExit("NumIndicesSummation=%u not yet supported in assembly unless globalReadIncsUseVgpr==0" \ + % kernel["ProblemType"]["NumIndicesSummation"] ) + else: + graInc = "GlobalReadIncs%s+%u"%(tc, loopIdx) + # subtract increments done by the inner iterations + # may be negative: + loopIdxPrev = loopIdx + 1 + dimIdxPrev = kernel["ProblemType"]["IndicesSummation"][loopIdxPrev] # dimension index + loopCharPrev = self.indexChars[dimIdxPrev] + stridePrev = self.strideRef(tc, dimIdxPrev) + isMirrorIdxPrev = dimIdxPrev in kernel["ProblemType"]["MirrorDims%s"%tc] + + module.addComment1("compute globalReadInc for higher-level loop") + + with self.allocTmpSgpr(3) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + # Summations always appear in both A and B, can compute number of iterations just once: + if loopIdxPrev==self.unrollIdx: + loopCounterName= self.loopCounterName(kernel, self.unrollIdx) + if tP["isA"]: + quotient = loopCounterName + dividend = "SizesSum+%u"%self.unrollIdx + divisor = kernel["DepthU"] + if self.noTailLoop and kernel["AssertSummationElementMultiple"] % kernel["DepthU"] != 0: + # round up SizesSum/DepthU for noTailLoop case + module.add(SAddI32(dst=sgpr(quotient), src0=(divisor - 1), src1=sgpr(dividend), \ + comment="round up SizeSum / DepthU" )) + module.add(scalarStaticDivideAndRemainder(quotient, None, quotient, \ + divisor, tmpSgprInfo, 0)) + else: + module.add(scalarStaticDivideAndRemainder(quotient, None, dividend, \ + divisor, tmpSgprInfo, 0)) + + if kernel["GlobalSplitU"] > 1: + module.add(self.calculateLoopNumIterGsu(kernel, loopCounterName, tmpSgprInfo)) + + module.add(SMulI32(dst=sgpr(loopCounterName), src0=sgpr(loopCounterName), \ + src1=kernel["GlobalSplitU"]*kernel["DepthU"], \ + comment="=loopCounterName*DepthU")) + module.add(SMulI32(dst=sgpr(graInc), src0=stridePrev, src1=sgpr(loopCounterName), \ + comment="tmp <- stride%s%s * myWgUnrollIters" %(tc, loopCharPrev))) + else: + module.add(SMulI32(dst=sgpr(graInc), src0=stridePrev, src1=self.sizeRef(dimIdxPrev), \ + comment="tmp <- stride%s%s * size%s%s" %(tc, loopCharPrev, tc, loopCharPrev))) + + # subtract amount that previous inner loop will have already incremented: + # graInc is used as temp for the prev loop calc + if isMirrorIdx and isMirrorIdxPrev: + module.add(SSubI32(dst=sgpr(graInc), \ + src0=sgpr(graInc), \ + src1=stride, \ + comment="incr%s%s = - stride%s%s"%(tc, loopChar, tc, loopChar) )) + elif isMirrorIdx: + module.add(SAddI32(dst=sgpr(graInc), \ + src0=stride, \ + src1=sgpr(graInc), \ + comment="incr%s%s = stride%s%s + "%(tc, loopChar, tc, loopChar) )) + module.add(SSubI32(dst=sgpr(graInc), \ + src0=0, \ + src1=sgpr(graInc), \ + comment="incr%s%s = - (stride%s%s + )"%(tc, loopChar, tc, loopChar) )) + elif isMirrorIdxPrev: + module.add(SAddI32(dst=sgpr(graInc), \ + src0=stride, \ + src1=sgpr(graInc), \ + comment="incr%s%s = stride%s%s + "%(tc, loopChar, tc, loopChar) )) + else: + module.add(SSubI32(dst=sgpr(graInc), \ + src0=stride, \ + src1=sgpr(graInc), \ + comment="incr%s%s = stride%s%s - "%(tc, loopChar, tc, loopChar) )) + + module.add(SLShiftLeftB32( + dst=sgpr(graInc), \ + src=sgpr(graInc), \ + shiftHex="Bpe%sLog2"%tc, + comment="<- scale by bpe")) + + if 0 and tP["isB"] and loopIdx==0: + module.add(self.getBomb()) + #module.add(self.getCmpAssert(self.asmAssert.ne, sgpr("WorkGroup1"),0)) + + #module.add(dump(vgpr("GlobalReadIncs%s"%tP["tensorChar"]))) + #module.add(SEndpgm()) + #if tP["isB"]: + # module.add(self.getBomb(0x100)) + return Module("graIncrements (Empty)") if self.dontAppendCode else module + + ############################################################################## + # Local Write Addresses: Tile Assignment A/B + ############################################################################## + def lwaTileAssignment(self, tP): + module = Module("lwaTileAssignment") + module.addComment0("lwaTileAssignment%s = %s" % (tP["tensorChar"], \ + vgpr(tP["gpr"]["lwoT"]))) + return module + + ############################################################################## + # Local Write Addresses: Unroll Assignment A/B + ############################################################################## + def lwaUnrollAssignment(self, kernel, tP): + module = Module("lwaUnrollAssignment") + uReg = tP["gpr"]["uReg2" if kernel["GlobalSplitU"] > 1 else "uReg"] + module.addComment0("lwaUnrollAssignment%s = %s" % (tP["tensorChar"], vgpr(uReg))) + if kernel.enabledSplitLDS and kernel["UnrollMajorLDS%s" % tP["tensorChar"]]: + if self.inTailLoop: + subIterReg = self.vgprPool.checkOut(1, "subIterReg") + module.addComment0("Each wg writes 1/%u of G2L data to LDS"%kernel["DepthULdsDivisor"]) + module.add(VLShiftRightB32(dst=vgpr(subIterReg), shiftHex=log2(kernel["_DepthULds"]), src=vgpr(uReg), comment="sub_G2L_idx = uIdx / DepthU_Compute")) + module.add(VAndB32(dst=vgpr(uReg), src0=vgpr(uReg), src1=(kernel["_DepthULds"]-1), comment="unrollIdx = unrollIdx % DepthU_Compute")) + tP["gpr"]["subIterReg"] = subIterReg + else: + module.addComment0("Each thd writes 1/%u of G2L data to LDS"%kernel["DepthULdsDivisor"]) + module.add(VLShiftRightB32(dst=vgpr(uReg), shiftHex=log2(kernel["DepthULdsDivisor"]), src=vgpr(uReg), comment="sub_G2L_idx = uIdx / DepthULdsDivisor")) + return module + + ############################################################################## + # Local Write Addresses: First Offset A/B + # uDu: which part of G2L buffer to write to LDS + ############################################################################## + def lwaFirstOffset(self, kernel, tP, uDu=0): + module = Module("lwaFirstOffset") + tc = tP["tensorChar"] + LdsPad = kernel["LdsPad%s"%tc] if kernel["LdsBlockSizePerPad%s"%tc] == 0 else 0 + #"lwFOA = lwA%s + lwA%s*MT%s" \ + # % (tP["tileChar"], self.unrollChar, tP["tileChar"]) + uReg = tP["gpr"]["uReg2" if kernel["GlobalSplitU"] > 1 else "uReg"] + if kernel["LocalWriteUseSgpr%s"%tc]: + destVgpr = self.vgprPool.checkOut(1, "destVgpr", self.preventVgprOverflowDuringNewTile) + else: + destVgpr = "LocalWriteAddr%s"%tc + + if kernel["UnrollMajorLDS%s" % tc]: + lds_stride = kernel["_DepthULds"] + LdsPad + module.add(VMulU32U24(dst=vgpr(destVgpr), src0=hex(lds_stride), src1=vgpr(tP["gpr"]["lwoT"]), \ + comment="lw%s%s**(DepthU_Compute + PAD)"%(tP["tensorChar"], self.unrollChar))) + module.add(VAddLShiftLeftU32(dst=vgpr(destVgpr), src0=vgpr(uReg), src1=vgpr(destVgpr), shiftHex=hex(log2(tP["bpe"])), \ + comment="lwFO%s = (lw%s%s + lw%s%s*(DepthU+PAD))*bpe" % (tc, tc, tc, tc, self.unrollChar) )) + else: + lds_stride = kernel["MacroTile%s"%tP["tensorChar"]] + LdsPad + module.add(VMulU32U24(dst=vgpr(destVgpr), src0=hex(lds_stride), src1=vgpr(uReg), \ + comment="lw%s%s**(MT%s + PAD)"%(tP["tensorChar"], self.unrollChar, tP["tensorChar"]))) + module.add(VAddLShiftLeftU32(dst=vgpr(destVgpr), src0=vgpr(tP["gpr"]["lwoT"]), src1=vgpr(destVgpr), shiftHex=hex(log2(tP["bpe"])), \ + comment="lwFO%s = (lw%s%s + lw%s%s*(MT%s+PAD))*bpe" % (tc, tc, tc, tc, self.unrollChar, tP["tileChar"]) )) + + # LdsBlockSizePerPad: add padding + if kernel["LdsBlockSizePerPad%s"%tc] != 0 and kernel["LdsPad%s"%tc] != 0: + tmpVgpr = self.vgprPool.checkOut(2) + tmpVgprRes = RegisterPoolResource(tmpVgpr, 2) + module.add(vectorStaticDivide(uReg, destVgpr, kernel["LdsBlockSizePerPad%s"%tc], tmpVgprRes, \ + "padding %u per block %u" % (kernel["LdsPad%s"%tc], kernel["LdsBlockSizePerPad%s"%tc]))) + with self.allocTmpSgpr(1) as tmpSgprInfo: + module.add(staticMultiply(vgpr(uReg), vgpr(uReg), kernel["LdsPad%s"%tc] * tP["bpe"], tmpSgprInfo, \ + "padding %u per block %u" % (kernel["LdsPad%s"%tc], kernel["LdsBlockSizePerPad%s"%tc]))) + module.add(VAddU32(dst=vgpr(destVgpr), src0=vgpr(uReg), src1=vgpr(destVgpr), \ + comment="add padding %u per block %u" % (kernel["LdsPad%s"%tc], kernel["LdsBlockSizePerPad%s"%tc]))) + self.vgprPool.checkIn(tmpVgpr) + + if tP["isB"]: + if kernel["LdsOffsetB"] != 0: # LdsOffsetB can be 0 if DirectToVgprA is enabled + module.add(VAddCOU32( + dst=vgpr(destVgpr), \ + dst1=VCC(), \ + src0=hex(kernel["LdsOffsetB"]*tP["bpe"]), \ + src1=vgpr(destVgpr), \ + comment="lwFOB = lwB%s + lwB%s*MT%s + LDS_OFFSET_B=%u*%u" % (tP["tileChar"], \ + self.unrollChar, tP["tileChar"], kernel["LdsOffsetB"], self.bpeAB) )) + + self.vgprPool.checkIn(tP["gpr"]["lwoT"]) + tP["gpr"]["lwoT"] = None + if kernel["GlobalSplitU"] > 1: + self.vgprPool.checkIn(tP["gpr"]["uReg2"]) + tP["gpr"]["uReg2"] = None + #LSC_ * LSP_ + numBytesPerElement = kernel["ProblemType"]["DataType"].numBytes() + validWIPerLoad = kernel[tP["lsc"]] * kernel[tP["lsp"]]//tP["glvw"] + validBytesPerLoad = kernel[tP["lsc"]] * kernel[tP["lsp"]] * numBytesPerElement + maxBytesPerLoad = kernel["NumThreads"] * tP["glvw"] * numBytesPerElement + + if kernel["WaveSeparateGlobalRead%s"%tc]: + validBytesPerLoad *= (kernel["NumThreads"] // self.kernel["WavefrontSize"]) + + assert (validBytesPerLoad <= maxBytesPerLoad) + assert (kernel[tP["lsc"]] * kernel[tP["lsp"]] % tP["glvw"] == 0) + + if validBytesPerLoad != maxBytesPerLoad: + with self.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + module.add(SMovB32(dst=sgpr(tmpSgpr), src=validWIPerLoad, \ + comment="lsc*lsp=%u*%u"%(kernel[tP["lsc"]],kernel[tP["lsp"]] ))) + module.add(VCmpLtU32( + dst=VCC(), \ + src0=vgpr("Serial"), \ + src1=sgpr(tmpSgpr), \ + comment="fractional: ensure tid < global read tile elements")) + tmpVgpr = self.vgprPool.checkOut(1, "tmpVgpr", self.preventVgprOverflowDuringNewTile) + module.add(VMovB32(dst=vgpr(tmpVgpr), src=hex(self.LdsOOB))) + module.add(VCndMaskB32( + dst=vgpr(destVgpr), \ + src0=vgpr(tmpVgpr), \ + src1=vgpr(destVgpr), \ + comment="Mask load so out-of-gr-tile bounds returns 0")) + self.vgprPool.checkIn(tmpVgpr) + + elif self.inTailLoop and kernel.enabledSplitLDS: # where (DepthU for global read) != (DepthU for compute) + # only for TN tensor + TN lds layout + assert tP["tlu"] == 0 + module.add(VCmpEQU32(dst=VCC(), src0=vgpr(tP["gpr"]["subIterReg"]), src1=uDu, comment="if sub_g2l_idx == %u ?"%uDu)) + + ldsOOB = self.vgprPool.checkOut(1, "lds OOB addr", self.preventVgprOverflowDuringNewTile) + module.add(VMovB32(dst=vgpr(ldsOOB), src=hex(self.LdsOOB), comment="lds OOB address")) + module.add(VCndMaskB32( \ + dst=vgpr(destVgpr), \ + src0=vgpr(ldsOOB), \ + src1=vgpr(destVgpr), \ + comment="Mask threads not belonging to current sub_g2l_idx by assigning OOB")) + self.vgprPool.checkIn(ldsOOB) + + if kernel["LocalWriteUseSgpr%s"%tc]: + # TODO: Can refactor code above to Compute this directly: + module.add(VReadfirstlaneB32( + dst=sgpr("LocalWriteAddr%s"%tc), \ + src=vgpr(destVgpr), \ + comment="Copy lds write address VGPR to SGPR")) + self.vgprPool.checkIn(destVgpr) + + self.vgprPool.checkIn(tP["gpr"]["uReg"]) + tP["gpr"]["uReg"] = None + if "subIterReg" in tP["gpr"]: + if tP["gpr"]["subIterReg"] is not None: + self.vgprPool.checkIn(tP["gpr"]["subIterReg"]) + tP["gpr"]["subIterReg"] = None + # dump lds write offsets + #if tP["isA"]: + #module.add(self.dump(vgpr("LocalWriteAddr%s"%tP["tensorChar"]))) + #module.add(self.getBomb(-40)) + # do not generate local write address code if DirectToVgpr is enabled + return Module("lwaUnrollAssignment (Empty)") if self.dontAppendCode or kernel["DirectToVgpr%s"%tc] else module + + ############################################################################## + # Local Read Addresses: Tile Assignment + ############################################################################## + def lraTileAssignment(self, kernel, tPA, tPB): + module = Module("lraTileAssignment") + + component = Component.LraTileAssignment.find(self) + + tP0 = tPA if tPB["tile01Idx"] else tPB + tP1 = tPB if tPB["tile01Idx"] else tPA + + if component: + # do not generate local read code if DirectToVgpr is enabled + tc = tP0["tensorChar"] + if not kernel["DirectToVgpr%s"%tc]: + module.add(component(self, kernel, tP0)) + # do not generate local read code if DirectToVgpr is enabled + tc = tP1["tensorChar"] + if not kernel["DirectToVgpr%s"%tc]: + module.add(component(self, kernel, tP1)) + + return module + + ############################################################################## + # Local Read Addresses: Final Offset A/B + ############################################################################## + def lraFinalOffset(self, kernel, tP): + module = Module("lraFinalOffset") + + # do not generate local read code if DirectToVgpr is enabled + tc = tP["tensorChar"] + if kernel["DirectToVgpr%s"%tc]: + return module + + # allocate resources + sgid = self.vgprPool.checkOut(1) # quotient + rReg = self.vgprPool.checkOut(1) # remainder, unused here + tmpVgpr = self.vgprPool.checkOutAligned(2, 2,"tmpVgpr") + tmpVgprRes = RegisterPoolResource(tmpVgpr, 2) + + # constant + tc = tP["tensorChar"] + tile01 = tP["tile01Idx"] + LdsPad = kernel["LdsPad%s" % tc] if kernel["LdsBlockSizePerPad%s" % tc] == 0 else 0 + divisor = kernel["SubGroup0"] * kernel["SubGroup1"] + mtAddPad = kernel["MacroTile%u" % tile01] + LdsPad + + # generate instruction + module.add(vectorStaticDivide(sgid, "Serial", divisor, tmpVgprRes, \ + "LSU offset: sgid = Serial / subGroup(%u)" % divisor)) + with self.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + module.add(SMovB32(dst=sgpr(tmpSgpr), src=mtAddPad, \ + comment="LSU offset: stride = MT%u(%u) + PAD%u(%u)" % (tile01, kernel["MacroTile%u" % tile01], tile01, LdsPad))) + module.add(VMulLOU32(dst=vgpr(sgid), src0=sgpr(tmpSgpr), src1=vgpr(sgid), \ + comment="LSU offset: lsuoffset = sgid*(MT%u+PAD)"%tile01)) + if not kernel["EnableMatrixInstruction"] and kernel["VectorWidth"] > 1: + module.add(staticMultiply(vgpr(tP["gpr"]["lro"]), vgpr(tP["gpr"]["lro"]), kernel["VectorWidth"], tmpSgprInfo, \ + "Final Offset: lr%sOffset * VW" % tc)) + + # final offset + finalVgpr = vgpr("LocalReadAddr%s"%tc) + if (kernel["DirectToLds%s" % tc] and \ + kernel["GlobalLoadVectorWidth%c"%tc] * tP["bpe"] > 4): + # DirectToLds + DGEMM case + # use bpr for LSU offset instead of bpe (DirectToLds needs _ds_load_b32) + module.add(VLShiftLeftB32(dst=vgpr(sgid), shiftHex=hex(log2(self.bpr)), src=vgpr(sgid), \ + comment="LSU offset: lsuoffset = lsuoffset * bpr")) + module.add(VLShiftLeftB32(dst=vgpr(tP["gpr"]["lro"]), shiftHex=hex(log2(tP["bpe"])), src=vgpr(tP["gpr"]["lro"]), \ + comment="Final Offset: offset = (lro%s*VW)*bpe+lsuoffset*bpr" % tile01)) + module.add(VAddU32(dst=finalVgpr, src0=vgpr(sgid), src1=vgpr(tP["gpr"]["lro"]))) + # need magic offset calc here (after final offset) + # offset calculation for TLU=1 when glvw * bpe * wavefrontsize > 256 + # x2/x4 directToLds stores 8/16 bytes into LDS like below + # address offset in LDS in bytes + # DWORD# written by LDS_DMA + # address offset in LDS (byte offset) + # 0 4 8 12 16 20 24 28 32 36 40 44 48 52 56 60 + # data dword#: + # 0 4 8 12 2 6 10 14 1 5 9 13 3 7 11 15 + # Noffset calculation for VW =1 (BPe=8) / VW =2 (BPE=4) + # use direcToLds for best VW and GRVW case; other cases requires bit more lane manipulation. + # offset calculation for B might benefit from some optimization. + # offset calculation for x2/x4 is basically manipulation lane offset based on layout + tmp1 = self.vgprPool.checkOut(1,"tmp1") + tmp2 = self.vgprPool.checkOut(1,"tmp2") + if (kernel["GlobalLoadVectorWidth%s"%tc] * tP["bpe"] == 8): + # (bit2<<3) | (bit3 >>1) | (bit4>>1) | (bit5>>1) + module.add(VAndB32(dst=vgpr(tmp1), src0=hex(0x4), src1=finalVgpr, comment="magic offset calc")) + module.add(VLShiftLeftB32(dst=vgpr(tmp1), shiftHex=hex(3), src=vgpr(tmp1))) + module.add(VAndB32(dst=vgpr(tmp2), src0=hex(0x38), src1=finalVgpr)) + module.add(VLShiftRightB32(dst=vgpr(tmp2), shiftHex=hex(1), src=vgpr(tmp2))) + module.add(VOrB32(dst=vgpr(tmp1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + module.add(VAndB32(dst=finalVgpr, src0=hex(0xffffffc3), src1=finalVgpr)) + module.add(VOrB32(dst=finalVgpr, src0=finalVgpr, src1=vgpr(tmp1))) + else: #if (kernel["GlobalLoadVectorWidth%s"%tc] * tP["bpe"] == 16): # most preferred case + # (bit2<<3) | (bit3 <<1) | (bit4>>2) | (bit5>>2) + module.add(VAndB32(dst=vgpr(tmp1), src0=hex(0x4), src1=finalVgpr, comment="magic offset calc")) + module.add(VLShiftLeftB32(dst=vgpr(tmp1), shiftHex=hex(3), src=vgpr(tmp1))) + module.add(VAndB32(dst=vgpr(tmp2), src0=hex(0x8), src1=finalVgpr)) + module.add(VLShiftLeftB32(dst=vgpr(tmp2), shiftHex=hex(1), src=vgpr(tmp2))) + module.add(VOrB32(dst=vgpr(tmp1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + module.add(VAndB32(dst=vgpr(tmp2), src0=hex(0x30), src1=finalVgpr)) + module.add(VLShiftRightB32(dst=vgpr(tmp2), shiftHex=hex(2), src=vgpr(tmp2))) + module.add(VOrB32(dst=vgpr(tmp1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + module.add(VAndB32(dst=finalVgpr, src0=hex(0xffffffc3), src1=finalVgpr)) + module.add(VOrB32(dst=finalVgpr, src0=finalVgpr, src1=vgpr(tmp1))) + # TODO: cover other cases + + # another address conversion for DirectToLds + NumLoadsCoalesced > 1 + newModule, dummy = self.lraOffsetConversionForDTLandNLC(kernel, tP, offset_val=0, generateAsm=True, \ + finalVgpr=finalVgpr, tmp1=tmp1, tmp2=tmp2) + module.add(newModule) + + self.vgprPool.checkIn(tmp1) + self.vgprPool.checkIn(tmp2) + else: + module.add(VAddLShiftLeftU32(dst=finalVgpr, src0=vgpr(sgid), src1=vgpr(tP["gpr"]["lro"]), shiftHex=hex(log2(tP["bpe"])), \ + comment="Final Offset: offset = (lro%s*VW+lsuoffset)*bpe" % tile01 )) + + # LdsBlockSizePerPad: add padding + if kernel["LdsBlockSizePerPad%s"%tc] != 0 and kernel["LdsPad%s"%tc] !=0: + module.add(vectorStaticDivide(rReg, "LocalReadAddr%s"%tc, kernel["LdsBlockSizePerPad%s"%tc], tmpVgprRes, \ + "Final Offset: padding %u per block %u" % (kernel["LdsPad%s"%tc], kernel["LdsBlockSizePerPad%s"%tc]))) + with self.allocTmpSgpr(1) as tmpSgprInfo: + module.add(staticMultiply(vgpr(rReg), vgpr(rReg), kernel["LdsPad%s"%tc] * tP["bpe"], \ + "Final Offset: padding %u per block %u" % (kernel["LdsPad%s"%tc], kernel["LdsBlockSizePerPad%s"%tc], tmpSgprInfo))) + module.add(VAddU32(dst=vgpr("LocalReadAddr%s"%tc), src0=vgpr(rReg), src1=vgpr("LocalReadAddr%s"%tc), \ + comment="Final Offset: add padding %u per block %u" % (kernel["LdsPad%s"%tc], kernel["LdsBlockSizePerPad%s"%tc]))) + + # release resources + self.vgprPool.checkIn(tmpVgpr) + self.vgprPool.checkIn(sgid) + self.vgprPool.checkIn(rReg) + self.vgprPool.checkIn(tP["gpr"]["lro"]) + + return module + + ############################################################################## + # Local Read Addresses offset conversion for DTL + NLC > 1 + ############################################################################## + def lraOffsetConversionForDTLandNLC(self, kernel, tP, offset_val, generateAsm=False, \ + finalVgpr=None, tmp1=None, tmp2=None): + module = Module("lraOffsetConversionForDTLandNLC") + # another address conversion for DirectToLds + NumLoadsCoalesced > 1 + divisorName = tP["lvc"] + divisor = kernel[divisorName] + width = kernel["WavefrontSize"] if tP["tlu"] else kernel["DepthU"] + if divisor < width: + # DirectToLds + above conditions, rotate offset_val bits to adjust LDS offset + lowerScale = tP["nrc"] + upperScale = (kernel["WavefrontSize"] // divisor) + # bit rotation necessary only when nrc > 1 + if lowerScale > 1: + tile01 = tP["tile01Idx"] + rightShift = int(log2(lowerScale)) # assuming power of 2 + leftShift = int(log2(upperScale)) # assuming power of 2 + line = kernel["MacroTile%u" % tile01] if tP["tlu"] else kernel["DepthU"] + ldsLineSize = line * tP["bpe"] // lowerScale + maskBitsLow = (lowerScale - 1) * ldsLineSize + maskBitsHigh = (upperScale - 1) * lowerScale * ldsLineSize + maskBitsAll = (maskBitsLow | maskBitsHigh) + + # offset_val conversion + low = offset_val & maskBitsLow + high = offset_val & maskBitsHigh + low <<= leftShift + high >>= rightShift + val = low | high + offset_val = (offset_val & (~maskBitsAll)) | val + + # generate asm code + if generateAsm: + with self.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr2 = tmpSgprInfo.idx + module.add(VAndB32(dst=vgpr(tmp1), src0=hex(maskBitsLow), src1=finalVgpr, \ + comment="Offset rotation for DirectToLds + %s > 1"%tP["lsc"])) + module.add(VAndB32(dst=vgpr(tmp2), src0=hex(maskBitsHigh), src1=finalVgpr)) + module.add(VLShiftLeftB32(dst=vgpr(tmp1), shiftHex=hex(leftShift), src=vgpr(tmp1))) + module.add(VLShiftRightB32(dst=vgpr(tmp2), shiftHex=hex(rightShift), src=vgpr(tmp2))) + module.add(VOrB32(dst=vgpr(tmp1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + module.add(SMovB32(dst=sgpr(tmpSgpr2), src=hex(maskBitsAll))) + module.add(VNotB32(dst=vgpr(tmp2), src=sgpr(tmpSgpr2))) + module.add(VAndB32(dst=finalVgpr, src0=vgpr(tmp2), src1=finalVgpr)) + module.add(VOrB32(dst=finalVgpr, src0=vgpr(tmp1), src1=finalVgpr)) + + return module, offset_val + + ############################################################################## + # Local Read Addresses: Declare Addresses A/B + ############################################################################## + def lraDeclareAddresses(self, kernel, tP): + module = Module("lraDeclareAddresses") + if tP["isA"]: + module.addComment0("N/A") + + else: + # no local read code if DirectToVgpr is enabled + # no need to generate add code if LdsOffset is 0 + if kernel["DirectToVgprB"] or kernel["LdsOffset%s"%tP["tensorChar"]] == 0: + module = Module("lraDeclareAddresses (Empty)") + else: + module.add(VAddCOU32( + dst=vgpr("LocalReadAddr%s+0"%tP["tensorChar"]), \ + dst1=VCC(), \ + src0=hex(kernel["LdsOffset%s"%tP["tensorChar"]]*tP["bpe"]), \ + src1=vgpr("LocalReadAddr%s+0"%tP["tensorChar"]), \ + comment=" += LdsOffset%s (lower)"%tP["tensorChar"])) + return module + + ############################################################################## + # openShadowInit + # Label after prefetches are launched. This is present even if ShadowInit not + # used. + ############################################################################## + def openShadowInit(self): + module = Module("openShadowInit") + module.add(Label("ShadowInitStart", "")) + return module + + ############################################################################## + # closeShadowInit + # Label after prefetches are launched. This is present even if ShadowInit not + # used. + ############################################################################## + def closeShadowInit(self, kernel): + module = Module("closeShadowInit") + assert(self.doShadowInit and kernel["PrefetchGlobalRead"]) + + module.add(self.checkLastIter(kernel)) + if kernel["SuppressNoLoadLoop"]: + loopChar = self.indexChars[ \ + kernel["ProblemType"]["IndicesSummation"][self.unrollIdx]] + lastIterEnd = Label("LoopEnd%s"%loopChar, "") + else: + lastIterEnd = Label("PrefetchGlobalLastIterEnd", "") + + # This branch could potentially be very far e.g. > SIMM16 + module.addComment1("after InitC, skip to end of prefetch last iter if numIter==0") + # use positive offset only long jump + module.add(self.longBranchScc1(lastIterEnd, posNeg=1)) + + return module + + ############################################################################## + # Initialize C + ############################################################################## + def initC(self, kernel): + module = Module("initC") + module.addComment1("initC: remove C-tile %u-%u from pool"%(self.startVgprValuC, self.startVgprValuC+self.numVgprValuC)) + self.vgprPool.remove(self.startVgprValuC, self.numVgprValuC, "ValuC") + numAccvgprs = self.totalAgprs + self.agprPool.remove(0, numAccvgprs, "ValuC") + module.addComment1("initC: remove AB-tile %u-%u from pool"%(self.startVgprValuA, self.lastValuAB)) + self.vgprPool.remove(self.startVgprValuA, self.lastValuAB - self.startVgprValuA, "ValuAB") + numCVgpr = max(self.numVgprValuC, numAccvgprs) + + startNumCVgpr = 0 + if self.useInitAccVgprOpt: + # init accvgpr opt. initialize only the last set of accvgpr instead of whole accvgpr + numRegistersOut = kernel["MIRegPerOut"] + accs_per_wave = kernel["MatrixInstM"] * kernel["MatrixInstN"] * kernel["MatrixInstB"] \ + // self.kernel["WavefrontSize"] * numRegistersOut + startNumCVgpr = numCVgpr - accs_per_wave + + if kernel["LdsInitCVgprs"]: + tmpAddr = self.vgprPool.checkOut(1,"tmp vgpr for lds init C registers") + module.add(VMovB32(dst=vgpr(tmpAddr), src=self.LdsOOB, comment="set out-of-bound addr")) + + for i in range(startNumCVgpr, numCVgpr): + copyInst = VMovB32 if self.numVgprValuC else VAccvgprWrite + regStr = vgpr("ValuC+%u"%i) if self.numVgprValuC else accvgpr(i) + if not kernel["LdsInitCVgprs"]: + module.add(copyInst(dst=regStr, src=hex(0), comment="initC")) + else: + module.add(DSLoadB32(dst=regStr, src=vgpr(tmpAddr), ds=DSModifiers(offset=0), comment="initC")) + + if kernel["LdsInitCVgprs"]: + self.vgprPool.checkIn(tmpAddr) + + return module + + ############################################################################## + # Calculate and apply stagger offsets and edge + # Output: Sets sgpr(StaggerRowMask) + ############################################################################## + def declareStaggerParms(self, kernel): + module = Module("declareStaggerParms") + if self.staggerU: + # this could be dynamic? + if kernel["StaggerUMapping"] == 0: + staggerInput = sgpr("WorkGroup0") + elif kernel["StaggerUMapping"] == 1: + staggerInput = sgpr("WorkGroup1") + elif kernel["StaggerUMapping"] == 2: + staggerInput = sgpr("WorkGroup2") + elif kernel["StaggerUMapping"] == 3: + with self.allocTmpSgpr(2) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + # wgSerial = (nwg0*ngw1)*wg2 + (nwg0)*wg1 + wg0 + wgSerial = tmpSgpr + tmp = tmpSgpr+1 + module.add(SMulI32(dst=sgpr(wgSerial), src0=sgpr("NumWorkGroups0"), src1=sgpr("NumWorkGroups1"), \ + comment="wgSerial = (nwg0*ngw1)*wg2 + (nwg0)*wg1 + wg0")) + module.add(SMulI32(dst=sgpr(wgSerial), src0=sgpr(wgSerial), src1=sgpr("WorkGroup2"))) + module.add(SMulI32(dst=sgpr(tmp), src0=sgpr("NumWorkGroups0"), src1=sgpr("WorkGroup1"))) + module.add(SAddU32(dst=sgpr(wgSerial), src0=sgpr(wgSerial), src1=sgpr(tmp))) + module.add(SAddU32(dst=sgpr(wgSerial), src0=sgpr(wgSerial), src1=sgpr("WorkGroup0"))) + staggerInput = sgpr(wgSerial) + elif kernel["StaggerUMapping"] == 4: + staggerInput = -1 + + module.add(SAndB32(dst=sgpr("StaggerUIter"), src0=sgpr("OrigStaggerUIter"), \ + src1=staggerInput, \ + comment="Compute actual stagger start for this tile")) + module.add(SLShiftLeftB32(dst=sgpr("StaggerUIter"), src=sgpr("StaggerUIter"), \ + shiftHex=kernel["_staggerStrideShift"], comment="shift by StaggerUStride")) + return module + + ############################################################################## + # Calculate and apply stagger offsets and edge + ############################################################################## + def calculateStagger(self, kernel, tP): + imod = Module("calculateStagger") + tc = tP["tensorChar"] + + if self.staggerU: + assert (kernel["BufferLoad"]) + + with self.allocTmpSgpr(2) as tmpSgprInfo: + staggerTmp = tmpSgprInfo.idx + + #--- + imod.addComment1("SRDs += (StaggerUIter) * GlobalReadIncs%s+%u"% (tc, self.unrollIdx)) + + # Calculate the stagger byte offset + imod.addModuleAsFlatItems(self.s_mul_i64_i32( + sgpr(staggerTmp), sgpr(staggerTmp+1), \ + sgpr("StaggerUIter"), sgpr("GlobalReadIncs%s+%u"%(tc, self.unrollIdx)), \ + " stagger byte offset")) + + # Amount of bytes to add to get back to start. + # on the llop iteration which matches StaggerUIter, this offset added instead of GlobalReadInc + imod.addModuleAsFlatItems(self.s_mul_i64_i32(sgpr("WrapU%s+0"%tc), sgpr("WrapU%s+1"%tc), \ + self.loopCounter(kernel, self.unrollIdx), sgpr("GlobalReadIncs%s+%u"%(tc,self.unrollIdx)), \ + "Number of bytes accessed by the unroll loop")) + + imod.add(SSubU32(dst=sgpr("WrapU%s+0"%tc), \ + src0=sgpr("GlobalReadIncs%s+%u"%(tc,self.unrollIdx)), \ + src1=sgpr("WrapU%s+0"%tc), \ + comment="remove one iteration")) + imod.add(SSubBU32(dst=sgpr("WrapU%s+1"%tc), \ + src0=0, \ + src1=sgpr("WrapU%s+1"%tc), \ + comment="remove one iteration")) + + imod.add(self.incrementSrd(tP, sgpr(staggerTmp), sgpr(staggerTmp+1))) + + if tP["isB"]: + # Convert passed in S' to S for easy loop comparison. S=S-(PGR-1)' + imod.add(SAddU32(dst=sgpr("StaggerUIter"), src0=sgpr("StaggerUIter"), \ + src1=(2 if kernel["PrefetchGlobalRead"] else 1), \ + comment="Subtract (PGR-1); StaggerUIter now contains target iteration to wrap")) + return imod + + ############################################################################## + # Remove stagger offset (before tail loop) + # | | | | + # |-- S'*I --| + # |---------- W' --------|-I-| + # ^ current SRD pos + # ^unrollLoopStart ^tailLoopStart (in summation0 dimension) + + # + # S = sgprStaggerUIter = S+(PGR+1)' + # W = sgprWrapU + # PGR = kernel["PrefetchGlobalRead"] + # + # S' = StaggUIter that is passed into the kernel = -PGR+1+S + # S'*I is also the global read offset (from unrollLoopStart) at unroll loop exit ? + # I = GlobalReadIncs + # W' = W + + # Need to move it to tailLoopStart + + # To compute position where tail loop should start: + # = W' - S'*I + I + # = W - (S+PGR+1)*I) + I + # = W - (S+PGR+1)*I + I + # = W - (S+2+PGR)*I + ############################################################################## + def removeStagger(self, kernel, tP): + imod = Module("removeStagger") + if self.staggerU: + tc = tP["tensorChar"] + with self.allocTmpSgpr(2) as tmpSgprInfo: + tmp = tmpSgprInfo.idx + # might be able to refactor this to eliminate signed math + imod.add(SSubI32(dst=sgpr(tmp), src0=3 if kernel["PrefetchGlobalRead"] else 2, \ + src1=sgpr("StaggerUIter"))) + imod.addModuleAsFlatItems(self.s_mul_i64_i32(sgpr(tmp), sgpr(tmp+1), \ + sgpr(tmp), sgpr("GlobalReadIncs%s+%u"%(tc,self.unrollIdx)), \ + "start offset S in bytes")) + imod.add(SSubU32(dst=sgpr(tmp), src0=sgpr(tmp), src1=sgpr("WrapU%s"%tc), comment="S - WrapU")) + imod.add(SSubBU32(dst=sgpr(tmp+1), src0=sgpr(tmp+1), src1=sgpr("WrapU%s+1"%(tc)), comment="S - WrapU")) + + imod.add(self.incrementSrd(tP, sgpr(tmp), sgpr(tmp+1))) + + return imod + + ############################################################################## + # Emit code to compute loop iterations for GSU. + # See same function in KernelWriterSource.py for background explanation + # This function is used to compute number of loop iters and also + # for computing the global read increment for GSU case. + # For multiple summation, the number of loop iterations needs to be reset + # for each iteration so replicate the code in addr inc and at open of unroll loop + + # tmpSgpr is allocation of at least 3 tmpSgpr + + # Output: SGPR(destName) contains the number of unroll iterations for + # this workgroup. + ############################################################################## + def calculateLoopNumIterGsu(self, kernel, destName, tmpSgprRes: RegisterPoolResource): + module = Module("calculateLoopNumIterGsu") + + loopCounter = sgpr(destName) + quotient = destName + remainder = "GSUSumIdx+1" # numIterPerWgRemainder + dividend = tmpSgprRes.idx + 2 # numIterMyWg + divisor = kernel["GlobalSplitU"] + if log(divisor,2).is_integer(): + module.add(SMovB32(dst=sgpr(dividend), src=loopCounter, comment="copy for divide IterGsu")) + module.add(scalarStaticDivideAndRemainder(quotient, remainder, dividend, divisor, tmpSgprRes , 1)) + else: + qReg = self.vgprPool.checkOut(1,"qReg") + rReg = self.vgprPool.checkOut(1,"rReg") + dReg = self.vgprPool.checkOut(1,"dReg") + tmpVgpr = self.vgprPool.checkOutAligned(2,2,"tmpReg") + tmpVgprRes = RegisterPoolResource(tmpVgpr, 2) + module.add(VMovB32(dst=vgpr(dReg), src=loopCounter, comment="copy for divide IterGsu")) + module.add(vectorStaticDivideAndRemainder(qReg, rReg, dReg, divisor, tmpVgprRes)) + module.add(VReadfirstlaneB32(dst=sgpr(quotient), src=vgpr(qReg))) + module.add(VReadfirstlaneB32(dst=sgpr(remainder), src=vgpr(rReg))) + self.vgprPool.checkIn(tmpVgpr) + self.vgprPool.checkIn(dReg) + self.vgprPool.checkIn(rReg) + self.vgprPool.checkIn(qReg) + + # if gsuSumIdx < numIterPerWgRemainder + module.add(SAddU32(dst=sgpr(tmpSgprRes.idx), src0=1, \ + src1=loopCounter, comment="tmp<-numIterMyWg+" )) + module.add(SCmpLtU32(src0=sgpr("GSUSumIdx"), src1=sgpr("GSUSumIdx+1"), \ + comment="gsuSumIdx < numIterPerWgRemainder" )) + module.add(SCMovB32(dst=loopCounter, src=sgpr(tmpSgprRes.idx), comment="numIterMyWg++ if needed")) + + return module + + ############################################################################## + # Calculate Loop Num Iter + # loopIdx is the index of the loop (used for contractions with multiple summations) + # 0 is outermost; self.unrollIdx is the unroll index. + # -1 is tail loop (used only for the unroll loop) + ############################################################################## + def calculateLoopNumIter(self, kernel, tPA, tPB, loopIdx): + module = Module("calculateLoopNumIter") + + tailLoop = loopIdx < 0 + if tailLoop: + loopIdx = self.unrollIdx + loopDim = kernel["ProblemType"]["IndicesSummation"][loopIdx] + loopChar = self.indexChars[loopDim] + + ######################################## + # Tail Loop + if tailLoop: + with self.allocTmpSgpr(4) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + loopCounterName = self.loopCounterName(kernel, loopIdx) + module.addSpaceLine() + if kernel["SuppressNoLoadLoop"]: + # If the tail loop is suppressed, then final iterations will have moved the Srd base forward + # (and also moved back the srd shadow limit) and slammed Limit to 0, so need to 'undo' + # those increments - see setTailSrd + assert(kernel["PrefetchGlobalRead"] == 1) #if >1 would need a multiply here + module.add(SCmpEQU32(src0=sgpr("OrigLoopCounter"), src1=0, comment="completely skipped unroll loop?")) + module.add(SCSelectB32(dst=sgpr(tmpSgpr+0), src0=0, src1=sgpr("GlobalReadIncsA"), comment="force to 0?")) + module.add(SCSelectB32(dst=sgpr(tmpSgpr+1), src0=0, src1=sgpr("GlobalReadIncsB"), comment="force to 0?")) + module.add(self.setTailSrd(tPA, sgpr(tmpSgpr+0))) + module.addSpaceLine() + module.add(self.setTailSrd(tPB, sgpr(tmpSgpr+1))) + module.addSpaceLine() + #module.add(self.getBomb()) + + module.addComment("numIter%s = (((size%s %% LOCAL_DEPTHU) + LOCAL_SPLITU - 1) / LOCAL_SPLITU)" \ + % (self.unrollChar, self.unrollChar)) + # size % DepthU + module.add(scalarStaticDivideAndRemainder(tmpSgpr, loopCounterName, \ + "SizesSum+%u"%loopIdx, kernel["DepthU"], RegisterPoolResource(tmpSgpr+2, 2), 2)) + loopCounter = sgpr(loopCounterName) + + if kernel["LocalSplitU"] > 1: + # (size % DepthU) + LSU - 1 + module.add(SAddU32(dst=loopCounter, src0=hex(kernel["LocalSplitU"]-1), src1=loopCounter, comment="(size % DepthU) + LSU - 1" )) + dividend = tmpSgpr+2 + module.add(SMovB32(dst=sgpr(dividend), src=loopCounter, comment="copy for divide LSU" )) + module.add(scalarStaticDivideAndRemainder( loopCounterName, None, dividend, kernel["LocalSplitU"], tmpSgprInfo, 0)) + + # if GSU numIter=0 if gsuSumIdx != remainder + if kernel["GlobalSplitU"] > 1: + module.add(SCmpLgU32(src0=sgpr("GSUSumIdx"), src1=sgpr("GSUSumIdx+1"), \ + comment="gsuSumIdx == numIterPerWgRemainder")) + module.add(SCMovB32(dst=loopCounter, src=hex(0), comment="numIter=0 if gsuSimIdx!=remainder")) + + # if tail numIter == 0 skip altogether + skipTailLoopLabel = Label.getFormatting("SkipTailLoop%s"%(loopChar) ) + module.add(SCmpEQU32(src0=loopCounter, src1=hex(0), comment="numIter%s == 0"%loopChar )) + module.add(SMovB32(dst=sgpr("OrigLoopCounter"), src=0, comment="repurpose to count each localRead increment")) + module.add(SCBranchSCC1(labelName=skipTailLoopLabel, \ + comment="skip to end of tail loop b/c numIter==0")) + + ######################################## + # Unrolled Loop + elif loopIdx == self.unrollIdx: + loopCounterName = self.loopCounterName(kernel, loopIdx) + loopCounter = sgpr(loopCounterName) + if not self.do["PreLoop"]: module.add(ValueEndif()) + + sumSize = "SizesSum+%u"%loopIdx + #sumSize = self.sumSize(kernel, loopIdx) + + # TODO - use named arguments + with self.allocTmpSgpr(3) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + quotient = loopCounterName + dividend = sumSize + divisor = kernel["DepthU"] + if self.noTailLoop and kernel["AssertSummationElementMultiple"] % kernel["DepthU"] != 0: + # round up SizesSum/DepthU for noTailLoop case + module.add(SAddI32(dst=sgpr(quotient), src0=(divisor - 1), src1=sgpr(dividend), \ + comment="round up SizeSum / DepthU" )) + module.add(scalarStaticDivideAndRemainder(quotient, None, quotient, divisor, tmpSgprInfo, 0)) + else: + module.add(scalarStaticDivideAndRemainder(quotient, None, dividend, divisor, tmpSgprInfo, 0)) + # if GSU numIter++ if gsuSumIdx < remainder + if kernel["GlobalSplitU"] > 1: + module.add(self.calculateLoopNumIterGsu(kernel, loopCounterName, tmpSgprInfo)) + + module.add(SMovB32(dst=sgpr("OrigLoopCounter"), \ + src=loopCounter, \ + comment="copy loop counter")) + else: + # other summation, not unroll loop + #printExit("no assembly support for 2+ dimensional summation") + module.addComment1("%sother summation, numIter%s = size%s" \ + % (self.indent, loopChar, loopChar)) + loopCounter = self.loopCounter(kernel, loopIdx) + module.add(SMovB32(dst=loopCounter, \ + src=sgpr("SizesSum+%u"%loopIdx), \ + comment="init loop counter")) + + return module + + ############################################################################## + # Open Loop + # uDu: 'None' means not generating branching label which decides which part of G2L + # buffer to write to LDS + ############################################################################## + def openLoop(self, kernel, tPA, tPB, loopIdx, uDu=None, noLabelGen=False, beginLabelOnly=False): + module = Module("openLoop") + # TODO - rewrite this function to simplify control-flow between tail-loop / unroll loop + tailLoop = loopIdx < 0 + if tailLoop: + loopIdx = self.unrollIdx + self.inTailLoop = True + loopChar = self.indexChars[ \ + kernel["ProblemType"]["IndicesSummation"][loopIdx]] + if not tailLoop and not noLabelGen: + module.add(Label("openLoop%s"%loopChar, "")) + loopLabelBegin = Label("%sLoopBegin%s%s"%("Tail" if tailLoop else "", loopChar, "_G2L%s"%uDu if uDu is not None else "" ), "" ) + loopLabelEnd = Label("%sLoopEnd%s%s"%("Tail" if tailLoop else "", loopChar, "_G2L%s"%uDu if uDu is not None else ""), "" ) + + if beginLabelOnly: + # generate only beginLabel, then, return + module.add(loopLabelBegin) + return module + + # is numIter at least 1? otherwise skip to end + # PGL needs a skip-check here if not bufferload + # If kernel["SuppressNoLoadLoop"] we don't have a special loop for the 'last iter' + loopCounter = self.loopCounter(kernel, loopIdx) + if tailLoop: + endCounter = 0 + elif kernel["PrefetchGlobalRead"] == 1: + if kernel["SuppressNoLoadLoop"]: + endCounter = 0 + else: + endCounter = 1 + elif kernel["PrefetchGlobalRead"] == 2: + if kernel["SuppressNoLoadLoop"]: + endCounter = 1 + else: + endCounter = 2 + else: + endCounter = 0 + + if tailLoop: + # LSU not all threads will do summation + if kernel["LocalSplitU"] > 1: + module.addComment1("apply exec mask for LSU") + tmpVgpr = self.vgprPool.checkOutAligned(2, 2, "tmpVgpr") + tmpVgprRes = RegisterPoolResource(tmpVgpr, 2) + dummy = self.vgprPool.checkOut(1,"dummy") + sgId = self.vgprPool.checkOut(1,"sgId") + divisor = kernel["SubGroup0"]*kernel["SubGroup1"] + module.add(vectorStaticDivide(sgId, "Serial", divisor, tmpVgprRes)) + numIter = self.vgprPool.checkOut(1,"numIter") + module.add(VMovB32(dst=vgpr(numIter), src=sgpr("SizesSum+0"), comment="sizeU to vgpr")) + divisor = kernel["DepthU"] + module.add(vectorStaticDivideAndRemainder(dummy, numIter, numIter, divisor, tmpVgprRes)) + self.vgprPool.checkIn(dummy) + #module.add() dump(vgpr(sgId)) ) + #module.add() dump(vgpr(numIter)) ) + module.add(VCmpXLtU32(dst=VCC(), \ + src0=vgpr(sgId), src1=vgpr(numIter), comment="sgId < numIter")) + self.vgprPool.checkIn(tmpVgpr) + #self.tailNumIter = numIter + #self.vgprPool.checkIn(numIter) + # thread is active is sgId < numIter % LocalSplitU + + # begin loop + if not noLabelGen: + module.add(loopLabelBegin) + + # LSU mask for this iteration + if kernel["LocalSplitU"] > 1: + module.add(VCmpXLtU32(dst=VCC(), \ + src0=vgpr(sgId), src1=vgpr(numIter), comment="sgId < numIter")) + module.add(VAddCOU32(dst=vgpr(sgId), dst1=VCC(), src0=hex(kernel["LocalSplitU"]), \ + src1=vgpr(sgId), comment="sgId+=LSU")) + self.vgprPool.checkIn(sgId) + self.vgprPool.checkIn(numIter) + #module.add() dump(vgpr(sgId)) ) + + else: # not tailloop: + + if loopIdx == self.unrollIdx: + # 1 loop check is necessary only when AssertSummationElementMultiple % (DepthU * 2) != 0 + if kernel["PrefetchGlobalRead"] == 2 and kernel["AssertSummationElementMultiple"] % (kernel["DepthU"] * 2) != 0: + module.add(SCmpEQU32( + src0=loopCounter, \ + src1=hex(endCounter-1), \ + comment="LoopCounter%s < EndCounter"%(loopChar) )) + toPGR1 = Label.getFormatting(self.labels.getName("toPGR1")) + module.add(SCBranchSCC1(labelName=toPGR1, comment="PGR=2 but only 1 loop, toPGR1")) + + module.add(SCmpLeU32( + src0=loopCounter, \ + src1=hex(endCounter), \ + comment="LoopCounter%s < EndCounter"%(loopChar) )) + jumpLabel = loopLabelEnd + if kernel["PrefetchGlobalRead"]==2 and (not kernel["SuppressNoLoadLoop"]) and kernel["ExpandPointerSwap"]: + # PGR=2 and EPS and no SuppressNoLoadLoop case, need to jump to EvenExit + jumpLabel = Label("LoopEnd%s_evenexit"%(loopChar), "" ) + module.add(SCBranchSCC1(labelName=jumpLabel.getLabelName(), \ + comment="do not enter Loop%s"%loopChar )) + + if not noLabelGen: + module.add(loopLabelBegin) + + if loopIdx != self.unrollIdx: + # reset LRO since these may have changed due to odd-iter exit ? + if kernel["PrefetchGlobalRead"]: + module.addComment0("openLoop - reset LRO for possible odd-iter exit") + module.add(self.localReadResetOffsets(kernel, tPA)) + module.add(self.localReadResetOffsets(kernel, tPB)) + + return module + + ############################################################################## + # Close Loop + # finalLoop : final unroll loop + # uDu: 'None' means not generating branching label which decides which part of G2L + # buffer to write to LDS + ############################################################################## + def closeLoop(self, kernel, tPA, tPB, loopIdx, finalLoop, uDu=None, emitEndLabelOnly=False, oddLabel=False): + module = Module("closeLoop") + if emitEndLabelOnly: + loopIdx = self.unrollIdx + loopChar = self.indexChars[ \ + kernel["ProblemType"]["IndicesSummation"][loopIdx]] + module.add(Label("SkipTailLoop%s"%(loopChar), "")) + return module + + finalJump = SCBranchSCC0 + nonFinalJumpNeeded = True + + tailLoop = loopIdx < 0 + if tailLoop: + loopIdx = self.unrollIdx + loopChar = self.indexChars[kernel["ProblemType"]["IndicesSummation"][loopIdx]] + loopLabelBegin = Label("TailLoopBegin%s%s"%(loopChar, "_G2L%s"%uDu if uDu is not None else ""), "" ) + loopLabelEnd = Label("TailLoopEnd%s%s"%(loopChar, "_G2L%s"%uDu if uDu is not None else ""), "" ) + loopLabelEndOddExit = Label("TailLoopEnd%s_oddexit"%(loopChar), "unroll loop odditer exit" ) + loopCounter = self.loopCounter(kernel, loopIdx) + + unrollInc = 1 + KinInnerUnroll = kernel["InnerUnroll"] + if kernel["EnableMatrixInstruction"]: + unrollInc *= kernel["MatrixInstK"] + KinInnerUnroll *= kernel["MatrixInstK"] + if kernel["AssertSummationElementMultiple"] % KinInnerUnroll == 0: + unrollInc *= kernel["InnerUnroll"] + + module.addComment1("closeLoop loop%s finalLoop=%d tailLoop=%d" % (loopChar, finalLoop, tailLoop)) + + module.add(SSubI32( + dst=loopCounter, \ + src0=loopCounter, \ + src1=hex(unrollInc), \ + comment="dec counter%s (tailLoop)"%(loopChar) )) + + # Track # LDS reads? + module.add(SAddU32( + dst=sgpr("OrigLoopCounter"), \ + src0=sgpr("OrigLoopCounter"), \ + src1=hex(unrollInc), + comment="inc counter%s"%(loopChar) )) + + endCounter = 0 + module.add(SCmpLeI32( + src0=loopCounter, \ + src1=hex(endCounter), \ + comment="counter%s<=%d"%(loopChar,endCounter) )) + else: # not tailloop + loopChar = self.indexChars[ \ + kernel["ProblemType"]["IndicesSummation"][loopIdx]] + loopLabelBegin = Label("LoopBegin%s"%(loopChar), "" ) + loopLabelEnd = Label("LoopEnd%s"%(loopChar), "" ) + loopLabelEndOddExit = Label("LoopEnd%s_oddexit"%(loopChar), "unroll loop odditer exit" ) + loopLabelEndEvenExit = Label("LoopEnd%s_evenexit"%(loopChar), "unroll loop eveniter exit" ) + loopCounter = self.loopCounter(kernel, loopIdx) + module.addComment1("closeLoop loop%s finalLoop=%d tailLoop=%d" % (loopChar, finalLoop, tailLoop)) + + # If PrefetchGlobalRead=1 the loads in the loop prefetch next macro-tile + # For the final trip through the unroll loop we need to ensure those loads stay in bounds. + + # One technique is to create a copy of the unroll loop with all loads removed. + # However buffer load doesn't need this loop copy since we OOB loads can be suppressed by buffer limit hardware + # So can do one more iteration (endCounter==0) in the main unroll loop, and adjust the pointer + # increments appropriately. + # Also sum idx other than unroll always compare against 0 (there is no PGR to account for) + if kernel["PrefetchGlobalRead"] == 1 and not kernel["SuppressNoLoadLoop"] and loopIdx == self.unrollIdx: + endCounter = 1 + elif kernel["PrefetchGlobalRead"] == 2 and not kernel["SuppressNoLoadLoop"] and loopIdx == self.unrollIdx: + endCounter = 2 + else: + endCounter = 0 + + if kernel["AssertSummationElementMultiple"] % (kernel["DepthU"] * 2) == 0 and endCounter > 0: + # if AssertSummationElementMultiple is multiple of DepthU*2, loop exit is necessary only once in 2 Loop iterations + # In endCounter % 2 == 1 case, exit at lc % 2 == 0 (= oddLabel). It means no exit if not oddLabel + # In endCounter % 2 == 0 case, exit at lc % 2 == 1 (= not oddLabel). It means no exit if oddLabel + # No exit case, no code is necessary except for final Loop + + # decrement by 2 if PGR=2 and StaggerU is 0, else 1 + decValue = 2 if kernel["PrefetchGlobalRead"]==2 and kernel["StaggerU"] == 0 else 1 + decCode = SSubU32(dst=loopCounter, src0=loopCounter, \ + src1=decValue, \ + comment="dec counter%s"%(loopChar) ) + condCode = SCmpEQI32(src0=loopCounter, \ + src1=hex(endCounter), \ + comment="counter%s==%d"%(loopChar,endCounter) ) + + noExit = False + + if endCounter%2 != 0: + if not oddLabel: + noExit = True + else: + if oddLabel: + noExit = True + + if noExit: + # No exit. No dec code if decValue is 2 + if decValue == 2: + decCode = "" + condCode = "" + nonFinalJumpNeeded = False + if finalLoop: + # No exit and finalLoop case, use s_branch (no condition) + finalJump = SBranch + + if decCode: module.add(decCode) + if condCode: module.add(condCode) + else: + module.add(SSubU32( + dst=loopCounter, src0=loopCounter, \ + src1=1, \ + comment="dec counter%s"%(loopChar) )) + + module.add(SCmpEQI32( + src0=loopCounter, \ + src1=hex(endCounter), \ + comment="counter%s==%d"%(loopChar,endCounter) )) + + jumpLabel = loopLabelEnd + if not tailLoop and not kernel["SuppressNoLoadLoop"] and kernel["ExpandPointerSwap"]: + # in this case, odd or/and even code is generated and use odd/even exit to avoid skipping odd/even code + # (end label is generated after odd/even code) + jumpLabel = loopLabelEndOddExit if oddLabel else loopLabelEndEvenExit + if not finalLoop: + if nonFinalJumpNeeded: + # just an exit check, else fall through to the next loop copy + module.add(SCBranchSCC1(labelName=jumpLabel.getLabelName(), comment="exit Loop%s"%loopChar )) + else: #finalLoop: + + if tailLoop and kernel.enabledSplitLDS: + tailLoopLabelEnd = Label.getFormatting( + "TailLoopEnd%s%s"%(loopChar, "_G2L%s"%(kernel["DepthULdsDivisor"]-1) if kernel.enabledSplitLDS else "") ) + module.add(SCBranchSCC1(labelName=tailLoopLabelEnd, comment="break Loop%s"%loopChar)) + thresForNextSubLoop = (uDu+1)*(kernel["_DepthULds"]) + module.add(SCmpGeU32(src0=sgpr("OrigLoopCounter"), src1=thresForNextSubLoop, + comment="OrigLoopCounter >= %u (G2L buffer %u/%u)"%(thresForNextSubLoop, uDu, kernel["DepthULdsDivisor"]) )) + + module.add(finalJump(labelName=loopLabelBegin.getLabelName(), comment="restart Loop%s"%(loopChar))) + + if not tailLoop and loopIdx == self.unrollIdx: + oddIterPreCode = Module() + oddIterCode = Module() + evenIterPreCode = Module() + evenIterCode = Module() + if not kernel["SuppressNoLoadLoop"] and kernel["ExpandPointerSwap"]: + oddIterPreCode.add(loopLabelEndOddExit) + # In this case we kept the 'no-load' loop which has LDS offsets assuming first bank of LDS + # if we exit the main loop at an odd iter - need to swap LDS read pointers + # so the ds_reads read from the 'high' buffer of LDS + oddIterPreCode.addComment1("Select high bank of LDS") + # Generate local read address code only if DirectToVgpr is not enabled + if not kernel["DirectToVgprA"]: + oddIterCode.add(self.localReadSwapOffsets(kernel, False, tPA)) + # Generate local read address code only if DirectToVgpr is not enabled + if not kernel["DirectToVgprB"]: + oddIterCode.add(self.localReadSwapOffsets(kernel, False, tPB)) + + evenIterPreCode.add(loopLabelEndEvenExit) + # generate even code here (so far, for PrefetchGlobalRead=2 only) + if kernel["PrefetchGlobalRead"]==2: + # Generate local write address code only for PrefetchGlobalRead==2 (localWriteSwapOffsets does nothing if DirectToVgpr is enabled) + # Code is unnecessary if DirectToLds is enabled, but internal SwapOffset is necessary if useInitAccVgprOpt is True + if kernel["DirectToLdsA"]: + if self.useInitAccVgprOpt: + self.localWriteSwapOffsets(kernel, True, tPA) + else: + evenIterCode.add(self.localWriteSwapOffsets(kernel, False, tPA)) + if kernel["DirectToLdsB"]: + if self.useInitAccVgprOpt: + self.localWriteSwapOffsets(kernel, True, tPB) + else: + evenIterCode.add(self.localWriteSwapOffsets(kernel, False, tPB)) + # swap internal write pointer as well (except for useInitAccVgprOpt case) + if not self.useInitAccVgprOpt: + evenIterCode.add(self.localWriteSwapOffsets(kernel, True, tPA)) + evenIterCode.add(self.localWriteSwapOffsets(kernel, True, tPB)) + + # generate even, odd exit code + # not oddLabel case, order is even -> odd + firstPreCode = evenIterPreCode + firstCode = evenIterCode + secondPreCode = oddIterPreCode + secondCode = oddIterCode + if oddLabel: + # oddLabel case, swap the order (odd -> even) + firstPreCode, secondPreCode = secondPreCode, firstPreCode + firstCode, secondCode = secondCode, firstCode + + module.add(firstPreCode) + module.add(firstCode) + + # if secondCode exist, add jump to skip secondCode + if secondCode.count(): + module.add(SBranch(labelName=loopLabelEnd.getLabelName(), \ + comment="exit unroll loop%s (and skip second exit code)"%(loopChar))) + module.add(secondPreCode) + module.add(secondCode) + + module.add(loopLabelEnd) + + if tailLoop: + if len(kernel["ProblemType"]["IndicesSummation"]) > 1: + # recover the 'damage' done to LRO: + + # if LRA is backed-up before (wlr case), we simply restore the addr (sub inc*loop doesn't work) + tPList = [] + if self.oriLraA != None: + if not kernel["DirectToVgprA"]: # no local read code if DirectToVgpr is enabled + module.add(VMovB32(dst=vgpr("LocalReadAddrA"), src=vgpr(self.oriLraA), comment="restore LRA")) + self.vgprPool.checkIn(self.oriLraA) + self.oriLraA = None + else: + tPList.append(tPA) + if self.oriLraB != None: + if not kernel["DirectToVgprB"]: # no local read code if DirectToVgpr is enabled + module.add(VMovB32(dst=vgpr("LocalReadAddrB"), src=vgpr(self.oriLraB), comment="restore LRA")) + self.vgprPool.checkIn(self.oriLraB) + self.oriLraB = None + else: + tPList.append(tPB) + for tP in tPList: + tc = tP["tensorChar"] + LdsPad = kernel["LdsPad%s" % tc] if kernel["LdsBlockSizePerPad%s"%tc] == 0 else 0 + inc = kernel["LocalSplitU"]*(kernel["MacroTile%s"%tc]+LdsPad)*tP["bpe"] + + # aligned with localReadInc + if kernel["EnableMatrixInstruction"]: + if kernel["UnrollMajorLDS%s" % tP["tensorChar"]]: + inc = kernel["LocalSplitU"] * tP["bpe"] + # No need to *= K, because LoopCounter is increased by K each time + # inc *= kernel["MatrixInstK"] + + if not kernel["DirectToVgpr%s"%tc]: # no local read code if DirectToVgpr is enabled + with self.allocTmpSgpr(1) as tmpSgprInfo: + stmp = tmpSgprInfo.idx + module.add(SMovB32(dst=sgpr(stmp), src=inc, comment="tailloop lds offset")) + module.add(SMulI32(dst=sgpr(stmp), src0=sgpr("OrigLoopCounter"), src1=sgpr(stmp), comment="scale by mul")) + module.add(VSubU32(dst=vgpr("LocalReadAddr%s"%tc), src0=vgpr("LocalReadAddr%s"%tc), src1=sgpr(stmp), comment="remove lro damage")) + # if LWA is backed-up before, we simply restore the addr + if self.oriLwaA != None: + if not kernel["DirectToVgprA"]: # no local write code if DirectToVgpr is enabled + module.add(VMovB32(dst=vgpr("LocalWriteAddrA"), src=vgpr(self.oriLwaA), comment="restore LWA")) + if not kernel["DirectToVgprB"]: # no local write code if DirectToVgpr is enabled + module.add(VMovB32(dst=vgpr("LocalWriteAddrB"), src=vgpr(self.oriLwaB), comment="restore LWA")) + self.vgprPool.checkIn(self.oriLwaA) + self.vgprPool.checkIn(self.oriLwaB) + self.oriLwaA = None + self.oriLwaB = None + + # restore all threads + if tailLoop and kernel["LocalSplitU"] > 1: + sgprCnt = self.laneSGPRCount + waveSize = kernel["WavefrontSize"] + module.addComment1("restore full exec mask") + + with self.allocTmpSgpr(sgprCnt) as tmpSgprInfo: + fullExec = tmpSgprInfo.idx + activeMask = "0xFFFFFFFF" if (waveSize == 32) else "0xFFFFFFFFFFFFFFFF" + SMovBX = SMovB32 if (waveSize == 32) else SMovB64 + SOrSaveExecBX = SOrSaveExecB32 if (waveSize == 32) else SOrSaveExecB64 + module.add(SMovBX(dst=sgpr(fullExec,sgprCnt), src=activeMask, comment="restore all threads active")) + module.add(SOrSaveExecBX (dst=sgpr(fullExec,sgprCnt), src=sgpr(fullExec,sgprCnt), comment="full mask -> exec" )) + return module + + ############################################################################## + # End Summation + ############################################################################## + def endSummation(self, kernel, label = None): + module = Module("endSummation") + + module.add(Label((self.labels.getUniqueNamePrefix("Summation_End") if label is None else label), "")) + + if kernel["StorePriorityOpt"]: + module.add(SSetPrior(prior=0, comment="optimization store")) + + vbegin = self.startVgprValuA + vsize = self.lastVgprForReads - self.startVgprValuA + + self.vgprPool.add(vbegin, vsize, "endSummation") + module.addComment0("endSummation: add vgpr [%u...%u) to pool" % \ + (vbegin, vbegin+vsize)) + + lastRegTag=None + for i in range(self.lastPostLoopSgpr, self.sgprPool.size()): + regTag = self.sgprPool.pool[i].tag + if regTag != lastRegTag: + lastRegTag = regTag + if self.sgprPool.pool[i].status == RegisterPool.Status.InUse: + module.add(self.undefineSgpr(regTag)) + + if self.db["InitVgpr"] & 0x2: + module.add(self.vgprPool.initTmps(self.initVgprValue,start=0, stop=100)) + if 0: # FIXME: Can remove? + for i in range(0,16+1): + module.add(VMovB32(dst=vgpr(21), src=vgpr(21), comment="hack tmp in pool")) + + # this doesn't seem to do anything - not being aggressive with lastPostLoopSgpr + if self.db["InitSgpr"] & 0x2: + module.add(self.sgprPool.initTmps(self.initSgprValue)) + + if self.db["ConservativeWaitCnt"] & 0x10: + module.add(SBarrier(comment="debug")) + module.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="")) + + if kernel["SuppressNoLoadLoop"]: + module.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="wait for all summation activity")) + + # copy accumulated C from agpr to vgpr + if kernel["EnableMatrixInstruction"]: + #TODO avoid s_nop if its possible + #instCycles = kernel["MatrixInstM"] // 2 # 32x32 is 64 cycles, 16x16 is 32 cycles, 4x4 is 8 cycles + #module.add(SNop(waitState=instCycles)) + module.add(self.MapAcctoArchRegs(kernel)) + if kernel["MIArchVgpr"]: + module.add(self.MulMIoutAlphaToArch(kernel)) + + return module + + ############################################################################## + # MFMA Iteration + ############################################################################## + def mfmaIter(self, kernel, tPA, tPB, u, innerUnroll, vregSetIdx, tail=False, firstIter=False): + imod = Module("mi") + shiftK = Module("shiftK") + m = (u) % (self.numVgprBuffer+1) # local to use for MACs + + # calculate constant + numRegistersIn = kernel["ProblemType"]["DataType"].numRegisters() + numRegistersOut = kernel["MIRegPerOut"] + loopCounterName = self.loopCounterName(kernel, self.unrollIdx) + accs_per_wave = kernel["MatrixInstM"] * kernel["MatrixInstN"] * kernel["MatrixInstB"] \ + // self.kernel["WavefrontSize"] * numRegistersOut + dividerFortidInK = kernel["MatrixInstN"] * kernel["MatrixInstB"] + numMIInput = kernel["MIInputPerThread"] + miInInstType, miOutInstType = dataTypeToMfmaInstTypePair(kernel["ProblemType"]["DataType"], \ + kernel["ProblemType"]["Fp16AltImpl"]) + vgprPerInput = int(numMIInput * numRegistersIn) + shiftPerElement = int(numRegistersIn * 32) + s_nop = 0 + gprfunc = accvgpr if not kernel["MIArchVgpr"] else vgpr + accumRegType = "acc" if not kernel["MIArchVgpr"] else "v" + mfma_1k = True if (kernel["MFMA_BF16_1K"] or kernel["ProblemType"]["Fp16AltImpl"]) else False + accStoreCIdx = 0 + + # alloc vgpr + kReg = None + abReg = None + tmpVgpr = None + dummy = None + + if (numRegistersIn < 1) and ((kernel["UnrollMajorLDSA"] == False) or (kernel["UnrollMajorLDSB"] == False)): + s_nop = 2 + + # here we remap index to where it read for wider local read + # ex. if we read 2 iteration at a time, + # original : _ds_load_b64 valuA_X0_I0 + # read 2 iter: _ds_load_b128 valuA_X0_I0 (we read valuA_X0_I0 and valuA_X1_I0) + # instead of using valuA_X1_I0, we use valuA_X0_I0+2 as mfma input + + vgprBufferA_new = (m//self.numIterPerCoalescedReadA)*self.numIterPerCoalescedReadA + vgprBufferA_new_offset = m%self.numIterPerCoalescedReadA*kernel["InnerUnroll"]*vgprPerInput + + vgprBufferB_new = (m//self.numIterPerCoalescedReadB)*self.numIterPerCoalescedReadB + vgprBufferB_new_offset = m%self.numIterPerCoalescedReadB*kernel["InnerUnroll"]*vgprPerInput + + numVgprPerBlockA = self.numVgprG2LA // 2 + numVgprPerBlockB = self.numVgprG2LB // 2 + + # handle multiple K element in MFMA instruction + if tail and kernel["MatrixInstK"] > 1: + kReg = self.vgprPool.checkOut(1,"kReg") # remainder + with self.allocTmpSgpr(1) as tmpSgprInfo: + shiftK.add(vectorStaticRemainder(dummy, kReg, "Serial", self.kernel["WavefrontSize"], tmpVgpr, tmpSgprInfo)) + shiftK.add(vectorStaticDivide(kReg, kReg, dividerFortidInK, tmpVgpr)) + shiftK.add(staticMultiply(vgpr(kReg), vgpr(kReg), numMIInput, tmpSgprInfo)) + + with self.allocTmpSgpr(3) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + # replace 0 for differnet thread + shiftK.add(VCmpGEI32(dst=sgpr(tmpSgpr, 2), src0=vgpr(kReg), src1=sgpr(loopCounterName), comment="check K index >= Size L")) + for bk in range(0, vgprPerInput): + for a in range(0, kernel["MIWaveTileA"]): + for iui in range(0, innerUnroll): + aStr = vgpr("ValuA_X%u_I%u+%u+%u" % (m, iui, a*vgprPerInput, bk), 1) + shiftK.add(VCndMaskB32(dst=aStr, src0=aStr, src1=hex(0), src2=sgpr(tmpSgpr, 2), comment="set 0 if K_idx >= sizeL")) + for b in range(0, kernel["MIWaveTileB"]): + for iui in range(0, innerUnroll): + bStr = vgpr("ValuB_X%u_I%u+%u+%u" % (m, iui, b*vgprPerInput, bk), 1) + shiftK.add(VCndMaskB32(dst=bStr, src0=bStr, src1=hex(0), src2=sgpr(tmpSgpr, 2), comment="set 0 if K_idx >= sizeL")) + + # replace 0 for same thread + if numMIInput > 1: + abReg = self.vgprPool.checkOutAligned(vgprPerInput, 2 if vgprPerInput>1 else 1, "abReg") + tmpVgpr = self.vgprPool.checkOutAligned(2,2,"tmpVgpr") + dummy = self.vgprPool.checkOut(1,"dummy") + shiftK.add(VSubU32(dst=vgpr(kReg), src0=sgpr(loopCounterName), src1=vgpr(kReg), comment="get distance between size and k index")) + shiftK.add(VCmpLtI32(dst=sgpr(tmpSgpr,2), src0=vgpr(kReg), src1=numMIInput, comment="set partial 0 if distance less than input per thread")) + shiftK.add(SAndB32(dst=sgpr(tmpSgpr+2), src0=sgpr(loopCounterName), src1=numMIInput-1, comment="get inputs for edge thread")) + shiftK.add(SSubU32(dst=sgpr(tmpSgpr+2), src0=numMIInput, src1=sgpr(tmpSgpr+2), comment="use shift to fill 0 for outside element")) + shiftK.add(SLShiftLeftB32(dst=sgpr(tmpSgpr+2), shiftHex=log2(shiftPerElement), src=sgpr(tmpSgpr+2), comment="use shift to fill 0 for outside element")) + if vgprPerInput == 1: + VShiftLeft = VLShiftLeftB32 + elif vgprPerInput == 2: + VShiftLeft = VLShiftLeftB64 + for a in range(0, kernel["MIWaveTileA"]): + for iui in range(0, innerUnroll): + iuiA_new = (iui//self.numReadsIterCoalescedA)*self.numReadsIterCoalescedA + iuiA_new_offset = iui%self.numReadsIterCoalescedA*vgprPerInput + a_new = a*vgprPerInput*self.numReadsIterCoalescedA + aStr = vgpr("ValuA_X%u_I%u+%u+%u+%u" % (vgprBufferA_new, iuiA_new, a_new, vgprBufferA_new_offset, iuiA_new_offset), vgprPerInput) + tmpVregIdx = 0 + shiftK.add(VShiftLeft(dst=vgpr(abReg, vgprPerInput), shiftHex=sgpr(tmpSgpr+2), src=aStr, comment="")) + for bk in range(0, vgprPerInput): + aStr = vgpr("ValuA_X%u_I%u+%u+%u+%u+%u" % (vgprBufferA_new, iuiA_new, a_new, vgprBufferA_new_offset, iuiA_new_offset, bk), 1) + if kernel["DirectToVgprA"]: + # overwrite aStr for DirectToVgprA + tmp = tmpVregIdx + bk + aStr = vgpr("G2LA+%u" % (tmp), vgprPerInput) + shiftK.add(VCndMaskB32(dst=aStr, src0=aStr, src1=vgpr(abReg+bk), src2=sgpr(tmpSgpr, 2), comment="")) + for b in range(0, kernel["MIWaveTileB"]): + for iui in range(0, innerUnroll): + iuiB_new = (iui//self.numReadsIterCoalescedB)*self.numReadsIterCoalescedB + iuiB_new_offset = iui%self.numReadsIterCoalescedB*vgprPerInput + b_new = b*vgprPerInput*self.numReadsIterCoalescedB + bStr = vgpr("ValuB_X%u_I%u+%u+%u+%u" % (vgprBufferB_new, iuiB_new, b_new, vgprBufferB_new_offset, iuiB_new_offset), vgprPerInput) + tmpVregIdx = 0 + shiftK.add(VShiftLeft(dst=vgpr(abReg, vgprPerInput), shiftHex=sgpr(tmpSgpr+2), src=bStr, comment="")) + for bk in range(0, vgprPerInput): + bStr = vgpr("ValuB_X%u_I%u+%u+%u+%u+%u" % (vgprBufferB_new, iuiB_new, b_new, vgprBufferB_new_offset, iuiB_new_offset, bk), 1) + if kernel["DirectToVgprB"]: + # overwrite bStr for DirectToVgprB + tmp = tmpVregIdx + bk + bStr = vgpr("G2LB+%u" % (tmp), 1) + shiftK.add(VCndMaskB32(dst=bStr, src0=bStr, src1=vgpr(abReg+bk), src2=sgpr(tmpSgpr, 2), comment="")) + + s_nop = 2 + + if s_nop != 0: + imod.add(SNop(waitState=(s_nop - 1), comment="")) + + for iui in range(0, innerUnroll): + iuiA_new = (iui//self.numReadsIterCoalescedA)*self.numReadsIterCoalescedA + iuiA_new_offset = iui%self.numReadsIterCoalescedA*vgprPerInput + iuiB_new = (iui//self.numReadsIterCoalescedB)*self.numReadsIterCoalescedB + iuiB_new_offset = iui%self.numReadsIterCoalescedB*vgprPerInput + zgemmVaddSrcCheck = [[], [], []] # to avoid generating redundant v_add + outer = 1 + loopSwap = False + # complex case, swap inner loop and outer loop so that idxA comes outer + # this is to re-use same tmp vgpr to nagate ai or ar + if kernel["ProblemType"]["DataType"].isComplex() and tPB["tile01Idx"]: + outer = 0 + loopSwap = True + inner = 1 - outer # inner is the opposite of outer + for idxOuter in range(0, kernel["MIWaveTile"][outer]): + for idxInner in range(0, kernel["MIWaveTile"][inner]): + idx0 = idxInner + idx1 = idxOuter + if loopSwap: + idx0, idx1 = idx1, idx0 + accIdx = idx1 * kernel["MIWaveTile"][0] + idx0 + accStart = accIdx * accs_per_wave + accEnd = accStart + accs_per_wave - 1 + accStartSrc1 = accStart + accEndSrc1 = accEnd + accStartSrc2 = accStart + accEndSrc2 = accEnd + if firstIter: + # use the last accs_per_wave as src (assuming only these are initialized to 0) + numAccvgprs = self.numVgprValuC if kernel["MIArchVgpr"] else self.totalAgprs + accStartSrc1 = numAccvgprs - accs_per_wave + accEndSrc1 = accStartSrc1 + accs_per_wave - 1 + idxA = idx0 if tPB["tile01Idx"] else idx1 + idxB = idx1 if tPB["tile01Idx"] else idx0 + a_new = idxA*vgprPerInput*self.numReadsIterCoalescedA + b_new = idxB*vgprPerInput*self.numReadsIterCoalescedB + aStr = "ValuA_X%u_I%u+%u+%u+%u" % (vgprBufferA_new, iuiA_new, a_new, vgprBufferA_new_offset, iuiA_new_offset) + bStr = "ValuB_X%u_I%u+%u+%u+%u" % (vgprBufferB_new, iuiB_new, b_new, vgprBufferB_new_offset, iuiB_new_offset) + if kernel["DirectToVgprA"]: + # overwrite aStr for DirectToVgprA + numVgprValuAPerBlock = kernel["MIWaveTileA"] * kernel["MIInputPerThread"] * tPA["bpe"] // self.bpr + # re-calculate vgprBufferA_new and offset for DirectToVgpr. Use u instead of m (number of local prefetch buffer does not matter) + vgprBufferA_new = (u//self.numIterPerCoalescedReadA)*self.numIterPerCoalescedReadA + vgprBufferA_new_offset = u%self.numIterPerCoalescedReadA*kernel["InnerUnroll"]*vgprPerInput + a_new += vregSetIdx * numVgprPerBlockA + (iuiA_new + vgprBufferA_new * kernel["InnerUnroll"]) * numVgprValuAPerBlock + aStr = "G2LA+%u+%u+%u" % (a_new, vgprBufferA_new_offset, iuiA_new_offset) + # self.vgprValuDouble case, need to change valuB to toggle double buffer + if self.vgprValuDouble and vregSetIdx > 0: + numOneSet = self.numVgprValuB//2 + bStr += "+%u"%(vregSetIdx * numOneSet) + if kernel["DirectToVgprB"]: + # overwrite bStr for DirectToVgprB + numVgprValuBPerBlock = kernel["MIWaveTileB"] * kernel["MIInputPerThread"] * tPB["bpe"] // self.bpr + # re-calculate vgprBufferB_new and offset for DirectToVgpr. Use u instead of m (number of local prefetch buffer does not matter) + vgprBufferB_new = (u//self.numIterPerCoalescedReadB)*self.numIterPerCoalescedReadB + vgprBufferB_new_offset = u%self.numIterPerCoalescedReadB*kernel["InnerUnroll"]*vgprPerInput + b_new += vregSetIdx * numVgprPerBlockB + (iuiB_new + vgprBufferB_new * kernel["InnerUnroll"]) * numVgprValuBPerBlock + bStr = "G2LB+%u+%u+%u" % (b_new, vgprBufferB_new_offset, iuiB_new_offset) + # self.vgprValuDouble case, need to change valuA to toggle double buffer + if self.vgprValuDouble and vregSetIdx > 0: + numOneSet = self.numVgprValuA//2 + aStr += "+%u"%(vregSetIdx * numOneSet) + aStr = vgpr(aStr, vgprPerInput) + bStr = vgpr(bStr, vgprPerInput) + Str0 = aStr if tPB["tile01Idx"] else bStr + Str1 = bStr if tPB["tile01Idx"] else aStr + + if kernel["ProblemType"]["DataType"].isComplex(): + # override because complex mul is emulated by 4 mfma insts + # TODO: adopt component system + miInInstType = miOutInstType #"f32" for SingleComplex, "f64" for DoubleComplex + ccA = kernel["ProblemType"]["ComplexConjugateA"] + ccB = kernel["ProblemType"]["ComplexConjugateB"] + ccVgprs = [None]*3 # three terms that can be negated: [real1, imag0, imag1] + ccInsts = [None]*3 + accImOffset = self.AccVgprImagNumOffset(kernel) + # for firstIter, need to use accStartSrc for img instead of adding accImOffset + accStartSrcImg1 = accStartSrc1 if firstIter else accStartSrc1+accImOffset + accEndSrcImg1 = accStartSrcImg1 + accs_per_wave - 1 + accStartSrcImg2 = accStartSrc2+accImOffset + accEndSrcImg2 = accStartSrcImg2 + accs_per_wave - 1 + + # vgpr A,B setting. In complex case, numRegistersIn does not match. Use numRegistersOut instead + ar = vgpr("ValuA_X%u_I%u+%u+%u+%u" % (vgprBufferA_new, iuiA_new, a_new, vgprBufferA_new_offset, iuiA_new_offset), numRegistersOut) + ai = vgpr("ValuA_X%u_I%u+%u+%u+%u+%u" % (vgprBufferA_new, iuiA_new, a_new, vgprBufferA_new_offset, iuiA_new_offset, numRegistersOut), numRegistersOut) + br = vgpr("ValuB_X%u_I%u+%u+%u+%u" % (vgprBufferB_new, iuiB_new, b_new, vgprBufferB_new_offset, iuiB_new_offset), numRegistersOut) + bi = vgpr("ValuB_X%u_I%u+%u+%u+%u+%u" % (vgprBufferB_new, iuiB_new, b_new, vgprBufferB_new_offset, iuiB_new_offset, numRegistersOut), numRegistersOut) + if kernel["DirectToVgprA"]: + ## overwrite aStr for DirectToVgprA + ar = vgpr("G2LA+%u+%u+%u" % (a_new, vgprBufferA_new_offset, iuiA_new_offset), numRegistersOut) + ai = vgpr("G2LA+%u+%u+%u+%u" % (a_new, vgprBufferA_new_offset, iuiA_new_offset, numRegistersOut), numRegistersOut) + if kernel["DirectToVgprB"]: + # overwrite bStr for DirectToVgprB + br = vgpr("G2LB+%u+%u+%u" % (b_new, vgprBufferB_new_offset, iuiB_new_offset), numRegistersOut) + bi = vgpr("G2LB+%u+%u+%u+%u" % (b_new, vgprBufferB_new_offset, iuiB_new_offset, numRegistersOut), numRegistersOut) + minus_ar = ar.getMinus() + minus_ai = ai.getMinus() + if miOutInstType == InstType.INST_F32: + VAddX = VAddF32 + elif miOutInstType == InstType.INST_F64: + VAddX = VAddF64 + else: + printExit("Unsupported v_add type %s"%miOutInstType) + offsetVgpr = [0,0,0] + forceGenerate = ccA and ccB # so far, v_add is always necessary for ccA and ccB case + if ccA == ccB: + arrayIndex = 0 + ccVgprs[arrayIndex] = self.vgprPool.checkOutAligned(numRegistersOut, numRegistersOut, "negate r1") + # generate negate code only when same code is not generated (avoid generating same (redundant) code again + if forceGenerate or (ai not in zgemmVaddSrcCheck[arrayIndex]): + ccInsts[arrayIndex] = VAddX(dst=vgpr(ccVgprs[arrayIndex] + offsetVgpr[arrayIndex], numRegistersOut), src0=minus_ai, src1=0, comment="Ai=-Ai") + zgemmVaddSrcCheck[arrayIndex].append(ai) + if ccA: + arrayIndex = 1 + ccVgprs[arrayIndex] = self.vgprPool.checkOutAligned(numRegistersOut, numRegistersOut, "negate i0") + # generate negate code only when same code is not generated (avoid generating same (redundant) code again + if forceGenerate or (ai not in zgemmVaddSrcCheck[arrayIndex]): + ccInsts[arrayIndex] = VAddX(dst=vgpr(ccVgprs[arrayIndex] + offsetVgpr[arrayIndex], numRegistersOut), src0=minus_ai, src1=0, comment="Ai=-Ai") + zgemmVaddSrcCheck[arrayIndex].append(ai) + if ccB: + arrayIndex = 2 + ccVgprs[arrayIndex] = self.vgprPool.checkOutAligned(numRegistersOut, numRegistersOut, "negate i1") + # generate negate code only when same code is not generated (avoid generating same (redundant) code again + if forceGenerate or (ar not in zgemmVaddSrcCheck[arrayIndex]): + ccInsts[arrayIndex] = VAddX(dst=vgpr(ccVgprs[arrayIndex] + offsetVgpr[arrayIndex], numRegistersOut), src0=minus_ar, src1=0, comment="Ar=-Ar") + zgemmVaddSrcCheck[arrayIndex].append(ar) + (src0, src1) = (br, ar) if kernel["SourceSwap"] else (ar, br) + for inst in ccInsts: + if inst is not None: + imod.add(inst) + variant = [kernel["MatrixInstM"], kernel["MatrixInstN"], kernel["MatrixInstK"]] + imod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=False, \ + acc=gprfunc(accStart, (accEnd-accStart+1)), a=src0, b=src1, acc2=gprfunc(accStartSrc1, (accEndSrc1-accStartSrc1+1)), \ + comment="Cr += Ar*Br")) + (src0, src1) = (bi, (vgpr(ccVgprs[0] + offsetVgpr[0], numRegistersOut) if ccVgprs[0] else ai)) if kernel["SourceSwap"] else ((vgpr(ccVgprs[0] + offsetVgpr[0], numRegistersOut) if ccVgprs[0] else ai), bi) + imod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=False, \ + acc=gprfunc((accStart+accStoreCIdx), (accEnd-accStart+1)), a=src0, b=src1, acc2=gprfunc(accStartSrc2, (accEndSrc2-accStartSrc2+1)), \ + comment="Cr += %sAi*Bi"%("-" if ccVgprs[0] else ""))) + (src0, src1) = (br, (vgpr(ccVgprs[1] + offsetVgpr[1], numRegistersOut) if ccVgprs[1] else ai)) if kernel["SourceSwap"] else ((vgpr(ccVgprs[1] + offsetVgpr[1], numRegistersOut) if ccVgprs[1] else ai), br) + imod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=False, \ + acc=gprfunc((accStart+accImOffset), (accEnd-accStart+1)), a=src0, b=src1, acc2=gprfunc(accStartSrcImg1, (accEndSrcImg1-accStartSrcImg1+1)), \ + comment="Ci += %sAi*Br"%("-" if ccVgprs[1] else ""))) + (src0, src1) = (bi, (vgpr(ccVgprs[2] + offsetVgpr[2], numRegistersOut) if ccVgprs[2] else ar)) if kernel["SourceSwap"] else ((vgpr(ccVgprs[2] + offsetVgpr[2], numRegistersOut) if ccVgprs[2] else ar), bi) + imod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=False, \ + acc=gprfunc((accStart+accImOffset+accStoreCIdx), (accEnd-accStart+1)), a=src0, b=src1, acc2=gprfunc(accStartSrcImg2, (accEndSrcImg2-accStartSrcImg2+1)), \ + comment="Ci += %sAr*Bi"%("-" if ccVgprs[2] else ""))) + + for v in ccVgprs: + if v is not None: self.vgprPool.checkIn(v) + else: + if kernel["SourceSwap"]: + src0 = Str1 + src1 = Str0 + else: + src0 = Str0 + src1 = Str1 + variant = [kernel["MatrixInstM"], kernel["MatrixInstN"], kernel["MatrixInstK"]] + imod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=gprfunc((accStart+accStoreCIdx), (accEnd-accStart+1)), \ + a=src0, b=src1, acc2=gprfunc(accStartSrc1, (accEndSrc1-accStartSrc1+1)), \ + comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) + + # release register + if kReg is not None: self.vgprPool.checkIn(kReg) + if abReg is not None: self.vgprPool.checkIn(abReg) + if tmpVgpr is not None: self.vgprPool.checkIn(tmpVgpr) + if dummy is not None: self.vgprPool.checkIn(dummy) + + mfmaMod = Module("mfmaCode") + mfmaMod.add(shiftK) + mfmaMod.add(imod) + + return mfmaMod + + ############################################################################## + # At Least 1 Unroll + # prefetch means this is in the prefetch code, either before unroll loop + # or in the PAP code. + # isOptNLL : this is for the store-interleaved NLL optimization + ############################################################################## + def openSumAtLeastUnroll(self, kernel, prefetch, isOptNLL): + module = Module("openSumAtLeastUnroll") + if prefetch: + if not isOptNLL: + module.add(self.checkLastIter(kernel)) + if kernel["StorePriorityOpt"]: + module.add(SSetPrior(prior=0, comment="optimization store")) + if self.doShadowInit: + shadowName = Label.getFormatting("ShadowInitStart") + module.add(SCBranchSCC1(labelName=shadowName, \ + comment="skip to ShadowInitStart iter b/c numIter==0")) + else: + loopChar = self.indexChars[ \ + kernel["ProblemType"]["IndicesSummation"][self.unrollIdx]] + labelName = Label.getFormatting("LoopEnd%s"%loopChar) + module.add(SCBranchSCC1(labelName=labelName, \ + comment="skip to unrollLoop end loop%s iter b/c numIter==0" % loopChar)) + elif isOptNLL: + skipOptNLL = Label("OptNLL_End", "") + with self.allocTmpSgpr(2) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + module.add(self.checkIsBetaZero(kernel, tmpSgpr, skipOptNLL)) + + # check alpha + if self.do["ApplyAlpha"]: + # (The new hgemm (h,h,h,h,s,s) is included in ComputeType=Single) + if kernel["ProblemType"]["ComputeDataType"].isHalf(): + # for (h,h,h,h,h,h) no HPA, + module.add(SMovB32(dst=sgpr(tmpSgpr), src=hex(0x3c003c00), comment="Packed alpha==1.0")) + module.add(SCmpEQU32(src0=sgpr("Alpha"), src1=sgpr(tmpSgpr), comment="alpha == 1.0?")) + + # Shouldn't go here. Currently, DataType=B->ComputeDataType=S + # (bf-gemm is included in ComputeType=Single) + elif kernel["ProblemType"]["ComputeDataType"].isBFloat16(): + module.add(SMovB32(dst=sgpr(tmpSgpr), src=hex(0x3f803f80), comment="Packed alpha==1.0")) + module.add(SCmpEQU32(src0=sgpr("Alpha"), src1=sgpr(tmpSgpr), comment="alpha == 1.0?")) + + elif kernel["ProblemType"]["ComputeDataType"].isInt32(): + module.add(SCmpEQU32(src0=sgpr("Alpha"), src1=1, comment="Alpha == 1.0 ?")) + + # This covers sgemm, bfgemm + HPA (b,b,b,b,s,s), and also hgemm (h,h,h,h,s,s) + elif kernel["ProblemType"]["ComputeDataType"].isSingle(): + module.add(SCmpEQU32(src0=sgpr("Alpha"), src1=1.0, comment="Alpha == 1.0 ?")) + + elif kernel["ProblemType"]["ComputeDataType"].isDouble(): + module.add(SMovB32(dst=sgpr(tmpSgpr+0), src=hex(0x00000000), comment="Low part of double 1.0")) + module.add(SMovB32(dst=sgpr(tmpSgpr+1), src=hex(0x3ff00000), comment="High part of double 1.0")) + module.add(SCmpEQU64(dst=sgpr("Alpha",2), src=sgpr(tmpSgpr,2), comment="Alpha == 1.0 ?")) + + elif kernel["ProblemType"]["ComputeDataType"].isSingleComplex(): + module.add(SMovB32(dst=sgpr(tmpSgpr+0), src=hex(1.0), comment="Real part of 1.0")) + module.add(SMovB32(dst=sgpr(tmpSgpr+1), src=hex(0.0), comment="Imaginary part of 1.0")) + module.add(SCmpEQU64(src0=sgpr("Alpha",2), src1=sgpr(tmpSgpr,2), comment="Alpha == 1.0 ?")) + + elif kernel["ProblemType"]["ComputeDataType"].isDoubleComplex(): + module.add(SMovB32(dst=sgpr(tmpSgpr+0), src=hex(0x00000000), comment="lsb of real part of 1.0")) + module.add(SMovB32(dst=sgpr(tmpSgpr+1), src=hex(0x3ff00000), comment="msb of real part of 1.0")) + module.add(SCmpEQU64(src0=sgpr("Alpha",2), src1=sgpr(tmpSgpr,2), comment="Alpha.real == 1.0 ?")) + module.add(SCBranchSCC0(labelName=skipOptNLL.getLabelName(), comment="branch if alpha.real != 1")) + module.add(SMovB32(dst=sgpr(tmpSgpr+0), src=hex(0x00000000), comment="lsb of imag part of 0.0")) + module.add(SMovB32(dst=sgpr(tmpSgpr+1), src=hex(0x00000000), comment="msb of imag part of 0.0")) + module.add(SCmpEQU64(src0=sgpr("Alpha+2",2), src1=sgpr(tmpSgpr,2), comment="Alpha.imag == 0.0 ?")) + + module.add(SCBranchSCC0(labelName=skipOptNLL.getLabelName(), comment="branch if alpha != 1")) + module.addSpaceLine() + + module.add(self.checkIsEdge(kernel, tmpSgpr, skipOptNLL)) + module.addSpaceLine() + + # Check tail loop required: + # Skip tail loop check if noTailLoop is true + if not self.noTailLoop: + loopChar = self.indexChars[ \ + kernel["ProblemType"]["IndicesSummation"][self.unrollIdx]] + module.add(scalarStaticDivideAndRemainder(tmpSgpr, tmpSgpr+1, "SizesSum+%u"%self.unrollIdx, \ + kernel["DepthU"], RegisterPoolResource(tmpSgpr+2, 2), 2)) + module.add(SCmpEQU32(src0=sgpr(tmpSgpr+1), src1=hex(0), comment="numIter%s == 0"%loopChar )) + module.add(SCBranchSCC0(labelName=skipOptNLL.getLabelName(), comment="skip if tail loop required")) + + # save the vgprPool for generating the normal path. + # dump the 'dirty' pool upon s_endpgm and swap back the 'clean' pool + # so we can avoid explicit vgpr check-in/out + self.savedVgprPool = deepcopy(self.vgprPool) + self.savedSgprPool = deepcopy(self.sgprPool) + + # comment out the following codes that attempt to reduce vgpr consumption + # however, the kernel vgpr count is governed by peak vgpr consumption so saving + # a few here shouldn't affect kernel's overall vgpr consumption. + # the following code is for reference and will be removed in the future + """ + added = [] # track registers added to pool + if kernel["PrefetchGlobalRead"]: + if not kernel["DirectToLdsA"]: + added.append(self.vgprPool.addRange(self.startVgprG2LA, \ + self.startVgprG2LA+self.numVgprG2LA-1, "startOptNLL")) + added.append(self.vgprPool.addRange(self.startVgprLocalWriteAddressesA, \ + self.startVgprLocalWriteAddressesA, "startOptNLL")) + if not kernel["DirectToLdsB"]: + added.append(self.vgprPool.addRange(self.startVgprG2LB, \ + self.startVgprG2LB+self.numVgprG2LB-1, "startOptNLL")) + added.append(self.vgprPool.addRange(self.startVgprLocalWriteAddressesB, \ + self.startVgprLocalWriteAddressesB, "startOptNLL")) + + if kernel["BufferLoad"]: + added.append(self.vgprPool.addRange(self.startVgprGlobalReadOffsetA, \ + self.startVgprGlobalReadOffsetB, "startOptNLL")) + else: + added.append(self.vgprPool.addRange(self.startVgprGlobalReadAddressesA, \ + self.startVgprGlobalReadAddressesB, "startOptNLL")) + module.addComment1("reclaim VGPRS: " + ", ".join(added)) + """ + + return module + + ############################################################################## + def closeSumAtLeastUnroll(self, kernel, tPA, tPB, prefetch, isOptNLL, isNGLL): + module = Module("closeSumAtLeastUnroll") + if not prefetch: + if isNGLL: + toPGR1 = Label(self.labels.getName("toPGR1"), "") + module.add(toPGR1) + else: + if isOptNLL: + endSumLabel = "Summation_End_OptNLL" + + module.addComment0("Stores for OptNLL") + module.add(self.endSummation(kernel, endSumLabel)) + + # perhaps could work with LSU>1 by adding other indices here, but not tested + assert (kernel["LocalSplitU"] == 1) + module.add(self.notLocalSplitUGlobalWriteIndices(kernel)) + + # add stores for opt NLL + (fullVw, elements) = self.notLocalFullTileElements(kernel, False) + alpha = False + beta = False + module.add(self.globalWriteElements(kernel, tPA, tPB, [fullVw], [elements], applyAlpha=alpha, betas=[beta], edges=[False])) + + self.cleanupGlobalWrite(kernel) + module.addSpaceLine() + module.add(self.functionEnd(False)) + module.add(Label("OptNLL_End", "")) + + else: + module.add(Label("PrefetchGlobalLastIterEnd", "")) + + # swap back vgpr pool if any + if self.savedVgprPool != None: + # in case pool size in current path is larger than pool size in main path + # and it will miss allocate vgpr since allocating vgpr is based on pool size in main path + oldSize = self.savedVgprPool.size() + newSize = self.vgprPool.size() + if newSize > self.savedVgprPool.size(): + for i in range(oldSize,newSize): + self.savedVgprPool.pool.append(self.savedVgprPool.Register(RegisterPool.Status.Available,"restore vgprPool")) + self.vgprPool = self.savedVgprPool # restore vgprPool before alternate path + self.savedVgprPool = None + # swap back sgpr pool if any + if self.savedSgprPool != None: + # in case pool size in current path is larger than pool size in main path + # and it will miss allocate vgpr since allocating vgpr is based on pool size in main path + oldSize = self.savedSgprPool.size() + newSize = self.sgprPool.size() + if newSize > self.savedSgprPool.size(): + for i in range(oldSize-1,newSize): + self.savedSgprPool.pool.append(self.savedSgprPool.Register(RegisterPool.Status.Available,"restore sgprPool")) + self.sgprPool = self.savedSgprPool # restore vgprPool before alternate path + self.savedSgprPool = None + return module + + ############################################################################## + # incLower must be constant or SGPR unsigned value + def incrementSrd(self, tP, incLower, incUpper, checkShadowLimitCopy=True): + imod = Module("incrementSrd") + tc = tP["tensorChar"] + + imod.add(SAddU32(dst=sgpr("Srd%s+0"%(tc)), \ + src0=sgpr("Srd%s+0"%(tc)), \ + src1=incLower, \ + comment="gra SRD += inc(lower)" )) + imod.add(SAddCU32(dst=sgpr("Srd%s+1"%(tc)), \ + src0=sgpr("Srd%s+1"%(tc)), \ + src1=incUpper, \ + comment="gra SRD += inc(upper)" )) + + # also have to move the boundary since we change the base + # so less buffers to the edge: + if self.use64bShadowLimit: + imod.add(SSubU32(dst=sgpr("ShadowLimit%s+0"%tc), \ + src0=sgpr("ShadowLimit%s+0"%tc), \ + src1=incLower, \ + comment="limit -= inc)")) + imod.add(SSubBU32(dst=sgpr("ShadowLimit%s+1"%tc), \ + src0=sgpr("ShadowLimit%s+1"%tc), \ + src1=incUpper, \ + comment="limit -= inc)" )) + if checkShadowLimitCopy: + imod.add(SCmpEQU32(src0=sgpr("ShadowLimit%s+1"%tc), src1=0, comment="are we within 2^32?")) + if self.staggerU: + # staggerU case, need to restore BufferLimit when ShadowLimit goes to negative value + imod.add(SCSelectB32(dst=sgpr("Srd%s+2"%tc), src0=sgpr("ShadowLimit%s+0"%tc), src1="BufferLimit", comment="Move shadow to real if we are within 2^32")) + else: + imod.add(SCMovB32(dst=sgpr("Srd%s+2"%tc), src=sgpr("ShadowLimit%s+0"%tc), comment="Move shadow to real if we are within 2^32")) + else: + imod.add(SSubU32(dst=sgpr("Srd%s+2"%(tc)), \ + src0=sgpr("Srd%s+2"%(tc)), \ + src1=incLower, \ + comment="limit -= inc)" )) + return imod + + ############################################################################## + # incLower must be constant or SGPR unsigned value + def setTailSrd(self, tP, incLower): + # In SuppressNoLoadLoop, the final loop iteration moves the SRD base forward + # and the ShadowLimit backwards by one extra 'click' of GlobalReadIncs[AB]. + # Note the ShadowLimit may become negative - for example edge tiles where the + # increment is > tile width. + # The SuppressNoLoadLoop mode also forces the SRD limit to 0 on the final iteration. + # The code here undoes the final click step by moving the base backwards and the + # limit forwards (reading from the ShadowLimit). + # It only works if use64bShadowLimit is enabled (since this enables use of the ShadowLimit) + + tc = tP["tensorChar"] + module = Module("setTailSrd") + incUpper = 0 + + module.add(SSubU32(dst=sgpr("Srd%s+0"%(tc)), \ + src0=sgpr("Srd%s+0"%(tc)), src1=incLower, \ + comment="gra SRD -= inc(lower)" )) + module.add(SSubBU32(dst=sgpr("Srd%s+1"%(tc)), \ + src0=sgpr("Srd%s+1"%(tc)), src1=incUpper, \ + comment="gra SRD -= inc(upper)" )) + + # using Shadow limit here which only works with 64-bit PBC: + assert(self.use64bShadowLimit) + + module.add(SAddU32(dst=sgpr("ShadowLimit%s+0"%tc), \ + src0=sgpr("ShadowLimit%s+0"%tc), src1=incLower, \ + comment="limit -= inc)")) + module.add(SAddCU32(dst=sgpr("ShadowLimit%s+1"%tc), \ + src0=sgpr("ShadowLimit%s+1"%tc), src1=incUpper, \ + comment="limit -= inc)" )) + module.add(SCmpEQU32(src0=sgpr("ShadowLimit%s+1"%tc), src1=0, comment="are we within 2^32?")) + module.add(SCMovB32(dst=sgpr("Srd%s+2"%tc), src=sgpr("ShadowLimit%s+0"%tc), comment="Move shadow to real if we are within 2^32")) + + return module + + ############################################################################## + # Global Read: Increment A/B + # loopIdx is summation idx: + # self.unrollIdx, or an idx from 0..NumIndicesSummation + # prefetchIndex is >0 (1...PrefetchGlobalRead) if this increment follows a + # global prefetch or 0 otherwise + ############################################################################## + def globalReadIncrement(self, kernel, imod, loopIdx, tP, prefetchIndex): + if not self.do["GlobalInc"]: return "" + tc = tP["tensorChar"] + loopChar = self.indexChars[ \ + kernel["ProblemType"]["IndicesSummation"][loopIdx]] + + imod.addComment1("global read inc %s loop%s"%(tc,loopChar)) + + if kernel["BufferLoad"]: + # TODO - does this handle N-dim tensors correctly? + #if tP["isB"]: + # module.add(SMovB32(dst=sgpr("OffsetB"), src=sgpr("SrdB+0"), comment="hack to save")) + if self.staggerU and loopIdx == self.unrollIdx: + # add a wrap increment, if needed: + with self.allocTmpSgpr(3) as tmpSgprInfo: + incLower = tmpSgprInfo.idx + incUpper = incLower + 1 + tmpS = incLower + 2 + if prefetchIndex: + imod.add(SAddU32(dst=sgpr(tmpS), src0=self.loopCounter(kernel, self.unrollIdx), src1=prefetchIndex, comment="remove pf(%u)"%prefetchIndex)) + imod.add(SCmpEQU32(src0=sgpr("StaggerUIter"), src1=sgpr(tmpS), comment="Is this wrapIter? (pf)")) + else: + imod.add(SCmpEQU32(src0=self.loopCounter(kernel, self.unrollIdx), \ + src1=sgpr("StaggerUIter"), comment="Is this the wrapIter?")) + imod.add(SCSelectB32(dst=sgpr(incLower), src0=sgpr("WrapU%s+0"%tc), src1=sgpr("GlobalReadIncs%s+%u"%(tc,self.unrollIdx)), \ + comment="incLower <- ?")) + imod.add(SCSelectB32(dst=sgpr(incUpper), src0=sgpr("WrapU%s+1"%tc), src1=0, + comment="incUpper <- ?")) + imod.add(self.incrementSrd(tP, sgpr(incLower), sgpr(incUpper), checkShadowLimitCopy=True)) + else: + if loopIdx != self.unrollIdx or (tc in ('A', 'B') and kernel["ProblemType"]["IndicesSummation"][self.unrollIdx] in kernel["ProblemType"]["MirrorDims%s"%tc]): + with self.allocTmpSgpr(1) as tmpSgprInfo: + incUpper = tmpSgprInfo.idx + # GRO may be negative for other summation if stride-other < stride-unroll or if mirror dim. + imod.add(SAShiftRightI32(dst=sgpr(incUpper), shiftHex=31, src=sgpr("GlobalReadIncs%s+%u"%(tc,loopIdx)), comment="sign-extend")) + else: + incUpper = 0 # GRO is positive for loop unroll + imod.add( self.incrementSrd(tP, sgpr("GlobalReadIncs%s+%u"%(tc,loopIdx)), incUpper)) + else: + graIdx = 0 + for _ in range(0, tP["nrp"]): + for _ in range(0, tP["nrpv"]): + for _ in range(0, tP["nrc"]): + for _ in range(0, tP["nrcv"]//tP["nrcvpi"]): + if self.globalReadIncsUseVgpr: + imod.add(VAddCOU32( \ + dst=vgpr("GlobalReadAddr%s+%u+0"%(tP["tensorChar"], graIdx)), \ + dst1=VCC(), \ + src0=vgpr("GlobalReadAddr%s+%u+0"%(tP["tensorChar"], graIdx)), \ + src1=vgpr("GlobalReadIncs%s+%u+0"%(tP["tensorChar"], 2*loopIdx)), \ + comment="gra += inc%s%s (lower)"%(tP["tensorChar"], loopChar))) + imod.add(VAddCCOU32( \ + dst=vgpr("GlobalReadAddr%s+%u+1"%(tP["tensorChar"], graIdx)), \ + dst1=VCC(), \ + src0=vgpr("GlobalReadAddr%s+%u+1"%(tP["tensorChar"], graIdx)), \ + src1=vgpr("GlobalReadIncs%s+%u+1"%(tP["tensorChar"], 2*loopIdx)), \ + src2=VCC(), \ + comment="gra += inc%s%s (upper)"%(tP["tensorChar"], loopChar))) + else: + imod.add(VAddCOU32( \ + dst=vgpr("GlobalReadAddr%s+%u+0"%(tP["tensorChar"], graIdx)), \ + dst1=VCC(), \ + src0=vgpr("GlobalReadAddr%s+%u+0"%(tP["tensorChar"], graIdx)), \ + src1=sgpr("GlobalReadIncs%s+%u"%(tP["tensorChar"], loopIdx)), \ + comment="gra += inc%s%s (lower)"%(tP["tensorChar"], loopChar))) + imod.add(VAddCCOU32( \ + dst=vgpr("GlobalReadAddr%s+%u+1"%(tP["tensorChar"], graIdx)), \ + dst1=VCC(), \ + src0=vgpr("GlobalReadAddr%s+%u+1"%(tP["tensorChar"], graIdx)), \ + src1=0, + src2=VCC(), \ + comment="gra += inc%s%s (upper)"%(tP["tensorChar"], loopChar))) + graIdx += self.rpga + #module.add(dump(vgpr("GlobalReadAddrA+0"))) + #module.add(dump(vgpr("GlobalReadAddrA+1"))) + #module.add(SEndpgm()) + + def globalReadIncrementAB(self, kernel, tPA, tPB, loopIdx, prefetchIndex): + imod = Module("globalReadIncrementAB%s") + + incCodeA = imod.add(Module("globalReadIncrementA")) + incCodeB = imod.add(Module("globalReadIncrementB")) + + self.globalReadIncrement(kernel, incCodeA, loopIdx, tPA, prefetchIndex) + self.globalReadIncrement(kernel, incCodeB, loopIdx, tPB, prefetchIndex) + return imod + + ############################################################################## + # Global Read: + # globalReadGuardK is called for loads in the tail loop + # Must ensure each load is in bounds - either using buffer bounds + # or exec-mask checks. + ############################################################################## + def globalReadGuardK(self, kernel, tP, vregSetIdx): + module = Module("globalReadGuardK") + tc = tP["tensorChar"] + problemType = self.kernel["ProblemType"] + graIdx = 0 + g2lIdx = 0 + loadWidth = tP["globalReadInstruction"].totalWidth + + ######################################## + # Calculate Max Addr + ######################################## + + if not kernel["BufferLoad"]: + with self.allocTmpSgpr(2) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + maxAddrSgpr = tmpSgpr + module.addComment0("flat addressing - max read address = size[n] * stride[n-1]") + dim = len(tP["ia"])-1 # dim + sizeIdx = tP["ia"][dim] + sizeIdxIsSum = sizeIdx in kernel["ProblemType"]["IndicesSummation"] + if sizeIdxIsSum: + sizeIdx -= kernel["ProblemType"]["NumIndicesC"] + # TODO-multiply by largest stride + module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(maxAddrSgpr+0), sgpr(maxAddrSgpr+1), \ + sgpr("Sizes%s+%u"%("Sum" if sizeIdxIsSum else "Free", sizeIdx)), \ + sgpr("Stride%s%s"%(tc, self.indexChars[tP['ia'][-1]])), \ + "64b tensor%s size in elements"%tc)) + module.add(SLShiftLeftB64(dst=sgpr(maxAddrSgpr,2), src=sgpr(maxAddrSgpr,2), \ + shiftHex=hex(log2(tP["bpe"])), comment="<- tensor%s size in bytes"%tc)) + + module.add(SAddU32( + dst=sgpr(maxAddrSgpr+0), \ + src0=sgpr(self.sgprs["AddressA"] if tP["isA"] else self.sgprs["AddressB"]), \ + src1=sgpr(maxAddrSgpr+0), \ + comment="prepend address lower")) + module.add(SAddCU32( + dst=sgpr(maxAddrSgpr+1), \ + src0=sgpr((self.sgprs["AddressA"] if tP["isA"] else self.sgprs["AddressB"])+1), \ + src1=sgpr(maxAddrSgpr+1), \ + comment="prepend address upper")) + # sgpr->vgpr + maxAddrVgpr = self.vgprPool.checkOutAligned(2, 2, "maxAddrVgpr") + module.add(VMovB32(dst=vgpr(maxAddrVgpr+0), src=sgpr(maxAddrSgpr+0), comment="sgpr->vgpr")) + module.add(VMovB32(dst=vgpr(maxAddrVgpr+1), src=sgpr(maxAddrSgpr+1), comment="sgpr->vgpr")) + + # full exec mask + fullExec = tmpSgpr + sgprCnt = self.laneSGPRCount + waveSize = kernel["WavefrontSize"] + activeMask = "0xFFFFFFFF" if (waveSize == 32) else "0xFFFFFFFFFFFFFFFF" + SMovBX = SMovB32 if (waveSize == 32) else SMovB64 + module.add(SMovBX(dst=sgpr(fullExec,sgprCnt), src=activeMask, comment="to restore all threads active")) + bpeVgpr = self.vgprPool.checkOut(1, "bpeVgpr") + module.add(VMovB32(dst=vgpr(bpeVgpr), src=hex(tP["bpe"]), comment="bpe")) + + # can remove this? + zeroVgpr = self.vgprPool.checkOut(1,"zeroVgpr") + module.add(VMovB32(dst=vgpr(zeroVgpr), src=hex(0), comment="zero")) + + + isGlc = True if tP["NonTemporal"]%2==1 else False + isSlc = True if tP["NonTemporal"]//2==1 else False + isLds = True if kernel["DirectToLds%s"%tc] else False + + directToLdsLoads = 0 + prevLdsOffset = 0 + # print("tc={}, nrp={}, nrpv={}, nrc={}, nrcv/nrcvpi={}, sgprforGRO={}".format(tc, tP["nrp"], tP["nrpv"], tP["nrc"], tP["nrcv"]//tP["nrcvpi"], problemType["ZeroPad%s"%tc], kernel["UseSgprForGRO"])) + + instOffset = 0 + loopCnt = -1 + + for perp in range(0, tP["nrp"]): + for sPerp in range(0, tP["nrpv"]): + for para in range(0, tP["nrc"]): + for sPara in range(0, tP["nrcv"]//tP["nrcvpi"]): + i = sPara + (tP["nrcv"] // tP["nrcvpi"]) * (para + tP["nrc"] * (sPerp + tP["nrpv"] * perp)) + loopCnt += 1 + graIdx = i * self.rpgo if kernel["BufferLoad"] else i * self.rpga + g2lIdx = i * loadWidth + + destVgprHi = None + dataIsI8 = False + packInt8Code = None + + instOffsetInc = 0 # increment value for instOffset. Need to apply after r loop + + r = 0 + numLoadVectorComp = loadWidth*self.bpr//tP["bpe"] + if kernel["ProblemType"]["DataType"].isDouble() and kernel["BufferLoad"]: + # adjustment for dgemm + BufferLoad + # use same buffer_load instruction for tail loop as out of tail loop + # this is mandatory for DirectToLds case. Also, it improves tail loop performance. + # so far, limit to double only + numLoadVectorComp = numLoadVectorComp // kernel["GlobalLoadVectorWidth%c"%tc] + + int8TempVgpr = numLoadVectorComp - 1 + # for each component in vector + while r < numLoadVectorComp: + numElementsPerLoad = 1 + if kernel["ProblemType"]["DataType"].isInt8(): + # TODO-Int8, Check this: + # if tP["glvw"]>1 and kernel["AssertSummationElementMultiple"] % 2 == 0: + # # Pack two FP16 values into a single load dword x2 + # numElementsPerLoad = 2 + # elif self.archCaps["HasEccHalf"]: + # destVgprHi = self.vgprPool.checkOut(1, 'destVgprHi') + + # Check out 3 regs once , for component 1,2,3 (r = 1,2,3) + if r == 1: + packInt8Code = Module() + destVgprHi = self.vgprPool.checkOut( int8TempVgpr , 'destVgprHi') + dataIsI8 = True + regIdx = r // 4 + elif kernel["ProblemType"]["DataType"].isHalf() or \ + kernel["ProblemType"]["DataType"].isBFloat16(): + if tP["glvw"]>1 and kernel["AssertSummationElementMultiple"] % 2 == 0: + # Pack two FP16 values into a single load dword x2 + numElementsPerLoad = 2 + elif self.archCaps["HasEccHalf"]: + # In some cards, loading half types into register will zero out + # the other half. Therefore we need to load into a separate register + # then pack 2 registers into one + destVgprHi = self.vgprPool.checkOut(1, 'destVgprHi') + regIdx = r // 2 + elif kernel["ProblemType"]["DataType"].isInt8x4() or \ + kernel["ProblemType"]["DataType"].isSingle(): + regIdx = r + elif kernel["ProblemType"]["DataType"].isDouble(): + numElementsPerLoad = kernel["GlobalLoadVectorWidth%c"%tc] # adjust numElementsPerLoad for DGEMM + regIdx = r*2 + elif kernel["ProblemType"]["DataType"].isSingleComplex(): + regIdx = r*2 + elif kernel["ProblemType"]["DataType"].isDoubleComplex() : + regIdx = r*4 + else: + printWarning("DataType unsupported") + module.addComment0("g2l=%u, load component %u"%(g2lIdx, r)) + + offset = 0 + + if kernel["BufferLoad"]: + # Use buffer limit to stay in-bounds - the limit was set to edge when SRD initialized + # and each increment of SRD base in the unroll loop does a corresponding decrement + # of the srd limit - so base+limit stays constant and also points at maximum + # element that should be accessed. + if kernel["_UseSgprForGRO"]: + offsetVgpr = "GlobalReadOffset%s+0"%(tc) + else: + offsetVgpr = "GlobalReadOffset%s+%u"%(tc, graIdx) + + # Vgpr for GRO + if not kernel["_UseSgprForGRO"]: + soffset = "0" + # instruction offset with Sgpr for GRO + elif kernel["DirectToLds%s"%tc] and kernel["UseInstOffsetForGRO"]: + soffset = sgpr("ScalarGlobalReadOffset%s+%u"%(tc, graIdx)) + # Sgpr for GRO + else: + soffset = "0" if graIdx == 0 else sgpr("ScalarGlobalReadOffset%s+%u"%(tc, graIdx-1)) + + unrollMirrorWithSoffset = kernel["ProblemType"]["IndicesSummation"][self.unrollIdx] in problemType["MirrorDims%s"%tc] and soffset != "0" + # ScalarGlobalReadOffset should be negative value with unroll mirroring. + # However, buffer_load uses soffset as uint value, so GRO - SGRO, SGRO = 0 + if unrollMirrorWithSoffset: + codeMod = Module("mirrorIdx%u"%loopCnt) + codeMod.add(VSubU32(dst=vgpr(offsetVgpr), src0=vgpr(offsetVgpr), src1=soffset, comment="mirror unroll: GRO=GRO-SGRO, soffset=0")) + module.add(codeMod) + soffset_prev = soffset + soffset = "0" + + if kernel["DirectToLds%s"%tc]: + # need to increment ldsInc only once per each loopCnt + # this is pre count up, so increment it at r == 0 + if r == 0: + ldsInc = (self.kernel["WavefrontSize"] if kernel["WaveSeparateGlobalRead%c"%tc] else kernel["NumThreads"]) * kernel["GlobalLoadVectorWidth%c"%tc] * tP["bpe"] + else: + ldsInc = 0 + if kernel["LdsBlockSizePerPad%s"%tc] != 0: + ldsInc += (ldsInc // kernel["LdsBlockSizePerPad%s"%tc]) * kernel["LdsPad%s"%tc] * tP["bpe"] + else: + padInterval = (self.kernel["WavefrontSize"] if kernel["WaveSeparateGlobalRead%c"%tc] else kernel["NumThreads"]) * self.bpr + ldsInc += (ldsInc // padInterval) * kernel["LdsPad%s"%tc] * tP["bpe"] + #print("ldsInc", ldsInc) + #print("GlobalLoadVectorWidth", kernel["GlobalLoadVectorWidth%c"%tc]) + #print("bpr", self.bpr) + if kernel["UseInstOffsetForGRO"]: + # buffer_load only support 12 bit instruction offset + # we have to increase m0 if offset is larger thant 12 bits + if instOffset >= self.buff_load_inst_offset_max: + inc = (instOffset // self.buff_load_inst_offset_max) * self.buff_load_inst_offset_max + module.add(SAddU32(dst=mgpr(0), src0=mgpr(0), src1=inc, comment="Move LDS write address to next base" )) + instOffset -= inc + elif directToLdsLoads != 0 and ldsInc > 0: + if tP["nrc"] > 1: + # another address conversion for DirectToLds + NumLoadsCoalesced > 1 + divisorName = tP["lvc"] + divisor = kernel[divisorName] + # DirectToLds + NumLoadsCoalesced>1 case, need to adjust m0 increment value to store values to correct location in LDS + wSize = max(self.kernel["WavefrontSize"], divisor) + lscaOffset = para * wSize * tP["bpe"] * tP["glvw"] + ldsOffset = ldsInc * tP["nrc"] * (sPerp + tP["nrpv"] * perp) + lscaOffset + ldsInc = ldsOffset - prevLdsOffset + prevLdsOffset = ldsOffset + module.add(SAddU32(dst=mgpr(0), src0=mgpr(0), src1=ldsInc, comment="Move LDS write address to next line" )) + + destVgpr=0 + elif kernel["DirectToVgpr%s"%tc]: + numVgprG2L = self.numVgprG2LA if tP["isA"] else self.numVgprG2LB + numVgprPerBlock = numVgprG2L // 2 # numVgprG2L is doubled for DirectToVgpr + idx = g2lIdx + vregSetIdx * numVgprPerBlock + destVgpr="G2L%s+%u+%u"%(tc, idx, regIdx) + else: + destVgpr="G2L%s+%u+%u"%(tc, g2lIdx, regIdx) + + offset = r * tP["bpe"] + instOffset + hi8 = 0 + hi16 = 0 + comment = "load one buffer value" + if kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16(): + if numElementsPerLoad==2: + # Pack two FP16 values into a single load dword x2 + r += 1 # skip next element since we loaded 2X here + comment = "load packed 2X half buffer value" + elif not kernel["DirectToLds%s"%tc]: + hi16=loopCnt%2 if tP["glvw"]==1 else r%2 + comment="load one buffer value" + + if kernel["ProblemType"]["DataType"].isInt8(): + # TODO-Int8, Check this: + # if numElementsPerLoad==2: + # # Pack two FP16 values into a single load dword x2 + # r += 1 # skip next element since we loaded 2X here + # comment = "load packed 2X half buffer value" + if not kernel["DirectToLds%s"%tc]: + hi8 = (loopCnt%4) %2 if tP["glvw"]==1 else (r%4) %2 + hi16 = (loopCnt%4)//2 if tP["glvw"]==1 else (r%4)//2 + comment="load one buffer value" + + bpl = numElementsPerLoad*self.bpeAB # bytesPerLoad + + # if hi8=1 or hi16=1 (component 1,2,3 for int8) or (component 1 for half), use the temp destVgprHi + # but only when hi16=1 we use the _d16_hi version instruction, see the below visualized int8 comment + loadVgpr = destVgprHi if ((hi16 or hi8) and destVgprHi != None) else destVgpr + if kernel["ProblemType"]["DataType"].isInt8() and (not self.archCaps["HasEccHalf"]): + module.add(VMovB32(dst=vgpr(loadVgpr), src=0, comment="set to zero to avoid unexpected value")) + module.add(self.chooseGlobalRead(True, \ + bpl, destVgpr=loadVgpr, \ + addr0=vgpr(offsetVgpr), addr1=sgpr("Srd%s"%tc, 4), \ + soffset=soffset, offset=offset, \ + glc=isGlc, slc=isSlc, lds=isLds, \ + hi16=hi16, \ + comment=comment)) + + if unrollMirrorWithSoffset: + codeMod = Module("mirrorIdx%u"%loopCnt) + codeMod.add(VAddU32(dst=vgpr(offsetVgpr), src0=vgpr(offsetVgpr), src1=soffset_prev, comment="mirror unroll: restore GRO=GRO+SGRO")) + module.add(codeMod) + + if kernel["DirectToLds%s"%tc] and kernel["UseInstOffsetForGRO"]: + instOffsetInc += ldsInc + # print(" bpl={}, destVgpr={}, soffset={}, offset={}, hi16={}".format(bpl, destVgpr, soffset, offset, hi16)) + + else: # Not buffer load, ie 'flat' load + # mask if current address if in bounds + module.add(VCmpXLtU64(dst=VCC(), \ + src0=vgpr("GlobalReadAddr%s+%u"%(tP["tensorChar"], graIdx),2), \ + src1=vgpr(maxAddrVgpr,2), \ + comment="addr < maxAddr")) + hi16=(kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16()) and r%2==1 + destVgpr="G2L%s+%u+%u"%(tc, g2lIdx, regIdx) + # load one element from address + module.add(self.chooseGlobalRead(False, \ + self.bpeAB, destVgpr=destVgprHi if (hi16 and destVgprHi != None) else destVgpr, \ + addr0=vgpr("GlobalReadAddr%s+%u"%(tc,graIdx),2), addr1="", \ + soffset=0, offset=0, \ + glc=isGlc, slc=isSlc, lds=isLds, \ + hi16=hi16, \ + comment="load one flat value")) + + # restore full exec mask + SOrSaveExecBX = SOrSaveExecB32 if self.kernel["WavefrontSize"] == 32 else SOrSaveExecB64 + module.add(SOrSaveExecBX(dst=VCC(), src=sgpr(fullExec,self.laneSGPRCount), comment="all threads active")) + + # increment address by 1 element (BPE) + module.add(VAddCOU32( + dst=vgpr("GlobalReadAddr%s+%u+0"%(tP["tensorChar"], graIdx)), \ + dst1=VCC(), \ + src0=vgpr("GlobalReadAddr%s+%u+0"%(tP["tensorChar"], graIdx)), \ + src1=vgpr(bpeVgpr), comment="gra += 1 (lower)")) + module.add(VAddCCOU32( + dst=vgpr("GlobalReadAddr%s+%u+1"%(tP["tensorChar"], graIdx)), \ + dst1=VCC(), \ + src0=vgpr("GlobalReadAddr%s+%u+1"%(tP["tensorChar"], graIdx)), \ + src1=vgpr(zeroVgpr), \ + src2=VCC(), comment="gra += 1 (upper)")) + + # int8 byte: + # |--------|--------|--------|---V0---|, r = 0, hi8=0, hi16=0, load d16 + # |--------|--------|--------|---V1---|, r = 1, hi8=1, hi16=0, load d16 + # |--------|---V2---|--------|--------|, r = 2, hi8=0, hi16=1, load d16_hi + # |--------|---V3---|--------|--------|, r = 3, hi8=1, hi16=1, load d16_hi + # V1, V3 -> shift left 8 bits, or 4 regs (pack) + # DestV0|=(V1 << 8), DestV0|= V2, DestV0|=(V3 << 8) + # Int8 (byte) + if dataIsI8 and (destVgprHi != None): + # hi8 -> r = 1,3 + # hi16 -> r = 2,3 + if hi8 or hi16: + # r = 1,2,3, vmcnt needed for one packing + packInt8Code.add(SWaitCnt(vmcnt=(int8TempVgpr-r), comment="")) + if hi8: + # r = 1,3, shift needed + packInt8Code.add(VLShiftLeftB32(dst=vgpr(destVgprHi), shiftHex=hex(0x8), src=vgpr(destVgprHi), comment="shift left to higher 8 bits")) + if hi8 or hi16: + # r = 1,2,3, packing + packInt8Code.add(VOrB32(dst=vgpr(destVgpr), src0=vgpr(destVgpr), src1=vgpr(destVgprHi), comment="pack a sub 8-bit with dest")) + destVgprHi += 1 + + # Half + elif destVgprHi != None and r % 2 == 1: + module.add(SWaitCnt(vmcnt=0, comment="")) + module.add(VOrB32(dst=vgpr(destVgpr), src0=vgpr(destVgpr), src1=vgpr(destVgprHi), comment="HasEccHalf: pack")) + + # For half (bf16). Note: for int8, we will checkin after loading all components + if (destVgprHi != None) and (not dataIsI8): + self.vgprPool.checkIn(destVgprHi) + destVgprHi = None + + r += 1 # next component (for half, byte) + + # end R loop + + instOffset += instOffsetInc # add increment value for instOffset. Need to apply after r loop + # increment once per r loop (at the end) + directToLdsLoads+=1 + + # for int8: + # we do the 3 packs, and checking the 3 extra vgprs after loading all components + if dataIsI8: + assert packInt8Code != None and destVgprHi != None + module.add(packInt8Code) + self.vgprPool.checkIn(destVgprHi - int8TempVgpr) + destVgprHi = None + + if self.db["ConservativeWaitCnt"] & 0x1: + module.add(SBarrier(comment="debug")) + module.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="")) + module.add(SBarrier(comment="debug")) + #module.add(self.getCmpAssert(self.asmAssert.lt, vgpr("Serial"), 64)) # examine second wavefront + + # TODO - can remove one of these m0 restores if A and B both TLU + if kernel["DirectToLds%s"%tP["tensorChar"]]: + module.add(SMovB32(dst=mgpr(0), src=hex(kernel["LdsNumElements"] * tP["bpe"]), \ + comment="Restore LDS clamp at %u bytes"%(kernel["LdsNumElements"] * tP["bpe"]))) + + if not kernel["BufferLoad"]: + self.vgprPool.checkIn(maxAddrVgpr) + self.vgprPool.checkIn(bpeVgpr) + self.vgprPool.checkIn(zeroVgpr) + + return module + + ############################################################################## + # DirectToLds M0 update: Do It A/B + ############################################################################## + def directToLdsM0Update(self, kernel, mode, tP, usePlaceHolder=False): + tc = tP["tensorChar"] + imod = Module("directToLdsM0Update%s_%u"%(tc,mode)) + DtldsModule = imod.add(Module("dtls_offset%s"%tP["tensorChar"])) + if not self.do["GlobalRead%s"%tP["tensorChar"]]: return imod + if kernel["DirectToLds%s"%tc]: + # DirectToLds only enabled for TLU=1 cases, where the registers are directly copied into LDS + # for cases both A&B are DTLS, updating m0 for each GlobalRead requires instruction schedule + # along with global reads + assert (kernel["LocalWriteUseSgpr%s"%tc]) + if kernel["ExpandPointerSwap"]: + DtldsModule.add(SAddU32(dst=mgpr(0), src0=sgpr("LocalWriteAddr%s"%tc), \ + src1=tP["localWriteSwapByteOffset"], comment="m0 <- LDS write address")) + else: + DtldsModule.add(SMovB32(dst=mgpr(0), src=sgpr("LocalWriteAddr%s"%tc), comment="m0 <- LDS write address")) + + # PrefetchGlobalRead=2 case, generate local read wait for DirectToLds + if kernel["PrefetchGlobalRead"]==2: + # do not generate local read wait for PGR=2 + DtldsModule.addComment0("before DirectToLds load, ensure prior ds_reads have finished") + DtldsModule.add(SWaitCnt(lgkmcnt=0, comment="")) + if not kernel["NoLdsWriteCode"]: + if usePlaceHolder: + waitStr = Holder(idx=0) + else: + waitStr = 0 + DtldsModule.add(SWaitCnt(vmcnt=waitStr, comment="")) + DtldsModule.add(SBarrier()) + + return imod + + ############################################################################## + # Global Read: Do It A/B + ############################################################################## + def globalReadDo(self, kernel, mode, tP, vregSetIdx=0): + tc = tP["tensorChar"] + problemType = self.kernel["ProblemType"] + imod = StructuredModule("globalReadDo%s_%u"%(tc,mode)) + if not self.do["GlobalRead%s"%tP["tensorChar"]]: return imod + + # sizeK % LOCAL_DEPTHU + guardK = (mode==2) + + graIdx = 0 + g2lIdx = 0 + loadWidth = tP["globalReadInstruction"].totalWidth # load width in elements? + bpl = self.bpeAB * tP["glvw"] # bytes per load + instOffset = 0 + + loopIdx = self.unrollIdx # TODO - does this handle multiple summation indices? + if kernel["SuppressNoLoadLoop"]: + if mode==1 and tP["isA"]: + imod.header.add(SCmpEQI32( + src0=self.loopCounter(kernel, loopIdx), \ + src1=1, \ + comment="%s"%"is this the last iteration")) + imod.header.add(SCMovB32( + dst=sgpr("SrdA+2"), src=0, + comment="Set limit to 0 for last iteration")) + imod.header.add(SCMovB32( + dst=sgpr("SrdB+2"), src=0, + comment="Set limit to 0 for last iteration")) + + # set the first tc for below wait code for DirectToLds + # if DirectToVgprA is enabled, change the first to B + tc1st = 'A' + if kernel["DirectToVgprA"]: + tc1st = 'B' + + if tc == tc1st and (kernel["DirectToLdsA"] or kernel["DirectToLdsB"]) and not kernel["PrefetchGlobalRead"]==2: + # generate local read wait for DirectToLds except for PrefetchGlobalRead=2 (for PGR=2, generate wait after m0 value setting) + imod.header.addComment0("before DirectToLds load, ensure prior ds_reads have finished") + if (kernel["DirectToVgprA"] or kernel["DirectToVgprB"]): # do not generate sync here if DirectToVgpr is enabled + imod.header.add(SWaitCnt(lgkmcnt=0, comment="")) + else: + imod.header.add(self.syncThreads(kernel)) + + + if guardK: + imod.middle.add(self.globalReadGuardK(kernel, tP, vregSetIdx)) + return imod + + # else not-guardK below: + + isGlc = True if tP["NonTemporal"]%2==1 else False + isSlc = True if tP["NonTemporal"]//2==1 else False + isLds = True if kernel["DirectToLds%s"%tc] else False + + directToLdsLoads = 0 + instOffset = 0 + prevLdsOffset = 0 + + loopCnt = -1 + for perp in range(0, tP["nrp"]): + for sPerp in range(0, tP["nrpv"]): + for para in range(0, tP["nrc"]): + for sPara in range(0, tP["nrcv"]//tP["nrcvpi"]): + i = sPara + (tP["nrcv"]//tP["nrcvpi"]) * (para + tP["nrc"] * (sPerp + tP["nrpv"] * perp)) + loopCnt += 1 + graIdx = i * self.rpgo if kernel["BufferLoad"] else i * self.rpga + g2lIdx = i * loadWidth + # Each load may contains a small bundle of instructions, package them together in loadModule: + loadModule = Module("load%u"%loopCnt) + imod.middle.add(loadModule) + + if kernel["BufferLoad"]: + if kernel["_UseSgprForGRO"]: + offsetVgpr= "GlobalReadOffset%s+0"%(tc) + else: + offsetVgpr= "GlobalReadOffset%s+%u"%(tc, graIdx) + + # vgpr for GRO + if not kernel["_UseSgprForGRO"]: + soffset = "0" + # instruction offset with Sgpr for GRO + elif kernel["DirectToLds%s"%tc] and kernel["UseInstOffsetForGRO"]: + soffset = sgpr("ScalarGlobalReadOffset%s+%u"%(tc, graIdx)) + # Sgpr for GRO + else: + soffset = "0" if graIdx == 0 else sgpr("ScalarGlobalReadOffset%s+%u"%(tc, graIdx-1)) + + unrollMirrorWithSoffset = kernel["ProblemType"]["IndicesSummation"][self.unrollIdx] in problemType["MirrorDims%s"%tc] and soffset != "0" + # ScalarGlobalReadOffset should be negative value with unroll mirroring. + # However, buffer_load uses soffset as uint value, so GRO - SGRO, SGRO = 0 + if unrollMirrorWithSoffset: + codeMod = Module("mirrorIdx%u"%loopCnt) + codeMod.add(VSubU32(dst=vgpr(offsetVgpr), src0=vgpr(offsetVgpr), src1=soffset, comment="mirror unroll: GRO=GRO-SGRO, soffset=0")) + loadModule.add(codeMod) + soffset_prev = soffset + soffset = "0" + + if kernel["DirectToLds%s"%tc]: + # use bpe with GlobalLoadVectorWidth + ldsInc = (self.kernel["WavefrontSize"] * kernel["GlobalLoadVectorWidth%c"%tc] if kernel["WaveSeparateGlobalRead%c"%tc] else kernel["NumThreads"] * kernel["GlobalLoadVectorWidth%c"%tc]) * tP["bpe"] + if kernel["LdsBlockSizePerPad%s"%tc] != 0: + ldsInc += (ldsInc // kernel["LdsBlockSizePerPad%s"%tc]) * kernel["LdsPad%s"%tc] * tP["bpe"] + else: + padInterval = (self.kernel["WavefrontSize"] if kernel["WaveSeparateGlobalRead%c"%tc] else kernel["NumThreads"]) * self.bpr + ldsInc += (ldsInc // padInterval) * kernel["LdsPad%s"%tc] * tP["bpe"] + + if kernel["UseInstOffsetForGRO"]: + # buffer_load only support 12 bit instruction offset + # we have to increase m0 if offset is larger thant 12 bits + if instOffset >= self.buff_load_inst_offset_max: + inc = (instOffset // self.buff_load_inst_offset_max) * self.buff_load_inst_offset_max + loadModule.add(SAddU32(dst=mgpr(0), src0=mgpr(0), src1=inc, comment="Move LDS write address to next base" )) + instOffset -= inc + elif directToLdsLoads != 0: + # m0 offset conversion (only for UseInstOffsetForGRO == 0) + # in tP["glvw"] == 1 and tP["nrc"] > 1 case, only m0 offset conversion is necessary. row and column index conversion is not necessary. + if tP["nrc"] > 1: + # another address conversion for DirectToLds + NumLoadsCoalesced > 1 + divisorName = tP["lvc"] + divisor = kernel[divisorName] + # DirectToLds + NumLoadsCoalesced>1 case, need to adjust m0 increment value to store values to correct location in LDS + wSize = max(self.kernel["WavefrontSize"], divisor) + lscaOffset = para * wSize * tP["bpe"] * tP["glvw"] + ldsOffset = ldsInc * tP["nrc"] * (sPerp + tP["nrpv"] * perp) + lscaOffset + ldsInc = ldsOffset - prevLdsOffset + prevLdsOffset = ldsOffset + loadModule.add(SAddU32(dst=mgpr(0), src0=mgpr(0), src1=ldsInc, comment="Move LDS write address to next line" )) + directToLdsLoads+=1 + destVgpr=0 + elif kernel["DirectToVgpr%s"%tc]: + # DirectToVgpr case. Need to toggle destination vreg set and adjust instOffset + destVgpr="G2L%s%u+%u"%(tc, vregSetIdx, g2lIdx) + else: + destVgpr="G2L%s+%u"%(tc, g2lIdx) + + # TODO: is it possible to load only hi16 when no in tail? (need to check INT8 too) + loadModule.add( self.chooseGlobalRead(kernel["BufferLoad"], \ + bpl, destVgpr=destVgpr, \ + addr0=vgpr(offsetVgpr), addr1=sgpr("Srd%s"%tc, 4), \ + soffset=soffset, offset=instOffset, \ + glc=isGlc, slc=isSlc, lds=isLds, \ + hi16=(kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16()) and loopCnt%2==1, \ + comment="G -> Reg %u_%u_%u_%u"%(para, sPara, perp, sPerp))) + + if unrollMirrorWithSoffset: + codeMod = Module("mirrorIdx%u"%loopCnt) + codeMod.add(VAddU32(dst=vgpr(offsetVgpr), src0=vgpr(offsetVgpr), src1=soffset_prev, comment="mirror unroll: restore GRO=GRO+SGRO")) + loadModule.add(codeMod) + + if kernel["DirectToLds%s"%tc] and kernel["UseInstOffsetForGRO"]: + instOffset += ldsInc + + #print "IM=", type(imod.instList[-1]), imod.instList[-1], + else: # not buffer load + # load one element from address + if kernel["DirectToVgpr%s"%tc]: + # DirectToVgpr case. Need to toggle destination vreg set and adjust instOffset + destVgpr="G2L%s%u+%u"%(tc, vregSetIdx, g2lIdx) + else: + destVgpr="G2L%s+%u"%(tc, g2lIdx) + loadModule.add( self.chooseGlobalRead(False, \ + bpl, \ + destVgpr=destVgpr, \ + addr0=vgpr("GlobalReadAddr%s+%u"%(tc,graIdx),2), addr1="", \ + soffset=0, offset=0, \ + glc=isGlc, slc=isSlc, lds=isLds, \ + hi16=(kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16()) and loopCnt%2==1, \ + comment="G -> Reg %u_%u_%u_%u"%(para, sPara, perp, sPerp ))) + + if self.db["ConservativeWaitCnt"] & 0x1: + imod.footer.add(SBarrier(comment="debug")) + imod.footer.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="conservative wait")) + imod.footer.add(SBarrier(comment="debug")) + #module.add(self.getCmpAssert(self.asmAssert.lt, vgpr("Serial"), 64)) # examine second wavefront + + # TODO - can remove one of these m0 restores if A and B both TLU + if kernel["DirectToLds%s"%tP["tensorChar"]] and not (mode == 1 and kernel["PrefetchGlobalRead"]==2): + dst = mgpr(0) + src = hex(kernel["LdsNumElements"] * tP["bpe"]) + comment = "Restore LDS clamp at %u bytes"%(kernel["LdsNumElements"] * tP["bpe"]) + # PGR=2 case, footer is located before global read. To avoid setting clamp before global read, store lds clamp code in middle + if kernel["PrefetchGlobalRead"] == 2: + imod.middle.add(SMovB32(dst=dst, src=src, comment=comment)) + else: + imod.footer.add(SMovB32(dst=dst, src=src, comment=comment)) + + return imod + + ############################################################################## + # Local Write: Swap Offsets A/B + ############################################################################## + def localWriteSwapOffsets(self, kernel, internalPointerSwap, tP): + if not self.do["LocalWrite"]: return Module("localWriteSwapOffsets (No local write)") + if kernel["1LDSBuffer"]: return Module("localWriteSwapOffsets (Empty)") + module = Module("localWriteSwapOffsets") + tc = tP["tensorChar"] + #fixme-iui need to use wrapping increment for double or triple buffering: + if internalPointerSwap: + tP["localWriteSwapByteOffset"] = 0 if tP["localWriteSwapByteOffset"] else kernel["LdsOffsetA_Blk"]*tP["bpe"] + module.addComment1("(EPS=1) local write swap internal offset -> %u" % tP["localWriteSwapByteOffset"]) + else: + if kernel["LocalWriteUseSgpr%s"%tc]: + module.add(SXorB32( + dst=sgpr("LocalWriteAddr%s"%tP["tensorChar"]), \ + src0=hex(kernel["LdsOffsetA_Blk"]*tP["bpe"]), \ + src1=sgpr("LocalWriteAddr%s"%tP["tensorChar"]), \ + comment="swap Red Blk SGPR")) + elif not kernel["DirectToVgpr%s"%tc]: # no local write code if DirectToVgpr is enabled + numLwa = self.numVgprLocalWriteAddressesA if tP["isA"] else self.numVgprLocalWriteAddressesB + for i in range(0,numLwa): + module.add(VXorB32( + dst=vgpr("LocalWriteAddr%s+%u"%(tc,i)), \ + src0=hex(kernel["LdsOffsetA_Blk"]*tP["bpe"]), \ + src1=vgpr("LocalWriteAddr%s+%u"%(tc,i)), \ + comment="swap Red Blk")) + return module + + ############################################################################## + # Local Write: Reset Offsets A/B + # used for global-read + tail-loop to reset to writing in red + ############################################################################## + def localWriteResetOffsets(self, kernel, internalPointerSwap, tP): + tc = tP["tensorChar"] + if not self.do["LocalWrite"]: return Module("localWriteResetOffsets (no local write)") + if kernel["1LDSBuffer"] or kernel["DirectToVgpr%s"%tc]: # no local write code if DirectToVgpr is enabled + return Module("localWriteResetOffsets (Empty)") + module = Module("localWriteResetOffsets") + resetMask = hex(kernel["LdsOffsetA_Blk"]*tP["bpe"]-1 | self.LdsOOB) + if internalPointerSwap: + tP["localWriteSwapByteOffset"] = 0 + else: + if kernel["LocalWriteUseSgpr%s"%tc]: + module.add(SAndB32( + dst=sgpr("LocalWriteAddr%s"%tP["tensorChar"]), \ + src0=resetMask, \ + src1=sgpr("LocalWriteAddr%s"%tP["tensorChar"]), \ + comment="reset to Red")) + else: + module.add(VAndB32( + dst=vgpr("LocalWriteAddr%s"%tP["tensorChar"]), \ + src0=resetMask, \ + src1=vgpr("LocalWriteAddr%s"%tP["tensorChar"]), \ + comment="reset to Red")) + return module + + ############################################################################## + # Calculate offset to use for LDS write + # Intro: + # Each WI has a 2D tile index (coal, perp). + # - Code above computes global mem address by scaling one dim by the + # lda and adding the other. + # - Here we compute a linear LDS offset by scaling one dim by the MT + # dim and adding the other. + # Result is we map a tile from global memory into LDS. Consecutive LDS + # locations contain elements from different summation 'rows' - therefore + # loading a row of LDS will feed computations for different C tile indices. + # Notes: + # Total load insts is nrc * nrp which load the macro-tile. + # Par and coalesced are ~synonyms referring to same dimension + # Either nrpv or nrvc must be 1 - can't have vectors in both dimensions. + # Thus either sPerp or sPara is 0. + # Inputs: + # perp : index of the load in perp dimension (0...nrp) + # par : index of the load in the para dim (0...nrc) + # sPerp : component index of the perp vector (0...nrpv) + # sPara : component index of the par vector (0...nrcv) + # Outputs: + # offsetBytes : Offset in bytes for the _ds_store instruction + # i : i-th instruction + # comment : Comment with the text version of the formula + ############################################################################# + def calculateLdsWriteOffset(self, perp, para, sPerp, sPara, kernel, tP): + tc = tP["tensorChar"] + mask = 0 + #print "tc ", tc, " perp ", perp, " para ", para, " sPerp ", sPerp, " sPara ", sPara + lscaOffset = para * kernel[tP["lsc"]] + perp_masked = perp + perp_rem = 0 + lspaOffset = perp_masked * kernel[tP["lsp"]] + rem = 0 + + # Add component offset to interleave from different regs + # and compute mysterious "i" + assert(sPerp==0 or sPara==0) + + if tP["tlu"] != kernel["UnrollMajorLDS%s" % tP["tensorChar"]]: + lspaOffset += sPerp & mask + lscaOffset += sPara + rem = (sPerp & ~mask) + i = sPara + (tP["nrcv"]//tP["nrcvpi"]) * (para + tP["nrc"] * (sPerp + tP["nrpv"] * perp_masked)) + #print "nrcv ", tP["nrcv"], " nrcvpi ", tP["nrcvpi"], " nrc ", tP["nrc"], " nrpv ", tP["nrpv"] + else: + lscaOffset += sPara + lspaOffset += sPerp + rem = 0 + i = sPara + (tP["nrcv"]//tP["nrcvpi"]) * (para * tP["glvw"] + tP["nrc"] * (sPerp + tP["glvw"] * tP["nrpv"] * perp )) + + #if not tP["tlu"]: + # tmp = sPara + # sPara = sPerp + # sPerp = tmp + # print("0lspaOffset", lspaOffset) + # print("0lscaOffset", lscaOffset) + + LdsPad = kernel["LdsPad%s"%tc] if kernel["LdsBlockSizePerPad%s"%tc] == 0 else 0 + lds_stride = (kernel["_DepthULds"] + LdsPad) if kernel["UnrollMajorLDS%s" % tP["tensorChar"]] \ + else (kernel[tP["mt"]] + LdsPad) + + if tP["tlu"] != kernel["UnrollMajorLDS%s" % tP["tensorChar"]]: + lspaOffset *= lds_stride + lspaOffset += rem + perp_rem + else: + lscaOffset *= lds_stride + lscaOffset += rem + + # print("1lspaOffset", lspaOffset) + # print("1lscaOffset", lscaOffset) + #if tP["tlu"]: + # lspaOffset *= tP["glvw"] + # lscaOffset *= tP["glvw"] + + # print("2lspaOffset", lspaOffset) + # print("2lscaOffset", lscaOffset) + offsetElements = (lspaOffset + lscaOffset) + # print("offsetElements", offsetElements) + offsetBytes = offsetElements*tP["bpe"] + + if kernel["LdsBlockSizePerPad%s"%tc] != 0 and kernel["LdsPad%s"%tc] != 0: + offsetBytes = offsetBytes + (offsetBytes // kernel["LdsBlockSizePerPad%s"%tc]) * kernel["LdsPad%s"%tc] * tP["bpe"] + + offsetBytes += tP["localWriteSwapByteOffset"] + + #print("offsetBytes", offsetBytes) + #print "offset", offset + + comment = "lwo%s_%u_%u_%u_%u = (%s%d*%s)" \ + % (tP["tensorChar"], \ + para, sPara, perp, sPerp, \ + (("%u + "%sPara) if tP["wtc"] else ""), \ + para, tP["lsc"] ) + if not tP["tlu"]: + comment += "*(MT%s+PAD)" % (tP["tileChar"]) + comment += " + (%d*%s)" % (perp, tP["lsp"]) + if tP["tlu"]: + comment += "(*MT%s+PAD)" % (tP["tileChar"]) + comment += " = %u" % (offsetBytes) + + return (offsetBytes, i, comment) + + def recalcLocalWriteAddresses(self, kernel, tP, uDu): + + tc = tP["tensorChar"] + + module = Module("recalcLocalWriteAddresses") + module.addComment1("recalculate LocalWriteAddr{}".format(tc)) + + lwvw = getattr(self, "localWriteWidth{}".format(tc)) + newInstIdx = self.selectMemoryInstruction("LocalWrite", lwvw*kernel["DepthULdsDivisor"], \ + False, \ + tP["localWrite2Coalesced"], tP["localWrite2Perpendicular"], + [tP["localWriteStrideTile"], tP["localWriteStrideUnroll"]] ) + tP["localWriteInstruction"] = self.memoryInstructions["LocalWrite"][newInstIdx] + + # global read tile assignment + module.add(self.graTileAssignment(kernel, tP)) + # global read tile offsets + module.add(self.graTileOffsets(kernel, tP)) + # global read unroll offsets + module.add(self.graUnrollOffsets(kernel, tP)) + # still needed for vgpr resource management + # intentionally not emitting code + self.graFinalOffsets(kernel, tP) + + # local write tile assignments + module.add(self.lwaTileAssignment(tP)) + # local write unroll assignments + module.add(self.lwaUnrollAssignment(kernel, tP)) + # local write local write first offsets + module.add(self.lwaFirstOffset(kernel, tP, uDu)) + + return module + + def recalcLocalReadAddressesAB(self, kernel, tPA, tPB): + imod = Module() + + if self.inTailLoop: + # it do 1 iteration each loop in tail loop, and is no use to wider local read next iteration. + # In 1 block MI, it remap localReadAddr in order to let each thread wider local read continuous k + # this decrease performance since it require more loop to handle continuous k in each thread. + # recalculate localReadAddr to cancel wider local read in tail loop + # TODO: If DepthULdsDivisor>1, local read addr is incremented for each K the loop iterates, which + # upon second sub-loop needs to be reset to its original value. Backing up local read address would + # be nicer than recomputing them + if kernel.enabledSplitLDS or ((self.numReadsIterCoalescedA > 1 or self.numReadsIterCoalescedB > 1) and kernel["MatrixInstB"] == 1): #and tP["isB"]: + self.numReadsIterCoalescedA = 1 + self.numReadsIterCoalescedB = 1 + self.lrvwA = kernel["MIInputPerThread"] + self.lrvwB = kernel["MIInputPerThread"] + + imod.add(self.lraTileAssignment(kernel, tPA, tPB)) + imod.add(self.lraFinalOffset(kernel, tPA)) + imod.add(self.lraDeclareAddresses(kernel, tPA)) + imod.add(self.lraFinalOffset(kernel, tPB)) + imod.add(self.lraDeclareAddresses(kernel, tPB)) + localRead2Perpendicular = False + instructions = self.memoryInstructions + + localReadWidth = tPA["bpe"] / self.bpr + if kernel["UnrollMajorLDSA"]: + localReadWidth = (kernel["MIInputPerThread"] * tPA["bpe"]) // self.bpr + localReadInstructionIdxA = \ + self.selectMemoryInstruction("LocalRead", localReadWidth, \ + False, \ + tPA["localRead2Coalesced"], localRead2Perpendicular, + [tPB["localReadStrideCoalesced"]] ) + tPA["localReadInstruction"] = instructions["LocalRead"][localReadInstructionIdxA] + + localReadWidth = tPB["bpe"] / self.bpr + if kernel["UnrollMajorLDSB"]: + localReadWidth = (kernel["MIInputPerThread"] * tPB["bpe"]) // self.bpr + localReadInstructionIdxB = \ + self.selectMemoryInstruction("LocalRead", localReadWidth, \ + False, \ + tPB["localRead2Coalesced"], localRead2Perpendicular, + [tPB["localReadStrideCoalesced"]] ) + tPB["localReadInstruction"] = instructions["LocalRead"][ \ + localReadInstructionIdxB] + + return imod + + ############################################################################## + # Local Write in Prefetch Pass (PreLoop): Do It A/B + ############################################################################## + def preLoopLocalWriteDo(self, kernel, tPA, tPB): + imod = Module() + + LWDoMod = imod.add(Module()) + LWDoA = self.localWriteDo(kernel, tPA) + LWDoB = self.localWriteDo(kernel, tPB) + LWDoMod.addComment1("local write a") + LWDoMod.add(LWDoA) + LWDoMod.addComment1("local write b") + LWDoMod.add(LWDoB) + return imod + + ############################################################################## + # Local Write: Do It A/B + # uDu: 'None' means to use fractional local write (where not all threads are active) + # when DepthULdsDivisor > 1 + ############################################################################## + def localWriteDo(self, kernel, tP, uDu=0): + if not self.do["LocalWrite"]: return "", -1 + + tc = tP["tensorChar"] + imod = Module() + + if (not kernel["DirectToLds%s"%tc]) and (not kernel["DirectToVgpr%s"%tc]): + instruction = tP["localWriteInstruction"] + numBlocks = instruction.numBlocks + numOffsets = instruction.numOffsets + blockWidth = instruction.blockWidth + #offsetMultiplier = instruction.offsetMultiplier + g2lIdx = 0 + #module.add(dump(vgpr("LocalWriteAddr%s"%tP["tensorChar"]))) + if 0: + print("\nLocalWrite", tP["tensorChar"]) + print("tlu", tP["tlu"]) + print("lsc", kernel[tP["lsc"]]) + print("lsp", kernel[tP["lsp"]]) + print("wtc", tP["wtc"]) + print("nrc", tP["nrc"]) + print("nrp", tP["nrp"]) + print("nwcv", tP["nwcv"]) + print("nwpv", tP["nwpv"]) + print("nrcvpi", tP["nrcvpi"]) + print("nwcvpi", tP["nwcvpi"]) + + tmpLocalWriteAddr = -1 + + # using _ds_store_b8: need one more vgpr space to do lshr + tmpVgprOffset = ((self.numVgprG2LA if (tP['tensorChar'] == 'A') else self.numVgprG2LB) / 2) if (blockWidth == 0.25) else 0 + + # if transposing, positions of sPerp and sPara are transposed + instructionCnt = -1 + fp16AltMap = {} + for perp in range(0, tP["nrp"]): + instructionCnt += 1 + localWriteCode = imod.add(Module("LocalWrite%u perp=%d"%(instructionCnt,perp))) + lwa = "LocalWriteAddr%s"%tc # default + + for para in range(0, tP["nrc"]): + if para>=1: + localWriteCode = imod.add(Module("LocalWrite%u perp=%d para=%d"%(instructionCnt,perp,para))) + + for s in range(0, max(tP["nwcv"],tP["nwpv"])//tP["nwcvpi"]): + sPerp = 0 + sPara = 0 + if tP["tlu"] != kernel["UnrollMajorLDS%s" % tP["tensorChar"]]: + if tP["wtc"]: + sPerp = s + else: + if tP["wtc"]: + sPara = s + + #print("perp:{}/{} para:{}/{} sPerp:{} sPara:{}".format(perp,tP["nrp"],para,tP["nrc"],sPerp,sPara)) + (offset, i, comment) = self.calculateLdsWriteOffset(perp, para, sPerp, sPara, kernel, tP) + + if uDu is None: + g2lIdx = int(i * blockWidth) + else: + # Example: DepthULdsDivisor=2 + # v0, v1, v2, v3 | v0, v1, v2, v3 | ... ----> unroll dim + # -----Thd 0----- -----Thd 1----- ... + # 1st subloop writes v0,v1 to LDS + # 2nd subloop writes v2,v3 to LDS + g2lIdx = int((i * kernel["DepthULdsDivisor"] + uDu) * blockWidth) + #print("uDu=%u, g2lIdx = %u, offset: %u"%(uDu, g2lIdx, offset)) + + # TODO- INT8: check uDu + if (blockWidth == 0.25) and ((s % 4) == 0): + src = "G2L%s+%u" % (tc, g2lIdx) + dst = "G2L%s+%u+%u" % (tc, tmpVgprOffset, g2lIdx) + localWriteCode.add(VMovB32(dst=vgpr(dst), src=vgpr(src), comment="another VGPR storing lshr 8-bit value")) + localWriteCode.add(VLShiftRightB32(dst=vgpr(dst), shiftHex=hex(0x8), src=vgpr(dst), comment="G2L Vpgr >> 8")) + + paramList = [] + for _ in range(0, numBlocks): + if blockWidth == 1: + paramList.append(vgpr("G2L%s+%u"%(tP["tensorChar"], g2lIdx))) + elif blockWidth == 0.25 and ((s % 2) == 1): # Int8, s = 1 or 3 (high8Bits) + paramList.append(vgpr("G2L%s+%u+%u"%(tc, tmpVgprOffset, g2lIdx))) + else: + paramList.append(vgpr("G2L%s+%u"%(tP["tensorChar"], g2lIdx), blockWidth)) + if self.db["ForceInputValue%s"%tc]: + localWriteCode.add(VMovB32(dst=vgpr("G2L%s+%u"%(tc, g2lIdx)), src=self.db["ForceValue%s"%tc], comment="ForceInputValue")) + if kernel["ProblemType"]["Fp16AltImpl"]: + numIters = 1 if blockWidth <= 1 else blockWidth + vgprTmp = self.vgprPool.checkOut(2) + for iter in range(0, numIters): + f16Tobf16Idx = g2lIdx + iter + if f16Tobf16Idx in fp16AltMap: + fp16AltMap[f16Tobf16Idx] += 1 + continue + fp16AltMap[f16Tobf16Idx] = 1 + sdwa = SDWAModifiers(src0_sel=SelectBit.WORD_1) + localWriteCode.add(VCvtF16toF32(dst=vgpr(vgprTmp), src=vgpr("G2L%s+%u"%(tc, f16Tobf16Idx)))) + localWriteCode.add(VCvtF16toF32(dst=vgpr(vgprTmp+1), src=vgpr("G2L%s+%u"%(tc, f16Tobf16Idx)),sdwa=sdwa)) + localWriteCode.add(VPackF16toB32(dst=vgpr("G2L%s+%u"%(tc, f16Tobf16Idx)), src0=vgpr(vgprTmp), src1=vgpr(vgprTmp+1), + vop3=VOP3PModifiers(op_sel=[1,1,0]))) + self.vgprPool.checkIn(vgprTmp) + + for oIdx in range(0, numOffsets): + paramList.append(offset) + + #print "offset", offset + + #comment = "Reg -> L %u_%u_%u_%u"%(para, sPara, perp, sPerp) + isHigh16Bits = False + if (kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16()): + if s%2==1: + isHigh16Bits = True + if tP["glvw"]==1 and instructionCnt%2==1: + isHigh16Bits = True + + # | hi16 | hi16 | | | + # | hi8 | | hi8 | | + ############################################# + # VGPR: |---w4---|---w3---|---w2---|---w1---| -> b8_d16: get w1 / _b8_d16_hi: get w3 + # LSHR: |--------|---w4---|--------|---w2---| -> b8_d16: get w2 / _b8_d16_hi: get w4 + elif kernel["ProblemType"]["DataType"].isInt8(): + isHigh16Bits = (s % 4) > 1 # 2,3 + # TODO + # if tP["glvw"]==1 and instructionCnt%2==1: + # isHigh16Bits = True + LocalWriteX = tP["localWriteInstruction"].getInst(isHigh16Bits) + if numBlocks == 1: + ds = DSModifiers(na=1, offset=paramList[1]) + writeInst = LocalWriteX(dstAddr=vgpr(lwa), src=paramList[0], ds=ds, comment=comment) + else: + ds = DSModifiers(na=2, offset0=paramList[2], offset1=paramList[3]) + writeInst = LocalWriteX(dstAddr=vgpr(lwa), src0=paramList[0], src1=paramList[1], ds=ds, comment=comment) + localWriteCode.add(writeInst) + + if tmpLocalWriteAddr != -1: + self.vgprPool.checkIn(tmpLocalWriteAddr) + + if 0 and tP["isB"]: # post-lds-write + localWriteCode.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="")) + localWriteCode.add(SBarrier(comment="dump LDS")) + localWriteCode.add(self.getCmpAssert(self.asmAssert.ne, sgpr("WorkGroup0"),1)) + #localWriteCode.add(self.getBomb()) + + return imod + + ############################################################################## + # Local Read: Swap Offsets A/B + # internalPointerSwap: swap internally tracked offsets - rather than + # emit specific instructions to do the pointer swap + ############################################################################## + def localReadSwapOffsets(self, kernel, internalPointerSwap, tP): + tc=tP["tensorChar"] + if (not self.do["LocalRead%s"%tc]) or kernel["DirectToVgpr%s"%tc]: # no local read code if DirectToVgpr is enabled + return Module("localReadSwapOffsets (no local read)") + if kernel["1LDSBuffer"]: + return Module("localReadSwapOffsets (Empty)") + module = Module("localReadSwapOffsets") + if internalPointerSwap: + tP["localReadSwapByteOffset"] = 0 if tP["localReadSwapByteOffset"] else kernel["LdsOffsetA_Blk"]*tP["bpe"] + module.addComment1("local read swap internal offset -> %u" % tP["localReadSwapByteOffset"]) + else: + module.add(VXorB32( + dst=vgpr("LocalReadAddr%s"%tP["tensorChar"]), \ + src0=hex(kernel["LdsOffsetA_Blk"]*tP["bpe"]), \ + src1=vgpr("LocalReadAddr%s"%tP["tensorChar"]), \ + comment="swap Red Blk")) + return module + + ############################################################################## + # Local Read: Reset Offsets A/B + # x % n == n & (n-1) for n power of 2 + # tP[localReadOffset] maintains running count of offsets + # This is called from the tail loop to reset read offsets? + ############################################################################## + def localReadResetOffsets(self, kernel, tP): + tc=tP["tensorChar"] + if not self.do["LocalRead%s"%tc]: return Module("localReadResetOffsets (no local read)") + if kernel["1LDSBuffer"] or kernel["DirectToVgpr%s"%tc]: # no local read code if DirectToVgpr is enabled + return Module("localReadResetOffsets (Empty)") + module = Module("localReadResetOffsets") + if tP["localReadInstruction"].numOffsets == 1: + tP["localReadSwapByteOffset"] = 0 + module.addComment1("localReadResetOffsets") + tP["localReadOffset"] = 0 + module.addComment0("handled internally") + module.add(VAndB32( + dst=vgpr("LocalReadAddr%s"%tP["tensorChar"]), \ + src0=hex(kernel["LdsOffsetA_Blk"]*tP["bpe"]-1), \ + src1=vgpr("LocalReadAddr%s"%tP["tensorChar"]), \ + comment="reset Red,Blk -> Red")) + return module + + ############################################################################## + # Local Read: Init Pointers A/B + ############################################################################## + def localReadInitPointers(self, kernel, tPA, tP): + tc=tP["tensorChar"] + if (not self.do["LocalRead%s"%tc]) or kernel["DirectToVgpr%s"%tc]:# no local read code if DirectToVgpr is enabled + return Module("localReadInitPointers (Empty)") + module = Module("localReadInitPointers") + if tPA["localReadInstruction"].numOffsets == 1: + module.addComment1("localReadInitPointers") + tP["localReadOffset"] = 0 + else: + module.add(VAndB32( + dst=vgpr("LocalReadAddr%s"%tP["tensorChar"]), \ + src0=hex(kernel["LdsOffset%s_Blk"%tP["tensorChar"]]*tP["bpe"]-1), \ + src1=vgpr("LocalReadAddr%s"%tP["tensorChar"]), \ + comment="init Red,Blk -> Red")) + return module + + ############################################################################## + # Local Read offset conversion for DirectToLds + ############################################################################## + def localReadOffsetConvForDTL(self, kernel, tP, offset_val): + tc = tP["tensorChar"] + bit2 = offset_val & 4 + bit3 = offset_val & 8 + bit4 = offset_val & 16 + bit5 = offset_val & 32 + if (kernel["GlobalLoadVectorWidth%s"%tc] * tP["bpe"] == 8): + # dword_x2 case + # (bit2<<3) | (bit3 >>1) | (bit4>>1) | (bit5>>1) + newVal = (bit2<<3) | (bit3 >>1) | (bit4>>1) | (bit5>>1) + else: #if (kernel["GlobalLoadVectorWidth%s"%tc] * tP["bpe"] == 16): # most preferred case + # dword_x4 case + # (bit2<<3) | (bit3 <<1) | (bit4>>2) | (bit5>>2) + newVal = (bit2<<3) | (bit3 <<1) | (bit4>>2) | (bit5>>2) + offset_val = offset_val & (~0x3c) + offset_val = offset_val | newVal + return offset_val + + ############################################################################## + # Local Read: Increment A/B + ############################################################################## + def localReadInc(self, kernel, iui, tP): + tc = tP["tensorChar"] + if not self.do["LocalRead%s" % tc] or kernel["DirectToVgpr%s"%tc]: # no local read code if DirectToVgpr is enabled + return Module("localReadInc (Empty)") + + module = Module("localReadInc") + + LdsPad = kernel["LdsPad%s"%tc] if kernel["LdsBlockSizePerPad%s"%tc] == 0 else 0 + + if self.inTailLoop: + inc = kernel["LocalSplitU"] * (kernel["MacroTile%s" % tP["tensorChar"]] + LdsPad) * tP["bpe"] + comment = " (LSU*(MT+PAD)*bpe)" + if kernel["EnableMatrixInstruction"]: + matrixInstK = kernel["MatrixInstK"] + if kernel["UnrollMajorLDS%s" % tc]: + if kernel["DirectToLds%s" % tc] and kernel["GlobalLoadVectorWidth%c"%tc] * tP["bpe"] > 4: + # DirectToLds special case. Need special address coonversion + localReadOffset = kernel["LocalSplitU"] * kernel["MatrixInstK"] * max(self.numReadsIterCoalescedA,self.numReadsIterCoalescedB) + localReadOffset *= tP["bpe"] + prev_offset_val = 0 if iui == 0 else localReadOffset * iui + offset_val = localReadOffset * (iui + 1) + # offset conversion or DirectToLds + prev_offset_val= self.localReadOffsetConvForDTL(kernel, tP, prev_offset_val) + offset_val= self.localReadOffsetConvForDTL(kernel, tP, offset_val) + inc = offset_val - prev_offset_val + matrixInstK = 1 # multiplying matrixInstK is not necessary + comment = "" + else: + inc = kernel["LocalSplitU"] * tP["bpe"] + comment = " (LSU*bpe)" + inc *= matrixInstK + + with self.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + module.add(SMovB32(dst=sgpr(tmpSgpr), src=hex(inc), comment="inc")) + module.add(VAddCOU32( + dst=vgpr("LocalReadAddr%s"%tP["tensorChar"]), \ + dst1=VCC(), \ + src0=sgpr(tmpSgpr), \ + src1=vgpr("LocalReadAddr%s"%tP["tensorChar"]), \ + comment="lr%s += %u%s"%(tP["tensorChar"], inc, comment) )) + else: + if tP["localReadInstruction"].numOffsets == 1: + if kernel["EnableMatrixInstruction"]: + if kernel["UnrollMajorLDS%s" % tP["tensorChar"]]: + tP["localReadOffset"] += kernel["LocalSplitU"] * kernel["MatrixInstK"] * max(self.numReadsIterCoalescedA,self.numReadsIterCoalescedB) + else: + if tc == "A": + if kernel["MatrixInstB"] != 1 or self.lrvwA == self.lrvwB: + tP["localReadOffset"] += kernel["LocalSplitU"] * (kernel["MacroTile%s"%tP["tensorChar"]] + LdsPad) * kernel["MatrixInstK"] * self.numReadsIterCoalescedA + else: + if (self.localReadDoCntA)%(kernel["LocalReadVectorWidth"]//self.lrvwA): + tP["localReadOffset"] += kernel["LocalSplitU"] * (kernel["MacroTile%s"%tP["tensorChar"]] + LdsPad) * self.lrvwA + else: + tP["localReadOffset"] += kernel["LocalSplitU"] * (kernel["MacroTile%s"%tP["tensorChar"]] + LdsPad) * (kernel["MatrixInstK"]*kernel["LocalReadVectorWidth"]//self.lrvwA-self.lrvwA*(kernel["LocalReadVectorWidth"]//self.lrvwA-1)) + else: + if kernel["MatrixInstB"] != 1 or self.lrvwA == self.lrvwB: + tP["localReadOffset"] += kernel["LocalSplitU"] * (kernel["MacroTile%s"%tP["tensorChar"]] + LdsPad) * kernel["MatrixInstK"] * self.numReadsIterCoalescedB + else: + if (self.localReadDoCntB)%(kernel["LocalReadVectorWidth"]//self.lrvwB): + tP["localReadOffset"] += kernel["LocalSplitU"] * (kernel["MacroTile%s"%tP["tensorChar"]] + LdsPad) * self.lrvwB + else: + tP["localReadOffset"] += kernel["LocalSplitU"] * (kernel["MacroTile%s"%tP["tensorChar"]] + LdsPad) * (kernel["MatrixInstK"]*kernel["LocalReadVectorWidth"]//self.lrvwB-self.lrvwB*(kernel["LocalReadVectorWidth"]//self.lrvwB-1)) + else: + tP["localReadOffset"] += kernel["LocalSplitU"] * (kernel["MacroTile%s"%tP["tensorChar"]] + LdsPad) + module.addComment0("N/A, lro->%d" % tP["localReadOffset"]) + module.addComment0("self.localReadDoCntA %d self.localReadDoCntB %d" % (self.localReadDoCntA,self.localReadDoCntB)) + else: + inc = kernel["LocalSplitU"] * (kernel["MacroTile%s" % tP["tensorChar"]] + LdsPad) + module.add(VAddCOU32( + dst=vgpr("LocalReadAddr%s"%tP["tensorChar"]), \ + dst1=VCC(), \ + src0=hex(inc), \ + src1=vgpr("LocalReadAddr%s"%tP["tensorChar"]), \ + comment="lr%s += %u (LSU+(MT+Pad)*bpe"%(tP["tensorChar"], inc) )) + + return module + + ############################################################################## + # Local Read: Do It A/B + # iui = Inner Unroll Idx + # uIdx - Unroll Idx + # epsi = expand pointer swap index. Only used for PAP + ############################################################################## + def localReadDo(self, kernel, bufferIdx, iui, epsi, tP): + + if not self.do["LocalRead%s" % tP["tensorChar"]]: + imod = Module("LocalReadDo%s_I%s" % (tP["tensorChar"], iui)) + pack = Module("pack%s_I%s" % (tP["tensorChar"], iui)) + return imod, pack + + component = Component.LocalRead.find(self) + if component: + return component(self, kernel, bufferIdx, iui, epsi, tP) + + ############################################################################## + # Save the local read pointers, for example when creating a duplicated + # optimized path (like optNLL) + ############################################################################## + def saveLocalPointers(self, kernel, tPA, tPB): + tPA["savedLocalReadOffset"] = tPA["localReadOffset"] + tPB["savedLocalReadOffset"] = tPB["localReadOffset"] + self.savedLocalReadDoCntA = self.localReadDoCntA + self.savedLocalReadDoCntB = self.localReadDoCntB + if kernel["ExpandPointerSwap"]: + tPA["savedLocalWriteSwapByteOffset"] = tPA["localWriteSwapByteOffset"] + tPB["savedLocalWriteSwapByteOffset"] = tPB["localWriteSwapByteOffset"] + + ############################################################################## + # Restore the saved local read pointers + # Must be paired with an earlier call to savePointers + ############################################################################## + def restoreLocalPointers(self, kernel, tPA, tPB): + tPA["localReadOffset"] = tPA["savedLocalReadOffset"] + tPB["localReadOffset"] = tPB["savedLocalReadOffset"] + self.localReadDoCntA = self.savedLocalReadDoCntA + self.localReadDoCntB = self.savedLocalReadDoCntB + if kernel["ExpandPointerSwap"]: + tPA["localWriteSwapByteOffset"] = tPA["savedLocalWriteSwapByteOffset"] + tPB["localWriteSwapByteOffset"] = tPB["savedLocalWriteSwapByteOffset"] + + ############################################################################## + # Shift Vector Components d0,1 + ############################################################################## + def shiftVectorComponents(self, kernel, tP): + component = Component.ShiftVectorComponents.find(self) + if component: + return component(self, kernel, tP) + + ############################################################################## + # LocalSplitU: Local Write + ############################################################################## + def localSplitULocalWrite(self, kernel): + module = Module("localSplitULocalWrite") + # wait for summation to be done with lds before writing reduction values + module.add(self.syncThreads(kernel, "pre-lsu local write")) + + tmpVgpr = self.vgprPool.checkOutAligned(2, 2, "tmpVgpr") + tmpVgprRes = RegisterPoolResource(tmpVgpr, 2) + lr0 = self.vgprPool.checkOut(1,"lr0") + lr1 = self.vgprPool.checkOut(1,"lr1") + sg = self.vgprPool.checkOut(1,"sg") + copy = self.vgprPool.checkOut(1,"copy") + + with self.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + + # lr0 = serial % SG0 + module.add(vectorStaticDivideAndRemainder(lr1, lr0, "Serial", \ + kernel["SubGroup0"], tmpVgprRes)) + + # lr1 = (serial / SG0) % SG1 + # sg = (serial / SG0) / SG1 + module.add(VMovB32(dst=vgpr(copy), src=vgpr(lr1), comment="copy for divide")) + module.add(vectorStaticDivideAndRemainder(sg, lr1, copy, \ + kernel["SubGroup1"], tmpVgprRes)) + + # lr0 *= VW + module.add(SMovB32(dst=sgpr(tmpSgpr), src=hex(kernel["VectorWidth"]*self.bpeCinternal), comment="VW")) + module.add(VMulLOU32(dst=vgpr(lr0), src0=sgpr(tmpSgpr), src1=vgpr(lr0), comment="lr0 *= VW")) + # lr1 *= VW*MT0 + module.add(SMovB32(dst=sgpr(tmpSgpr), \ + src=hex(kernel["VectorWidth"]*kernel["MacroTile0"]*self.bpeCinternal), comment="VW*MT0")) + module.add(VMulLOU32(dst=vgpr(lr1), src0=sgpr(tmpSgpr), src1=vgpr(lr1), comment="lr1 *= VW*MT0")) + # sg *= MT0*MT1 + module.add(SMovB32(dst=sgpr(tmpSgpr), \ + src=hex(kernel["MacroTile0"]*kernel["MacroTile1"]*self.bpeCinternal), comment="MT0*MT1")) + module.add(VMulLOU32(dst=vgpr(sg), src0=sgpr(tmpSgpr), src1=vgpr(sg), comment="sg *= MT0*MT1")) + + # thread offset + addr = lr0 + module.add(VAddCOU32(dst=vgpr(addr), dst1=VCC(), src0=vgpr(lr1), src1=vgpr(addr))) + module.add(VAddCOU32(dst=vgpr(addr), dst1=VCC(), src0=vgpr(sg), src1=vgpr(addr), comment="threadOffset")) + self.vgprPool.checkIn(lr0) + self.vgprPool.checkIn(lr1) + self.vgprPool.checkIn(sg) + self.vgprPool.checkIn(copy) + self.vgprPool.checkIn(tmpVgpr) + + # dump addr + # module.add(dump(vgpr(addr))) + + # do writes + # LDS Layout example (for Sgemm, LSU=4, TT=8x8, WG=[8,4,4]), 128 WI/WG + # VectorWidth = GlobalWriteVectorWidth = 4 + # SubGroup0 (WI:00-32) : LDS 0x0000- + # SubGroup1 (WI:33-64) : LDS 0x2000- + # SubGroup2 (WI:65-95) : LDS 0x4000- + # SubGroup3 (WI:96-127) : LDS 0x6000- + + # Interleave within a subgroup is interesting... + # Start LDS Addr + # WI00 - 0x000 + # WI01 - 0x010 + # ... + # WI07 - 0x070 + # WI08 - 0x400 + # WI09 - 0x410 + # ... + # WI0F - 0x470 + # WI10 - 0x800 + # ... + # ... + # WI1f - 0xc70 + # WI20 - 0x1000 (start SubGroup1) + + # so a zoom-in on the pattern at beginning of LDS, for the case above: + # WI (hex) |x00-|x01-|... |x07-|0x0-|0x1-|...|0x7-|0x0-| ... ... ||0x8-| + # ValuC |0123|0123|... |0123|4567|4567|...|4567|89AB| ... ... ||0123 + # | | | | + # LDS Addr 0x0 0x80 0x100 0x400 + + bytesPerElem = kernel["ProblemType"]["ComputeDataType"].numBytes() + regsPerElem = kernel["ProblemType"]["ComputeDataType"].numRegisters() + bytesPerVector = kernel["VectorWidth"] * bytesPerElem + bytesPerStep = min(bytesPerVector, 16) # max length of ds inst is 16 bytes(128bits) + regsPerStep = int((bytesPerStep+3)//4) + elementStep = bytesPerStep // bytesPerElem + + for j in range(0, kernel["ThreadTile1"]//kernel["VectorWidth"]): + for i in range(0, kernel["ThreadTile0"]//kernel["VectorWidth"]): + for s in range(0, kernel["VectorWidth"]): + for vc in range(0, kernel["VectorWidth"], elementStep): + # for half, write 2 elements (4 bytes) + # for single, write 1 element (4 bytes) + # double doesn't work yet + writeOffset = vc \ + + i*kernel["SubGroup0"]*kernel["VectorWidth"] \ + + s*kernel["MacroTile0"] \ + + j*kernel["MacroTile0"]*kernel["SubGroup1"]*kernel["VectorWidth"] + regIdx = vc \ + + i*kernel["VectorWidth"] \ + + s*kernel["ThreadTile0"] \ + + j*kernel["ThreadTile0"]*kernel["VectorWidth"] + regIdx = int(regIdx * regsPerElem) + + DSStoreBX = {128: DSStoreB128, + 64: DSStoreB64, + 32: DSStoreB32, + 16: DSStoreB16, + 8: DSStoreB8}[bytesPerStep*8] + module.add(DSStoreBX(dstAddr=vgpr(addr), src=vgpr("ValuC+%u"%regIdx, regsPerStep), \ + ds=DSModifiers(offset=(writeOffset*self.bpeCinternal)), + comment="j=%u i=%u s=%u vc=%u"%(j,i,s,vc))) + + module.add(SWaitCnt(lgkmcnt=0, vscnt=0, comment="wait for all writes")) + module.add(self.syncThreads(kernel, "post-lsu local write")) + # module.add(self.dumpLDS(kernel, 0, 16)) + #module.add(self.getBomb(5)) + return module + + ############################################################################## + # LocalSplitU: Local Read + ############################################################################## + def localSplitULocalRead(self, kernel): + # calculate parameters + bytesPerElem = kernel["ProblemType"]["ComputeDataType"].numBytes() + regsPerElem = kernel["ProblemType"]["ComputeDataType"].numRegisters() + bytesPerVector = kernel["GlobalWriteVectorWidth"] * bytesPerElem + bytesPerStep = 16 + while (bytesPerVector % bytesPerStep) != 0: + bytesPerStep //= 2 + regsPerStep = int((bytesPerStep+3)//4) + elementStep = bytesPerStep // bytesPerElem + + # alloc resource + baseAddr = self.vgprPool.checkOut(1,"baseAddr") + + with self.allocTmpSgpr(1) as tmpSgprInfo: + # generate source + module = Module("localSplitULocalRead") + module.add(staticMultiply(vgpr(baseAddr), vgpr("Serial"), kernel["GlobalWriteVectorWidth"]*self.bpeAB, tmpSgprInfo)) + DSLoadBX = {128: DSLoadB128, + 64: DSLoadB64, + 32: DSLoadB32}[bytesPerStep*8] + # Load values for each subgroup + for r in range(0, kernel["LocalSplitU"]): + for i in range(0, kernel["NumGlobalWriteVectorsPerThread"]): + for s in range(0, kernel["GlobalWriteVectorWidth"], elementStep): + offset = s + i*kernel["NumThreads"]*kernel["GlobalWriteVectorWidth"] + r * kernel["MacroTile0"]*kernel["MacroTile1"] + regIdx = int((s + i*kernel["GlobalWriteVectorWidth"] + r*kernel["GlobalWriteVectorWidth"]*kernel["NumGlobalWriteVectorsPerThread"]) * regsPerElem) + module.add(DSLoadBX(dst=vgpr("ValuC+%u"%regIdx,regsPerStep), src=vgpr(baseAddr), \ + ds=DSModifiers(offset=(offset*self.bpeCinternal)), comment="r=%u i=%u s=%u"%(r,i,s))) + module.add(SWaitCnt(lgkmcnt=0, comment="wait for all reads")) + + if self.archCaps["SeparateVscnt"]: + module.add(SWaitCnt(vscnt=0)) + + # free resources + self.vgprPool.checkIn(baseAddr) + + return module + + ############################################################################## + # LocalSplitU: Reduction + ############################################################################## + def localSplitUReduction(self, kernel): + module = Module("localSplitUReduction") + + is_non_hpa_fp16 = kernel["ProblemType"]["DataType"].isHalf() and (not kernel["ProblemType"]["HighPrecisionAccumulate"]) + elementStep = 2 if is_non_hpa_fp16 else 1 + regsPerElem = kernel["ProblemType"]["DataType"].numRegisters() + + for r in range(1, kernel["LocalSplitU"]): + for i in range(0, kernel["NumGlobalWriteVectorsPerThread"]): + for s in range(0, kernel["GlobalWriteVectorWidth"], elementStep): + cIdx = int((s + i * kernel["GlobalWriteVectorWidth"]) * regsPerElem) + regIdx = int((s + i * kernel["GlobalWriteVectorWidth"] + r * kernel["GlobalWriteVectorWidth"] * kernel["NumGlobalWriteVectorsPerThread"]) * regsPerElem) + + if is_non_hpa_fp16: + module.add(VAddPKF16(dst=vgpr("ValuC+%u"%cIdx), src0=vgpr("ValuC+%u" % regIdx), src1=vgpr("ValuC+%u"%cIdx), \ + comment="c[%u] += c[%u]"%(cIdx, regIdx) )) + elif kernel["ProblemType"]["DataType"].isInt8x4(): + module.add(VAddI32(dst=vgpr("ValuC+%u"%cIdx), src0=vgpr("ValuC+%u" % regIdx), src1=vgpr("ValuC+%u"%cIdx), \ + comment="c[%u] += c[%u]"%(cIdx, regIdx))) + + elif kernel["ProblemType"]["DataType"].isSingle(): + module.add(VAddF32(dst=vgpr("ValuC+%u"%cIdx), src0=vgpr("ValuC+%u" % regIdx), src1=vgpr("ValuC+%u"%cIdx), \ + comment="c[%u] += c[%u]"%(cIdx, regIdx))) + elif kernel["ProblemType"]["DataType"].isDouble(): + module.add(VAddF64(dst=vgpr("ValuC+%u"%cIdx,2), src0=vgpr("ValuC+%u" % regIdx,2), src1=vgpr("ValuC+%u"%cIdx,2), \ + comment="c[%u] += c[%u]"%(cIdx, regIdx))) + elif kernel["ProblemType"]["DataType"].isSingleComplex(): + module.add(VAddF32(dst=vgpr("ValuC+%u"%(cIdx+0)), src0=vgpr("ValuC+%u" % (regIdx+0)), src1=vgpr("ValuC+%u"%(cIdx+0)), \ + comment="c[%u] += c[%u], real part"%(cIdx, regIdx) )) + module.add(VAddF32(dst=vgpr("ValuC+%u"%(cIdx+1)), src0=vgpr("ValuC+%u" % (regIdx+1)), src1=vgpr("ValuC+%u"%(cIdx+1)), \ + comment="c[%u] += c[%u], imaginary part"%(cIdx+1, regIdx+1) )) + elif kernel["ProblemType"]["DataType"].isDoubleComplex(): + module.add(VAddF64(dst=vgpr("ValuC+%u"%(cIdx+0),2), src0=vgpr("ValuC+%u" % (regIdx+0),2), src1=vgpr("ValuC+%u"%(cIdx+0),2), \ + comment="c[%u] += c[%u], real part"%(cIdx, regIdx) )) + module.add(VAddF64(dst=vgpr("ValuC+%u"%(cIdx+2),2), src0=vgpr("ValuC+%u" % (regIdx+2),2), src1=vgpr("ValuC+%u"%(cIdx+2),2), \ + comment="c[%u] += c[%u], imaginary part"%(cIdx+2, regIdx+2) )) + else: + # TODO: hpa_half, int8 + assert(0) # unsupported data type, need to modify here and LSU write/read code + return module + + ############################################################################## + # computeStoreSrd + # Add tile assignment fields to store srd + # This is based on WG not the WI/TT assignment + ############################################################################## + def computeStoreSrdStart(self, kernel): + module = Module("computeStoreSrdStart") + + with self.allocTmpSgpr(3) as tmpSgprInfo: + tmpS0 = tmpSgprInfo.idx + tmpS1 = tmpS0+1 + wgMT1 = tmpS0+2 + + # Compute and save wg1*MT1 - the element offset that is top of the macro-tile in output space + assert kernel["BufferStore"] + module.addSpaceLine() + module.add(SMulI32( + dst=sgpr(wgMT1), \ + src0="MT1", \ + src1=sgpr("WorkGroup1"), \ + comment="<- wg1*MT1")) + + # Overall strategy is to set the SRD to the top-left of the macro-tile. + # TT offsets are from this base (and include the column) + + # In non-packed mode: + # higher-order tensor dims are static since this kernel operates within + # the 2D Tensor formed by Index0 and Indexa. + # Index0 and Index1 vary for each work-item (aka 'dynamic') so roll these into the VGPR + + # In packed mode: + # Higher-order dimensions may be packed into coord0 / coord1 - see rowstart calculation below + + # Walk through addressing components (each tensor index) in C + # For static dims add to SrdC / SrdD to compute a new base. + # For dynamic (based on TT assignment) - save in coutRowPtr in computeStoreVgprs, + # which saves the TT assignment for each WI scaled by StrideC0 + # TODO - future opportunities for store vgpr and other optimization + # - coutRowPtr and tid1 are strongly related - can we merge or remove one of these? + # Packed follows same philosophy but may have more vector components + indices = list(range(0, kernel["ProblemType"]["NumIndicesC"])) + numDim = len(indices) + addrSrcSgpr = "Address" # use "Address" only for the first iteration + for i in range(1, numDim): + if i == kernel["ProblemType"]["Index0"]: + # Used if the output is transposed? + addToSrd = False + elif i == kernel["ProblemType"]["Index1"] and len(kernel["PackedC1IndicesX"]) == 1: + coord = sgpr(wgMT1) + addToSrd = True + elif i != kernel["ProblemType"]["Index0"] and i != kernel["ProblemType"]["Index1"] and not isPackedIndex(kernel, i): + # group index, this is higher-order Tensor dimension, just add to SRD base: + isStridedBuffer = kernel["ProblemType"]["StridedBatched"] or kernel["_GlobalAccumulation"] + coord = sgpr("WorkGroup2") if isStridedBuffer else None + addToSrd = True if isStridedBuffer else False + else: + # could be packed higher-order index, just ignore + coord = None + addToSrd = False + + if addToSrd: + # These are constant across all workitems, just add to the SRD: + strideC = "StrideC%s"%self.indexChars[i] + module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tmpS0), sgpr(tmpS1), coord, sgpr(strideC), "CScale %s by Stride"%coord)) + module.add(SLShiftLeftB64(dst=sgpr(tmpS0,2), src=sgpr(tmpS0,2), shiftHex=log2(self.bpeCexternal), comment="scale by bpe")) + + module.add(SAddU32(dst=sgpr("SrdC+0"), src0=sgpr("%sC+0"%addrSrcSgpr), src1=sgpr(tmpS0), comment="add lo to SRD")) + module.add(SAddCU32(dst=sgpr("SrdC+1"), src0=sgpr("%sC+1"%addrSrcSgpr), src1=sgpr(tmpS1), comment="add hi to SRD")) + + # These are constant across all workitems, just add to the SRD: + stride = "StrideD%s" % (self.indexChars[i]) + module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tmpS0), sgpr(tmpS1), coord, sgpr(stride), "Scale %s by Stride"%coord)) + module.add(SLShiftLeftB64(dst=sgpr(tmpS0,2), src=sgpr(tmpS0,2), shiftHex=log2(self.bpeCexternal), comment="scale by bpe")) + + module.add(SAddU32(dst=sgpr("SrdD+0"), src0=sgpr("%sD+0"%addrSrcSgpr), src1=sgpr(tmpS0), comment="add lo to SRD")) + module.add(SAddCU32(dst=sgpr("SrdD+1"), src0=sgpr("%sD+1"%addrSrcSgpr), src1=sgpr(tmpS1), comment="add hi to SRD")) + + module.addSpaceLine() + + addrSrcSgpr = "Srd" # update src Sgpr for the second or later iterations + + if kernel["_GlobalAccumulation"] == 'MultipleBuffer': + # GSU algorithm 2: adjust output buffer address to per GSU buffer + with self.allocTmpSgpr(5) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + module.addComment("GSU Output Buffer offset: Free0 + (Free1-1)*StrideC1J + (Free2-1)*StrideCK * GSUIdx * bpe%s") + module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tmpSgpr+0), sgpr(tmpSgpr+1), sgpr("SizesFree+0"), sgpr("GSUSumIdx"), "Free0")) + for i in range(1, numDim): + module.add(SSubU32(dst=sgpr(tmpSgpr+4), src0=sgpr("SizesFree+%u"%i), src1=1, comment="Free%u" % i)) + module.add(SMulI32(dst=sgpr(tmpSgpr+4), src0=sgpr(tmpSgpr+4), src1=sgpr("GSUSumIdx"), comment="Free%u" % i)) + module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tmpSgpr+2), sgpr(tmpSgpr+3), sgpr(tmpSgpr+4), sgpr("StrideC%s"%self.indexChars[i]), "Free%u" % i)) + module.add(SAddU32(dst=sgpr(tmpSgpr+0), src0=sgpr(tmpSgpr+0), src1=sgpr(tmpSgpr+2), comment="Free%u" % i)) + module.add(SAddCU32(dst=sgpr(tmpSgpr+1), src0=sgpr(tmpSgpr+1), src1=sgpr(tmpSgpr+3), comment="Free%u" % i)) + module.add(SLShiftLeftB64(dst=sgpr(tmpSgpr+0,2), src=sgpr(tmpSgpr+0,2), shiftHex=log2(self.bpeCexternal), comment="scale by bpe")) + module.add(SAddU32(dst=sgpr("SrdD+0"), src0=sgpr("SrdD+0"), src1=sgpr(tmpSgpr+0), comment="add lo GSU offset to SRD")) + module.add(SAddCU32(dst=sgpr("SrdD+1"), src0=sgpr("SrdD+1"), src1=sgpr(tmpSgpr+1), comment="add hi GSU offset to SRD")) + + for cdir in (0,1): + indices = kernel["PackedC%uIndicesX"%cdir] + packedSizes = "PackedSize%u"%cdir + if len(indices) > 1: + for i,idx in enumerate(indices[1:]): + if i==0: + module.add(SMulI32(dst=sgpr(packedSizes), src0=self.sizeRef(indices[0]), \ + src1=self.sizeRef(idx), comment="first packed size")) + else: + module.add(SMulI32(dst=sgpr(packedSizes), src0=sgpr(packedSizes), \ + src1=self.sizeRef(idx), comment="first packed size")) + + return module + + ############################################################################## + # computeStoreVgprs + # Compute workitem/TT offsets in VGPRS + # and coord0/coord1 + # tid0Scale specifies the number of output elements in 0/coalesced dim + # that should be written by each work-item in each batch element. + ############################################################################## + def computeStoreVgprs(self, kernel, divisor, tid0Scale, tid1Scale): + module = Module("computeStoreVgprs") + module.addComment0("computeStoreVgprs") + component = Component.ComputeStoreVgprs.find(self) + if component: + module.add(component(self, kernel, divisor, tid0Scale, tid1Scale)) #FIXME + return module + + ############################################################################## + # globalWriteWorkGroupInit: + ############################################################################## + def globalWriteWorkGroupInit(self, kernel): + module = Module("globalWriteWorkGroupInit") + if kernel["BufferStore"]: + module.add(self.allocPostLoopSrd("D")) + module.add(self.allocPostLoopSrd("C")) + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + module.add(self.allocPostLoopSrdSuppress("Bias")) + module.add(self.computeStoreSrdStart(kernel)) + return module + + ############################################################################## + # LocalSplitU: Global Write Indices + ############################################################################## + def localSplitUGlobalWriteIndices(self, kernel): + module = Module("localSplitUGlobalWriteIndices") + + # lr0 = serial % SG0 + module.add(self.computeStoreVgprs(kernel, \ + divisor = kernel["MacroTile0"] // kernel["GlobalWriteVectorWidth"], \ + tid0Scale=kernel["GlobalWriteVectorWidth"], \ + tid1Scale=1)) + + if kernel["BufferStore"]: + #print "----AddressC-LocalSplitU" + #print self.vgprPool.state() + self.vgprs.addrD = -1 + self.vgprs.addrC = -1 + self.vgprs.addrBias = -1 + else: + self.vgprs.addrD = self.vgprPool.checkOut(2) + module.add(VMovB32( + dst=vgpr(self.vgprs.addrD+0), \ + src=sgpr("AddressD+0"), \ + comment="sgpr -> vgpr")) + module.add(VMovB32( + dst=vgpr(self.vgprs.addrD+1), \ + src=sgpr("AddressD+1"), \ + comment="sgpr -> vgpr")) + self.vgprs.addrC = self.vgprPool.checkOut(2) + module.add(VMovB32( + dst=vgpr(self.vgprs.addrC+0), \ + src=sgpr("AddressC+0"), \ + comment="sgpr -> vgpr")) + module.add(VMovB32( + dst=vgpr(self.vgprs.addrC+1), \ + src=sgpr("AddressC+1"), \ + comment="sgpr -> vgpr")) + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + self.vgprs.addrBias = self.vgprPool.checkOut(2, 'addrBias') + module.add(VMovB32( \ + dst=vgpr(self.vgprs.addrBias+0), \ + src=sgpr("AddressBias+0"), \ + comment="sgpr -> vgpr")) + module.add(VMovB32( \ + dst=vgpr(self.vgprs.addrBias+1), \ + src=sgpr("AddressBias+1"), \ + comment="sgpr -> vgpr")) + + return module + + ############################################################################## + def allocPostLoopSrd(self, ch: str): + module = Module("allocPostLoopSrd") + # Buffer-load uses one base read pointer stored in the SRD - set it here: + module.add(SMovB32(dst=sgpr("Srd%s+0"%ch), src=sgpr("Address%s+0"%ch), comment="init SRD base address (lower)" )) + module.add(SMovB32(dst=sgpr("Srd%s+1"%ch), src=sgpr("Address%s+1"%ch), comment="init SRD base address (upper) + other fields" )) + module.add(SMovB32(dst=sgpr("Srd%s+2"%ch), src=hex(0x80000000))) + module.add(SMovB32(dst=sgpr("Srd%s+3"%ch), src="Srd127_96", comment="Set bits 127_96 in post-loop SRD")) + module.addSpaceLine() + return module + + def allocPostLoopSrdSuppress(self, ch: str): + module = Module("allocPostLoopSrdSuppress") + label = Label("%sAddrValid"%ch, "") + label2 = Label("%sAddrValid_End"%ch, "") + # Buffer-load uses one base read pointer stored in the SRD - set it here: + module.add(SMovB32(dst=sgpr("Srd%s+0"%ch), src=sgpr("Address%s+0"%ch), comment="init SRD base address (lower)" )) + module.add(SMovB32(dst=sgpr("Srd%s+1"%ch), src=sgpr("Address%s+1"%ch), comment="init SRD base address (upper) + other fields" )) + module.add(SBranchIfNotZero("Address%s"%ch, DataType('int64'), label)) + module.add(SMovB32(dst=sgpr("Srd%s+2"%ch), src=0)) + module.add(SBranch(label2.getLabelName())) + module.add(label) + module.add(SMovB32(dst=sgpr("Srd%s+2"%ch), src=hex(0x80000000))) + module.add(SMovB32(dst=sgpr("Srd%s+3"%ch), src="Srd127_96", comment="Set bits 127_96 in post-loop SRD")) + module.add(label2) + module.addSpaceLine() + return module + + ############################################################################## + # Not LocalSplitU: Global Write Indices + ############################################################################## + def notLocalSplitUGlobalWriteIndices(self, kernel): + #print "GlobalWriteIndices" + if not self.do["PostLoop"]: return "" + module = Module("notLocalSplitUGlobalWriteIndices") + + module.add(self.computeStoreVgprs(kernel, + divisor = kernel["SubGroup0"],\ + tid0Scale=kernel["VectorWidth"], \ + tid1Scale=kernel["VectorWidth"])) + + if kernel["BufferStore"]: + #print "----AddressC-nonLSU-----" + #print self.vgprPool.state() + self.vgprs.addrD = -1 + self.vgprs.addrC = -1 + self.vgprs.addrBias = -1 + else: + self.vgprs.addrD = self.vgprPool.checkOut(2, 'addrD') + module.add(VMovB32( + dst=vgpr(self.vgprs.addrD+0), \ + src=sgpr("AddressD+0"), \ + comment="sgpr -> vgpr")) + module.add(VMovB32( + dst=vgpr(self.vgprs.addrD+1), \ + src=sgpr("AddressD+1"), \ + comment="sgpr -> vgpr")) + self.vgprs.addrC = self.vgprPool.checkOut(2, 'addrC') + module.add(VMovB32( + dst=vgpr(self.vgprs.addrC+0), \ + src=sgpr("AddressC+0"), \ + comment="sgpr -> vgpr")) + module.add(VMovB32( + dst=vgpr(self.vgprs.addrC+1), \ + src=sgpr("AddressC+1"), \ + comment="sgpr -> vgpr")) + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + self.vgprs.addrBias = self.vgprPool.checkOut(2, 'addrBias') + module.add(VMovB32( \ + dst=vgpr(self.vgprs.addrBias+0), \ + src=sgpr("AddressBias+0"), \ + comment="sgpr -> vgpr")) + module.add(VMovB32( \ + dst=vgpr(self.vgprs.addrBias+1), \ + src=sgpr("AddressBias+1"), \ + comment="sgpr -> vgpr")) + return module + + ############################################################################## + # Release any resources used by the global write + def cleanupGlobalWrite(self, kernel): + self.vgprPool.checkIn(self.vgprs.coord0) + self.vgprPool.checkIn(self.vgprs.coord1) + + if kernel["StoreRemapVectorWidth"]: + self.vgprPool.checkIn(self.vgprs.storeRemapLW) + self.vgprPool.checkIn(self.vgprs.storeRemapLR) + self.vgprPool.checkIn(self.vgprs.storeRemapCoord0) + self.vgprPool.checkIn(self.vgprs.storeRemapCoord1) + self.vgprPool.checkIn(self.vgprs.storeRemapOffsetCoord1) + if kernel["BufferStore"]: + self.vgprPool.checkIn(self.vgprs.cinRowPtr) + self.vgprPool.checkIn(self.vgprs.coutRowPtr) + if not kernel["BufferStore"]: + self.vgprPool.checkIn(self.vgprs.addrD) + self.vgprPool.checkIn(self.vgprs.addrC) + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + self.vgprPool.checkIn(self.vgprs.addrBias) + + ############################################################################## + # Return max global write vector width, in elements + def maxGwvw(self, kernel): + atomic = (kernel["GlobalSplitU"] > 1) and (kernel["_GlobalAccumulation"] != 'MultipleBuffer') + + if kernel["BufferStore"]: + if atomic: + return self.getVectorAtomicWidth(kernel) + else: + return 1000 # no limit + else: + if atomic: + return 1 # flat vector atomic is not tested + else: + return 1000 # no limit + + def getVectorAtomicWidth(self, kernel): + if kernel["ProblemType"]["DataType"].isHalf() and (not kernel["_GlobalAccumulation"]): + return 2 + return 1 + + ############################################################################## + # Partition thread-tile into writeElements for store code + # This function creates the writeElement mapping for full tiles + # (ie non-edge cases) + ############################################################################## + def notLocalFullTileElements(self, kernel, edge): + component = Component.NotLocalFullTileElements.find(self) + if component: + return component(self, kernel, edge) + + ############################################################################## + # Store Remap: Local Write + ############################################################################## + def storeRemapAddLocalWrite(self, ss, addrCalc, srcVgpr): + """ + Add localWrite for the element with addrCalc and srcVgpr. + """ + + module = Module("storeRemapAddLocalWrite srcVgpr %s"%str(srcVgpr)) + + bps = self.bpeCexternal * ss.cfg.gwvw + rpv = self.bpeCexternal * ss.cfg.gwvw / self.bpr + + addr0 = vgpr(self.vgprs.storeRemapLW) + offset = addrCalc.coordOffset0 * self.bpeCexternal + ds = DSModifiers(offset=offset) + + if bps==2: + module.add(DSStoreB16(dstAddr=addr0, src=vgpr(srcVgpr, rpv*2), \ + ds=ds, comment="storeRemap lw")) + elif bps==4: + module.add(DSStoreB32(dstAddr=addr0, src=vgpr(srcVgpr, rpv), \ + ds=ds, comment="storeRemap lw")) + elif bps==8: + module.add(DSStoreB64(dstAddr=addr0, src=vgpr(srcVgpr, rpv), \ + ds=ds, comment="storeRemap lw")) + elif bps==16: + module.add(DSStoreB128(dstAddr=addr0, src=vgpr(srcVgpr, rpv), \ + ds=ds, comment="storeRemap lw")) + else: + assert 0, "StoreRemap: bad bps!" + + return module + + ############################################################################## + # Store Remap: Local Read and Global Write + ############################################################################## + def storeRemapAddStore(self, kernel, tmpVgpr, tmpS01, edge): + module = Module("storeRemapAddStore") + + module.add(SWaitCnt(lgkmcnt=0, comment="wait for LDS write")) + + numStoreInst = 0 + + #Data exchange between different waves + #Make sure LDS writes are finished of all waves + if kernel["MIWaveGroup"][0] > 1: + # FIXME: Indent? + module.add(SBarrier(comment="wait all lds write finished")) + module.addSpaceLine() + + gwvw = kernel["StoreRemapVectorWidth"] + nElements = kernel["MacroTile0"]*kernel["MatrixInstN"]//kernel["MIWaveGroup"][0]//self.kernel["WavefrontSize"] + + bpe = self.bpeCexternal + bps = bpe * gwvw + rpe = self.bpeCexternal / self.bpr + rpv = rpe * gwvw + + # num registers to check out + storeRegs = [] + for i in range(0, nElements, gwvw): + storeRegs.append(self.vgprPool.checkOutAligned(int(rpv), int(rpv), "store element d")) + src = vgpr(self.vgprs.storeRemapLR) + for rIdx, i in enumerate(range(0, nElements, gwvw)): + offset = self.storeRemapLrOffset * bpe * (i//gwvw) + ds = DSModifiers(offset=offset) + dst = vgpr(storeRegs[rIdx], rpv) + if bps==4: + module.add(DSLoadB32(dst=dst, src=src, ds=ds, comment="storeRemap lr")) + elif bps==8: + module.add(DSLoadB64(dst=dst, src=src, ds=ds, comment="storeRemap lr")) + elif bps==16: + module.add(DSLoadB128(dst=dst, src=src, ds=ds, comment="storeRemap lr")) + else: + assert 0, "StoreRemap: bad bps!" + + module.addSpaceLine() + + # Global Write + ntStr = "" + if kernel.enabledSetPrioSplitLDS: + module.add(SSetPrior(prior=1)) + if kernel["NonTemporalD"]%2==1: + ntStr += " glc" + if kernel["NonTemporalD"]//2==1: + ntStr += " slc" + + addr1 = sgpr("SrdD", 4) + packedD1 = kernel["PackedC1IndicesX"] + strideD1 = "StrideD%s" % (self.indexChars[packedD1[0]]) + + vTmp = self.vgprPool.checkOut(1, "SR Store temp addr0") + addr0 = vgpr(vTmp) + + if not edge: + for rIdx, i in enumerate(range(0, nElements, gwvw)): + if i == 0: + module.add(VMovB32(dst=addr0, src=vgpr(self.vgprs.storeRemapOffsetCoord1), comment="coord1")) + else: + currentStep = i//gwvw + module.add(VAddU32(dst=addr0, src0=vgpr(self.vgprs.storeRemapOffsetCoord1), src1=(self.storeRemapNCPL * currentStep), comment="coord1 += nColPerLoad")) + + module.add(VMulLOU32(dst=addr0, src0=addr0, src1=sgpr(strideD1), comment="coord1 offset = coord1 * StrideD")) + module.add(VAddLShiftLeftU32(dst=addr0, src0=addr0, src1=vgpr(self.vgprs.storeRemapCoord0), shiftHex=hex(log2(bpe)), comment="global write D address")) + + lgkmcnt = min((nElements-i)//gwvw - 1, 15) + module.add(SWaitCnt(lgkmcnt=lgkmcnt, comment="wait for LDS read")) + + numStoreInst += 1 + module.add(self.chooseGlobalWrite(True, bps, storeRegs[rIdx], rpv, addr0, addr1, 0, ntStr)) + else: + tmpS23 = tmpS01+self.laneSGPRCount + coord0 = tmpVgpr + coord1 = coord0+1 + lrVw = kernel["StoreRemapVectorWidth"] + edgeVw = min(kernel["AssertFree0ElementMultiple"],kernel["StoreRemapVectorWidth"]) + bps = self.bpeCexternal * edgeVw + rpv = self.bpeCexternal / self.bpr * edgeVw + for rIdx, i in enumerate(range(0, nElements, lrVw)): + for vi in range (0, lrVw, edgeVw): + + if vi == 0: + lgkmcnt = min((nElements-i)//lrVw - 1, 15) + module.add(SWaitCnt(lgkmcnt=lgkmcnt, comment="wait for LDS read")) + + sizeBoundary = [0,0] + sizeBoundary[0] = \ + sgpr("PackedSize0") if len(kernel["PackedC0IndicesX"]) > 1 \ + else self.sizeRef(kernel["ProblemType"]["Index0"]) + sizeBoundary[1] = \ + sgpr("PackedSize1") if len(kernel["PackedC1IndicesX"]) > 1 \ + else self.sizeRef(kernel["ProblemType"]["Index1"]) + + currentStep = i//lrVw + + # calculate global coordination + module.add(VAddU32(dst=vgpr(coord1), src0=vgpr(self.vgprs.storeRemapCoord1), src1=(self.storeRemapNCPL * currentStep), comment="coord1 += nColPerLoad")) + module.add(VAddU32(dst=vgpr(coord0), src0=vgpr(self.vgprs.storeRemapCoord0), src1=vi, comment="coord0 += element index of load vector")) + module.add(VAddU32(dst=addr0, src0=vgpr(self.vgprs.storeRemapOffsetCoord1), src1=(self.storeRemapNCPL * currentStep), \ + comment="offset coord1 += nColPerLoad")) + + module.add(VCmpLtU32(dst=sgpr(tmpS01,self.laneSGPRCount), src0=vgpr(coord0), src1=sizeBoundary[0], comment="coord0 < size0" )) + module.add(VCmpLtU32(dst=sgpr(tmpS23,self.laneSGPRCount), src0=vgpr(coord1), src1=sizeBoundary[1], comment="coord1 < size1" )) + SAndBX = SAndB32 if self.kernel["WavefrontSize"] == 32 else SAndB64 + module.add(SAndBX( + dst=sgpr(tmpS23,self.laneSGPRCount), + src0=sgpr(tmpS01,self.laneSGPRCount), + src1=sgpr(tmpS23,self.laneSGPRCount), comment="in0 && in1" )) + + module.add(VMulLOU32(dst=addr0, src0=addr0, src1=sgpr(strideD1), comment="coord1 element offset = coord1 * StrideD")) + module.add(VAddLShiftLeftU32(dst=addr0, src0=addr0, src1=vgpr(coord0), shiftHex=hex(log2(bpe)), comment="scale to BPE")) + module.add(VCndMaskB32(dst=addr0, src0=-1, src1=addr0, src2=sgpr(tmpS23,self.laneSGPRCount), comment="clip if OOB. offset" )) + + sumIdx = storeRegs[rIdx] + int(vi*rpe) + numStoreInst += 1 + if bps == 2: + module.add(self.chooseGlobalWrite(True, bpe, sumIdx, rpe, addr0, addr1, 0, ntStr, hi16=vi%2)) + else: + module.add(self.chooseGlobalWrite(True, bps, sumIdx, rpv, addr0, addr1, 0, ntStr)) + + module.addSpaceLine() + self.vgprPool.checkIn(vTmp) + for v in storeRegs: + self.vgprPool.checkIn(v) + + #Data exchange between different waves + #Make sure LDS reads are finished of all waves + if kernel["MIWaveGroup"][0] > 1: + module.add(SBarrier(comment="wait all lds read finished")) + + return module, numStoreInst + + ############################################################################## + # Store remap compute vgprs: + ############################################################################## + def storeRemapComputeStoreVgprs(self, kernel): + module = Module("storeRemapComputeStoreVgprs") + module.addComment0("Store Remap Local Write address") + + with self.allocTmpSgpr(2) as tmpSgprInfo: + tmpS0 = tmpSgprInfo.idx + wgMT1 = tmpS0 + 1 + + wg0="WorkGroup0" + wg1="WorkGroup1" + + tid0 = self.vgprPool.checkOut(1, "SR coord0") + tid1 = self.vgprPool.checkOut(1, "SR coord1") + coord1Offset = self.vgprPool.checkOut(1, "SR coord1 offset") + storeRemapLW = self.vgprPool.checkOut(1, "SR local write") + storeRemapLR = self.vgprPool.checkOut(1, "SR local read") + + tmpV0 = self.vgprPool.checkOut(5, "tmpV0") + waveCoord0 = tmpV1 = tmpV0+1 + ldsStride = tmpV0+2 + coord0 = tmpV0+3 + waveCoord1 = tmpV0+4 + + gwvw = kernel["StoreRemapVectorWidth"] + ldsPad = max(kernel["StoreRemapVectorWidth"],kernel["MIOutputVectorWidth"]) + + #calculate local write Address: v[vgprLocalWriteAddrC] + module.add(vectorStaticDivideAndRemainder(tid1, tid0, "Serial", self.kernel["WavefrontSize"]*kernel["MIWaveGroup"][0], \ + RegisterPoolResource(tmpV0, 2))) + + module.add(VMulLOU32(dst=vgpr(waveCoord1), + src0=hex(kernel["MatrixInstN"]), src1=vgpr(tid1), comment="coord1 offset of LDS for each Wave")) + module.add(VAndB32(dst=vgpr(tid1), + src0=hex(kernel["MatrixInstN"]-1), src1=vgpr("Serial"), comment="coord1 offset of LDS for each thread")) + module.add(VAddU32(dst=vgpr(tid1), src0=vgpr(waveCoord1), src1=vgpr(tid1), comment="coord1 offset in MacroTile")) + module.add(VMovB32(dst=vgpr(ldsStride), src=hex(kernel["MacroTile0"]+ldsPad), \ + comment="lds stride = MT0 + PAD")) + module.add(VMulLOU32(dst=vgpr(tmpV0), src0=vgpr(tid1), src1=vgpr(ldsStride), \ + comment="lds coord1 offset = Col-id* lds stride")) + + module.add(vectorStaticDivideAndRemainder(waveCoord0, tid0, tid0, self.kernel["WavefrontSize"], RegisterPoolResource(tmpV0, 2))) + module.add(VLShiftRightB32(dst=vgpr(coord0), + shiftHex=hex(log2(kernel["MatrixInstN"])), src=vgpr(tid0), \ + comment="tid / matrixInstN")) + + if kernel["MIOutputVectorWidth"] > 1: + module.add(VLShiftLeftB32(dst=vgpr(coord0), shiftHex=hex(log2(kernel["MIOutputVectorWidth"])), src=vgpr(coord0), \ + comment="lds coord0 offset *= 4 (each thread hold 4 element)")) + + module.add(VMadU32U24(dst=vgpr(coord0), src0=(kernel["MatrixInstM"]*kernel["MatrixInstBM"]), src1=vgpr(waveCoord0), src2=vgpr(coord0), \ + comment="coord0 += waveCoord0 * wave M shape(blockM*MiM)")) + + module.add(VAddLShiftLeftU32( + dst=vgpr(storeRemapLW), \ + src0=vgpr(tmpV0), \ + src1=vgpr(coord0), \ + shiftHex=hex(log2(self.bpeCexternal)), \ + comment="local write C address")) + + module.addSpaceLine() + # calculate local read address : v[vgprLocalReadAddrC] + + module.addComment0("Store Remap Local Read address") + + module.add(vectorStaticDivideAndRemainder(tid1, tid0, "Serial", self.kernel["WavefrontSize"], \ + RegisterPoolResource(tmpV0, 2))) + module.add(VMulLOU32(dst=vgpr(waveCoord1), + src0=hex(kernel["MatrixInstN"]//kernel["MIWaveGroup"][0]), src1=vgpr(tid1), comment="coord1 offset of LDS for each Wave")) + + nThreadPerCol = kernel["MacroTile0"] // gwvw + nColPerLoad = self.kernel["WavefrontSize"] // nThreadPerCol + self.storeRemapLrOffset = (kernel["MacroTile0"]+ldsPad) * nColPerLoad + self.storeRemapNCPL = nColPerLoad + + module.add(VLShiftRightB32(dst=vgpr(tmpV1),\ + shiftHex=hex(log2(nThreadPerCol)), src=vgpr(tid0), \ + comment="tid / nThreadPerCol")) + module.add(VAddU32(dst=vgpr(coord1Offset), src0=vgpr(waveCoord1), src1=vgpr(tmpV1), comment="coord1 offset in MacroTile")) + module.add(VMulLOU32(dst=vgpr(tmpV0), src0=vgpr(coord1Offset), src1=vgpr(ldsStride), \ + comment="lds coord1 offset = Col-id* lds stride")) + + module.add(VAndB32(dst=vgpr(coord0), src0=hex(nThreadPerCol-1), src1=vgpr(tid0), + comment="coord0 offset of LDS for each thread")) + module.add(VLShiftLeftB32(dst=vgpr(coord0), shiftHex=hex(log2(gwvw)), src=vgpr(coord0), \ + comment="lds coord0 offset *= gwvw (each thread hold gwvw element)")) + + module.add(VAddLShiftLeftU32( + dst=vgpr(storeRemapLR), \ + src0=vgpr(tmpV0), \ + src1=vgpr(coord0), \ + shiftHex=hex(log2(self.bpeCexternal)), \ + comment="local read C address")) + module.addSpaceLine() + + # calculate global write coord0 and coord1 + module.addComment0("Store Remap global write coord0 and coord1") + module.add(vectorStaticDivideAndRemainder(tid1, tid0, "Serial", self.kernel["WavefrontSize"]*kernel["MIWaveGroup"][0], \ + RegisterPoolResource(tmpV0, 2))) + + ColsPerBlockShape = kernel["MatrixInstN"] * kernel["MatrixInstBN"] + + module.add(VMulLOU32(dst=vgpr(waveCoord1), src0=hex(ColsPerBlockShape), src1=vgpr(tid1), + comment="coord1 offset of global memory for each Wave")) + + module.add(vectorStaticDivideAndRemainder(tid1, tid0, tid0, self.kernel["WavefrontSize"], \ + RegisterPoolResource(tmpV0, 2))) + module.add(VMadU32U24(dst=vgpr(waveCoord1), src0=(kernel["MatrixInstN"]//kernel["MIWaveGroup"][0]), src1=vgpr(tid1), src2=vgpr(waveCoord1), \ + comment="waveCoord1 += waveCoord0 * MiN / WaveGroupM")) + + module.add(VLShiftRightB32(dst=vgpr(tmpV1), shiftHex=hex(log2(nThreadPerCol)), src=vgpr(tid0), \ + comment="tid / nThreadPerCol")) + + module.add(VAddU32(dst=vgpr(coord1Offset), src0=vgpr(waveCoord1), src1=vgpr(tmpV1), comment="coord1 offset in MacroTile")) + + module.add(SMulI32(dst=sgpr(tmpS0), src0=hex(kernel["MacroTile0"]), src1=sgpr(wg0), comment="%s = wg0*MT0"%sgpr(tmpS0))) + + module.add(VAddCOU32(dst=vgpr(tid0), dst1=VCC(), src0=sgpr(tmpS0), src1=vgpr(coord0), comment="coord0 = coord0 + wg0 * MT0")) + + module.add(SMulI32(dst=sgpr(wgMT1), src0="MT1", src1=sgpr(wg1), comment="<- wg1*MT1")) + module.add(VAddCOU32(dst=vgpr(tid1), dst1=VCC(), src0=sgpr(wgMT1), src1=vgpr(coord1Offset), comment="coord1 = tid1*VW + wg1*MT1")) + + module.addSpaceLine() + + module.add(self.syncThreads(kernel, "StoreRemap Start")) + + self.vgprs.storeRemapLW = storeRemapLW #local write + self.vgprs.storeRemapLR = storeRemapLR #local read + self.vgprs.storeRemapCoord0 = tid0 #global coord0 + self.vgprs.storeRemapCoord1 = tid1 #global coord1 + self.vgprs.storeRemapOffsetCoord1 = coord1Offset #offset coord1 + + self.vgprPool.checkIn(tmpV0) + + return module + + ############################################################################## + # Not LocalSplitU: Global Write + # Determine write batching pattern + # element() specifies TT 'coordinate' to write + # vectorWidths specifies width of vector to store + # TODO - why does this use VectorWidth to control store width? Could be GlobalWriteVectorWidth? + # + # Function creates one mapping for full tiles and one for edge tiles, + # then calls globalWriteElements to generate the code for the new tiles. + ############################################################################## + def notLocalSplitUGlobalWrite(self, kernel, tPA, tPB): + if not self.do["PostLoop"]: return "" + elements = [[] for y in range(2)] # 2D array for Full, Edge + + (fullVw, elements[False]) = self.notLocalFullTileElements(kernel, False) + (edgeVw, elements[True]) = self.notLocalFullTileElements(kernel, True) + + # print("len(elements[False])= ", len(elements[False])) + # print("len(elements[True])= ", len(elements[True])) + vectorWidths = [fullVw, edgeVw] + + module = Module("notLocalSplitUGlobalWrite") + module.add(self.globalWriteElements(kernel, tPA, tPB, vectorWidths, elements)) + + self.cleanupGlobalWrite(kernel) + + return module + + ############################################################################## + # LocalSplitU: Global Write + ############################################################################## + def localSplitUGlobalWrite(self, kernel, tPA, tPB): + if not self.do["PostLoop"]: return "" + + fullVw = kernel["GlobalWriteVectorWidth"] if kernel["_VectorStore"] else 1 + fullVw = min(fullVw, self.maxGwvw(kernel)) + elements = [[] for y in range(2)] # 2D array for Full, Edge + # Full tile loop: + for tt1 in range(0, kernel["NumGlobalWriteVectorsPerThread"]): + for vc1 in range(0, 1): + for tt0 in range(0, 1): + for vc0 in range(0, kernel["GlobalWriteVectorWidth"], fullVw): # note step by fullVw + element = (tt1, tt0, vc1, vc0) + elements[False].append(element) + + # Edge tile loop - note if we know AF0EM we can can use a larger vector + # and reduce the boundary checks accordingly. But if no AF0EM guarantee + # then use a conservative 1 + edgeVw = kernel["GlobalWriteVectorWidth"] if kernel["_VectorStore"] else 1 + edgeVw = min(edgeVw, self.maxGwvw(kernel), kernel["AssertFree0ElementMultiple"]) + assert(kernel["GlobalWriteVectorWidth"]%edgeVw == 0) + for tt1 in range(0, kernel["NumGlobalWriteVectorsPerThread"]): + for vc1 in range(0, 1): + for tt0 in range(0, 1): + for vc0 in range(0, kernel["GlobalWriteVectorWidth"], edgeVw): + element = (tt1, tt0, vc1, vc0) + elements[True].append(element) + + vectorWidths = [fullVw, edgeVw] + module = Module("localSplitUGlobalWrite") + module.add(self.globalWriteElements(kernel, tPA, tPB, vectorWidths, elements)) + self.cleanupGlobalWrite(kernel) + return module + + ############################################################################## + # checkIsBetaZero + # tmpSgpr is one temp sgpr + # betaLabel is label to branch to if beta != 0 + ############################################################################## + def checkIsBetaZero(self, kernel, tmpSgpr, betaLabel, isLongBranch=False, posNeg: int=0): + module = Module("checkIsBetaZero label %s"%betaLabel) + assert(isinstance(betaLabel, Label)) + betaLabelName = betaLabel.getLabelName() + if kernel["ProblemType"]["UseBeta"]: + if self.bpeCinternal <= self.bpr: # 1 register to check for Beta==0 + module.add(SCmpKEQU32(src=sgpr("Beta"), simm16=hex(0), comment="Beta == 0")) + else: # multiple registers to check for Beta==0 + module.add(SMovB32(dst=sgpr(tmpSgpr), src=sgpr("Beta+0"), comment="tmp = Beta[0]")) + for i in range(1, self.bpeCinternal//self.bpr): + module.add(SOrB32(dst=sgpr(tmpSgpr), src0=sgpr("Beta+%u"%i), src1=sgpr(tmpSgpr), comment="tmp |= Beta[%u] " % i)) + module.add(SCmpKEQU32(src=sgpr(tmpSgpr), simm16=hex(0), comment="Beta == 0")) + if isLongBranch: + module.add(self.longBranchScc0(betaLabel, posNeg)) + else: + module.add(SCBranchSCC0(labelName=betaLabelName, comment="Branch if Beta is not zero")) + module.addSpaceLine() + return module + + ############################################################################## + # checkIsEdge + # tmpSgpr must have at least 6 free SGPR + # isEdgeTarget is the branch target if edges are required + ############################################################################## + def checkIsEdge(self, kernel, tmpSgpr, isEdgeTarget): + assert(isinstance(isEdgeTarget, Label)) + isEdgeTargetLabel = isEdgeTarget.getLabelName() + module = Module("checkIsEdge") + tmpS0 = tmpSgpr + tmpS1 = tmpS0 + 1 + tmpS23 = tmpS1 + 1 + + wg0="WorkGroup0" + wg1="WorkGroup1" + + # check edge0 ### + # s23 = rMT0 = Size0 % MT0 + #-- + sizeBoundary = [0,0] + sizeBoundary[0] = \ + sgpr("PackedSize0") if len(kernel["PackedC0IndicesX"]) > 1 \ + else self.sizeRef(kernel["ProblemType"]["Index0"]) + sizeBoundary[1] = \ + sgpr("PackedSize1") if len(kernel["PackedC1IndicesX"]) > 1 \ + else self.sizeRef(kernel["ProblemType"]["Index1"]) + + module.add(scalarStaticDivideAndRemainder(tmpS1, tmpS0, sizeBoundary[0], kernel["MacroTile0"], \ + RegisterPoolResource(tmpS23, 2), 2)) + # s23 = nwg0-1 + module.add(SAddU32(dst=sgpr(tmpS1), src0=hex(-1), src1=sgpr("NumWorkGroups0"))) + module.add(SCmpGeU32(src0=sgpr(wg0), src1=sgpr(tmpS1), comment="wg0 >= nwg0-1 ?")) + module.add(SCSelectB32(dst=sgpr(tmpS0), src0=sgpr(tmpS0), src1=0, comment="set rMT0")) + # s01 now = myMT0 = wg0 < nwg0-1 ? MT0 : rMT0 + + # if rMT0 > 0 goto label_B?_E1 + if self.do["EdgeWrite"]: + module.add(SCmpKGtU32(src=sgpr(tmpS0), simm16=hex(0), comment="rMT0 > 0")) + if self.db["ForceEdgeStores"]: + module.add(SCmpEQU32(src0=sgpr(tmpS0), src1=sgpr(tmpS0), comment="ForceEdgeStores!")) + module.add(SCBranchSCC1(labelName=isEdgeTargetLabel, comment="jump if edges required")) + + # check edge1 ### + # TODO-packed - this only needs to change to handle packing into C1 index + # change would be similar to above - multiply by product of packed sizes in C1 + # -- + + # s23 = rMT1 = Size1 % MT1 + module.add(scalarStaticDivideAndRemainder(tmpS1, tmpS0, sizeBoundary[1], kernel["MacroTile1"], \ + RegisterPoolResource(tmpS23, 2), 2)) + # s01 now = myMT1 = wg1 < nwg1-1 ? MT1 : rMT1 + + # s23 = nwg1-1 + module.add(SAddU32(dst=sgpr(tmpS1), src0=hex(-1), src1=sgpr("NumWorkGroups1"))) + module.add(SCmpGeU32(src0=sgpr(wg1), src1=sgpr(tmpS1), comment="wg1 >= nwg1-1")) + module.add(SCSelectB32(dst=sgpr(tmpS0), src0=sgpr(tmpS0), src1=0, comment="set rMT1")) + + # if rMT1 > 0 goto label_B?_E1 + if self.do["EdgeWrite"]: + module.add(SCmpKGtU32(src=sgpr(tmpS0), simm16=hex(0), comment="rMT1 > 0")) + module.add(SCBranchSCC1(labelName=isEdgeTargetLabel, comment="jump if edges required")) + + return module + + ############################################################################## + # Global Write Elements + ############################################################################## + + class BF16CVTVgprStruct(NamedTuple): # class for bf16 vgprs + vgprBf16Temp: int = -1 + vgprBf16Mask: int = -1 + vgprFp32Nan: int = -1 + vgprBf16Inc: int = -1 + + class ActivationSetPCStruct(NamedTuple): + sgprOffsetActivation: int = -1 + sgprOffsetBack: int = -1 + vgprActCopy: int = -1 + + def globalWriteElements(self, kernel, tPA, tPB, vectorWidths, elements, + applyAlpha=True, # defaults to generating *=alpha codes + betas=None, # if left unspecified, then let global parameter decide + edges=None): + if not self.do["PostLoop"]: return Module("GlobalWriteElements (Empty)") + module = Module("GlobalWriteElements") + atomic = (kernel["GlobalSplitU"] > 1) and (kernel["_GlobalAccumulation"] != 'MultipleBuffer') + activation = ActivationModule() + + # write possibilities and labels + # if beta/edge combo not specified fall back to global param definition + if betas is None: + hasBeta = kernel["ProblemType"]["UseBeta"] and (kernel["_GlobalAccumulation"] != 'MultipleBuffer') + betas = [False, True] if hasBeta else [False] + if edges is None: + edges = [False, True] if self.do["EdgeWrite"] else [False] + writeLabels = {} + for beta in betas: + writeLabels[beta] = {} + for edge in edges: + writeLabels[beta]["EdgeCheck0"] = Label(self.labels.getNameInc("GW_B%u_E%u_EdgeCheck0" % ( 1 if beta else 0, 1 if edge else 0) ), "") + writeLabels[beta]["EdgeCheck1"] = Label(self.labels.getNameInc("GW_B%u_E%u_EdgeCheck1" % ( 1 if beta else 0, 1 if edge else 0) ), "") + writeLabels[beta][edge] = Label(self.labels.getNameInc("GW_B%u_E%u" % ( 1 if beta else 0, 1 if edge else 0) ), "") + if not beta: + betaLabel = Label(self.labels.getNameInc("GW_Beta"), "") + endLabel = Label(self.labels.getNameInc("GW_End"), "") + + # Layout + """ + if B1 goto label_B1 + if E1 goto label_B0_E1 + label_B0_E0: + writes + goto label_End + label_B0_E1: + writes + goto label_End + label_B1: + if E1 goto label_B1_E1 + label_B1_E0: + writes + goto label_End + label_B1_E1: + writes + goto label_End + label_End + """ + + ######################################## + # Vgprs + if kernel["BufferStore"]: + numTmpVgpr = 2 + if len(kernel["PackedC0IndicesX"]) > 1: + numTmpVgpr += 1 + else: + numTmpVgpr = 2 + 3 # GLOBAL_OFFSET_C needs 3, plus 2 tmps? + # Get max vgpr and sgpr for activation + actPCGwvwVgpr = 0 + actPCMaxTempSgpr = 0 + actTempSgpr = 0 + if kernel["ActivationFuncCall"]: + maxVw = max(vectorWidths) + usage = activation.getAllGprUsage(kernel["ProblemType"]["ActivationComputeDataType"]) + actPCMaxTempVgpr = 0 + for _, gprs in usage.items(): + actPCMaxTempVgpr = max(actPCMaxTempVgpr, gprs["vgpr"]) + actPCMaxTempSgpr = max(actPCMaxTempSgpr, gprs["sgpr"]) + actPCGwvwVgpr = int(ceil(maxVw * kernel["ProblemType"]["ActivationComputeDataType"].numRegisters())) + numTmpVgpr = max(numTmpVgpr, actPCMaxTempVgpr + actPCGwvwVgpr) + tmpVgpr = self.vgprPool.checkOutAligned(numTmpVgpr, 2, "store tmps") + + bf16CVTVgprStruct = None + bf16CVTVgpr = None + if kernel["ProblemType"]["DestDataType"].isBFloat16() and kernel["ProblemType"]["HighPrecisionAccumulate"]: + bf16CVTVgpr = self.vgprPool.checkOut(4) + bf16CVTVgprStruct = self.BF16CVTVgprStruct(vgprBf16Temp=bf16CVTVgpr, vgprBf16Mask=(bf16CVTVgpr+1), \ + vgprFp32Nan=(bf16CVTVgpr+2), vgprBf16Inc=(bf16CVTVgpr+3)) + elif kernel["ProblemType"]["DataType"].isBFloat16() and kernel["ProblemType"]["UseBias"]: + bf16CVTVgpr = self.vgprPool.checkOut(1) + bf16CVTVgprStruct = self.BF16CVTVgprStruct(vgprBf16Temp=None, vgprBf16Mask=(bf16CVTVgpr), \ + vgprFp32Nan=None, vgprBf16Inc=None) + + activationSetPCStruct = None + activationLabelList = None + activationEnumStrList = None + toActModuleList = None + isInsertActFunctionCallAddrCalc = True + if kernel["ActivationFuncCall"]: + sgprOffsetActivation = self.sgprPool.checkOut(2) + sgprOffsetBack = self.sgprPool.checkOut(2) + activationSetPCStruct = self.ActivationSetPCStruct(sgprOffsetActivation=sgprOffsetActivation, \ + sgprOffsetBack=sgprOffsetBack, vgprActCopy=tmpVgpr) + activationCDataType = kernel["ProblemType"]["ActivationComputeDataType"] + activationLabelList = {} + toActModuleList = {} + activationEnumStrList = ActivationType.getEnumStrList(activationCDataType) + for gwvw in vectorWidths: + if gwvw in activationLabelList: + continue + activationLabelList[gwvw] = [] + toActModuleList[gwvw] = [] + for enumStr in activationEnumStrList: + name = self.labels.getNameInc("Activation_%s_VW%u"% (enumStr.capitalize(), gwvw)) + activationLabelList[gwvw].append(Label(name, "")) + toActModuleList[gwvw].append(Label("To_%s"% (name), "")) + # Add branch here if all elements are identical + if vectorWidths.count(vectorWidths[0]) == len(vectorWidths): + isInsertActFunctionCallAddrCalc = False + module.add(self.insertActFunctionCallAddrCalc(activationSetPCStruct.sgprOffsetActivation, \ + vectorWidths[0], toActModuleList, activationEnumStrList, activationLabelList)) + + ######################################## + # Sgprs + + # allocate tmps for the store header (before the batch implementations) + # branch B1 or B0 + betaLabel = Label(self.labels.getNameInc("GW_Beta"), "") + + betaModules = Module("Betas") + currentInstLength = 0 + for idx0 in reversed(range(len(betas))): + beta = betas[idx0] + betaModule = Module("Beta_%u"%idx0) + # start B1 + if beta: + betaModule.add(betaLabel) + + ######################################## + # branch if Edge0 or Edge1 + if False in edges and True in edges: + with self.allocTmpSgpr(4) as tmpSgprInfo: + betaModule.add(self.checkIsEdge(kernel, tmpSgprInfo.idx, writeLabels[beta][True])) + + mod_pos = len(betaModule.items()) + currentInstLength += betaModule.countType(Instruction) + # by now we either jumped to E1 or stayed at E0 + for idx1 in reversed(range(len(edges))): + edge = edges[idx1] + edgeModule = Module("edge_%u"%idx1) + edgeModule.add(writeLabels[beta][edge]) + + # for storeRemap edge case, non-beta still can enable vector stores + if kernel["StoreRemapVectorWidth"] and not beta: + edgeI = False + else: + edgeI = edge + #edgeI = True # set to True to disable vector stores + gwvw = vectorWidths[edgeI] + + #print "globalWriteElements: edge=", edge, "beta=", beta, "atomic=", atomic + + ######################################## + # Calculate Vgprs for Write Batching + ######################################## + + ss = StoreState(self, kernel, gwvw, edge, beta, atomic, elements[edgeI]) + + # how many vgprs are needed for zero elements + # 2 for addressC in vgpr for addition - already checked out + # 2 for coord0,1 of thread - already checked out + # 2 for tmp - already checked out + + # 5 = how many vgprs are needed per element (flat) + # - 2 for addr + # - 3 for GLOBAL_OFFSET_C calculation (can overlap below, therefore max) + # - if beta gwvw*rpe for new value + # - if atomic 2*rpe for old and cmp values + + #print self.vgprPool.state() + # Use VGPR up to next occupancy threshold: + maxVgprs = self.getMaxRegsForOccupancy(kernel["NumThreads"], self.vgprPool.size(), \ + self.getLdsSize(kernel), self.agprPool.size(), self.unifiedVgprRegs) + if self.serializedStore: # get aggressive when serializedStore is on; not necessarily exclusive to this parameter + _growPool(self.vgprPool, self.vgprPool.size()-self.vgprPool.available(), maxVgprs, 1, \ + "grow-pool up to next occupancy for GlobalWrite") + # Get numVgprAvailable + numVgprAvailable = self.vgprPool.availableBlock(ss.numVgprsPerElement, ss.align) + + # Grow the register pool if needed - we need enough regs for at least one element + # Unfortunate since this means the write logic is setting the VGPR requirement + # for the entire kernel but at least we have a functional kernel. + # Before growing the pool, see if we can shrink the write vector width instead? + # TODO : the vgprSerial is needed for-ever and if we grow here will split the + # range of the tmps. Maybe want to move vgprSerial to first vgpr? + + # TODO: Minimum elems for StoreRemap + # TODO: Which of DataType or DestDataType is in a better sense? 0114: Check Using DestDataType + HSS + minElements = 2 if (kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16()) else 1 + minNeeded = minElements * ss.numVgprsPerElement + shrinkDb = 0 + if shrinkDb: + print("numVgprAvailable=", numVgprAvailable, "minElements=", minElements, "minNeeded=", minNeeded) + if numVgprAvailable < minNeeded: + gwvwOrig = gwvw + currentOccupancy = self.getOccupancy(kernel["NumThreads"], self.getLdsSize(kernel), \ + self.vgprPool.size(), self.agprPool.size(), self.unifiedVgprRegs) + futureOccupancy = self.getOccupancy(kernel["NumThreads"], self.getLdsSize(kernel), \ + self.vgprPool.size() - numVgprAvailable + minNeeded, self.agprPool.size(), self.unifiedVgprRegs) + + if shrinkDb: + print("currentOccupancy=%u futureOccupancy=%u VGPRs=%u numVgprAvail=%u vgprPerElem=%u" \ + % (currentOccupancy, futureOccupancy, self.vgprPool.size(), \ + numVgprAvailable, minElements*ss.numVgprsPerElement)) + if futureOccupancy > currentOccupancy: + if shrinkDb: + print("warning: %s growing VGPR for GlobalWrite batching - this may bloat VGPR usage" % \ + (self.kernelName)) + print(" numVgprAvailable=", numVgprAvailable, \ + "numVgprsPerElement=", ss.numVgprsPerElement, "atomic=", atomic, \ + "beta=", beta, "gwvw=", gwvw) + elif gwvw != gwvwOrig: + ss.cfg.gwvw = gwvw # make both representations consistent + if shrinkDb: + print2("info: %s shrank gwvw from %u to %u but kept occupancy same=%u." \ + % (self.kernelName, gwvwOrig, gwvw, currentOccupancy)) + + if numVgprAvailable < minElements*ss.numVgprsPerElement: + print2("info: growing pool += %d * %d for GlobalWrite\n" \ + % (minElements,ss.numVgprsPerElement)) + print2(self.vgprPool.state()) + _growPool(self.vgprPool, 0, minElements, ss.numVgprsPerElement, \ + "grow-pool for GlobalWrite") + numVgprAvailable = self.vgprPool.available() + print2(self.vgprPool.state()) + + # set atomicW after we potentially resize GWVW + atomicW = min(gwvw, self.getVectorAtomicWidth(kernel)) + + # print("NumVgprAvailable", numVgprAvailable) + if ss.numVgprsPerElement: + numElementsPerBatch = numVgprAvailable // ss.numVgprsPerElement + else: + numElementsPerBatch = len(elements[edgeI]) # max, do 'em all + + assert(self.numVgprValuC % gwvw == 0) # sanity check + + numElementsPerBatch = numElementsPerBatch if not kernel["NumElementsPerBatchStore"] else min(kernel["NumElementsPerBatchStore"],numElementsPerBatch) + + if shrinkDb: + print("NumElementsPerBatch=", numElementsPerBatch, "LimitedBySgprs=", ss.cfg.numElementsPerBatchLimitedBySgprs, \ + "WARNING" if ss.cfg.numElementsPerBatchLimitedBySgprs < numElementsPerBatch else "okay") + if ss.cfg.numElementsPerBatchLimitedBySgprs < numElementsPerBatch: + numElementsPerBatch = ss.cfg.numElementsPerBatchLimitedBySgprs + + # TODO: Which of DataType or DestDataType is in a better sense? 0114: Check Using DestDataType + HSS + if (kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16()): + # only do an even number of halves - since these share hi/lo pieces of some registers? + if numElementsPerBatch > 1: + numElementsPerBatch = int(numElementsPerBatch/2)*2 + elif not kernel["EnableMatrixInstruction"]: + # The globalWriteBatch routine below can't handle odd elements per batch + # and 0 elements per batch is illegal. + # so if we don't have *GPR resources to handle a larger batch then need + # to mark overflowedResources rather than generate a kernel that won't work. + # It might be possible to fix globalWriteBatch to handle this case but these + # are likely to be low-performing so likely not worth optimizing. + if shrinkDb: + print("WARNING: half requires at least two elements per batch") + self.overflowedResources = 3 + + assert numElementsPerBatch > 0, "numElementsPerBatch=0 for %s"%self.kernelName + + #numElementsPerBatch=min(2,numElementsPerBatch) # hack to control number of batches + if atomic and (ss.optSingleColVgpr or ss.optSharedColVgpr): + # hack to avoid re-using address vgpr across rows + # atomics need to perform several memory operations + # if the batch spans multiple rows, need multiple address vgpr + # which is not currently supported in the two opt*ColVgpr modes + firstRow = [e for e in elements[edgeI] if e[0]==0 and e[2]==0] + numElementsPerBatch=min(len(firstRow),numElementsPerBatch) + + # check best numElementsPerBatch to handle a column block + # elements of column block must be multiple size of numElementsPerBatch + if kernel["StoreRemapVectorWidth"]: + firstRow = [e for e in elements[edgeI] if e[0]==0 and e[2]==0] # format for element = (tt1, tt0, vc1, vc0) + # find the largest factor and smaller than numElementPerBatch + nBatchesPerRow = 1 + for d in range(1, len(firstRow)+1): + largestFactor = len(firstRow)//d + if len(firstRow)%d == 0 and largestFactor <= numElementsPerBatch: + numElementsPerBatch = largestFactor + nBatchesPerRow = d + break + + # if no atomics and no edge, then write whole vectors + #if not atomic and not edge: + # numVectorsPerBatch = numElementsPerBatch / kernel["GlobalWriteVectorWidth"] + # #print " NumVectorsPerBatch", numVectorsPerBatch + # numElementsPerBatch = numVectorsPerBatch * kernel["GlobalWriteVectorWidth"] + numBatches = max(1, ceilDivide(len(elements[edgeI]),numElementsPerBatch)) + + numSgprs = ss.cfg.numTempSgprPerBatch + ss.cfg.numMaskSgprPerBatch + ss.cfg.numMaskSgprPerElement * numElementsPerBatch + + if activationLabelList and isInsertActFunctionCallAddrCalc: + assert activationSetPCStruct, activationEnumStrList and activationLabelList and toActModuleList + numSgprs = max(actPCMaxTempSgpr, numSgprs) + edgeModule.add(self.insertActFunctionCallAddrCalc(activationSetPCStruct.sgprOffsetActivation, \ + gwvw, toActModuleList, activationEnumStrList, activationLabelList, \ + idx0, idx1)) + + if self.db["PrintStoreRegisterDb"]: + print("edgeI", edgeI, "NumBatches", numBatches, "NumElementsPerBatch", numElementsPerBatch, "numVgprsPerElement", ss.numVgprsPerElement, "len(elements[edgeI])", len(elements[edgeI])) + print ("numSgprs=", numSgprs, "sgprPool.size()=", self.sgprPool.size(), "numTempSgprPerBatch=", ss.cfg.numTempSgprPerBatch, + "numMaskSgprPerBatch=", ss.cfg.numMaskSgprPerBatch, "numMaskSgprPerElement=", ss.cfg.numMaskSgprPerElement) + print(self.sgprPool.state()) + edgeModule.addComment1("edge=%d, allocate %u sgpr. perBatchTmpS=%u perBatchMaskS=%u perElementMaskS=%u elementsPerBatch=%u" % + (edgeI, numSgprs, ss.cfg.numTempSgprPerBatch, ss.cfg.numMaskSgprPerBatch, ss.cfg.numMaskSgprPerElement, numElementsPerBatch)) + #edgeModule.addComment("storeStats, %d, %d, %d"% (edgeI, numSgprs, numElementsPerBatch)) + # so if we don't have *GPR resources to handle a larger batch then need + # to mark overflowedResources rather than generate a kernel that won't work. + with self.allocTmpSgpr(numSgprs, 2) as tmpSgprRes: + tmpSgpr = tmpSgprRes.idx + actTempSgpr = tmpSgpr # Get sgpr start address, should always be the same + elementSgprs = tmpSgpr + ss.cfg.numTempSgprPerBatch + codeAccVgprRead = deepcopy(self.codeAccVgprRead) if self.serializedStore else None + codeMulAlpha = deepcopy(self.codeMulAlpha) if self.serializedStore else None + + self.alphaBeforeLoadC = False + if kernel["MIArchVgpr"] and applyAlpha: + codeAccVgprRead = None + + #Only apply when 2 wave optimization features are enabled + if (kernel["StorePriorityOpt"] or kernel["StoreSyncOpt"]) and beta: + self.alphaBeforeLoadC = True + else: + codeMulAlpha = None + + for batchIdx in range(0, numBatches): + elementStartIdx = batchIdx * numElementsPerBatch + elementStopIdx = min( elementStartIdx + numElementsPerBatch, len(elements[edgeI]) ) + elementsThisBatch = elements[edgeI][elementStartIdx:elementStopIdx] + #print("BATCH[%u/%u]: elements[edgeI][%u:%u] VGPRs=%u" % (batchIdx, numBatches, elementStartIdx, elementStopIdx,ss.numVgprsPerElement )) + # elementVgprs can be large and should be perfectly tuned to the number of available + # VGPRS. We do not want to accidentally overflow and grow the pool here: + + if kernel["StoreRemapVectorWidth"]: + #Indication if this batch is last batch for this column block shape + self.StoreRemapLastBatch = 1 if (batchIdx+1) % nBatchesPerRow == 0 else 0 + + edgeModule.add(self.globalWriteBatch(kernel, tPA, tPB, activation, ss, batchIdx, \ + applyAlpha, beta, edge, atomic, gwvw, atomicW, \ + elementsThisBatch, self.vgprs.addrD, self.vgprs.addrC, self.vgprs.addrBias, \ + tmpVgpr, bf16CVTVgprStruct, activationSetPCStruct, \ + elementSgprs, tmpSgpr, codeAccVgprRead, codeMulAlpha)) + + + if currentInstLength >= 16384: + with self.allocTmpSgpr(3) as tmpSgprInfo: + edgeModule.add(SLongBranchPositive(endLabel, tmpSgprInfo, comment="jump to end")) + else: + edgeModule.add(SBranch(labelName=endLabel.getLabelName(), comment="jump to end")) + currentInstLength += edgeModule.countType(Instruction) + betaModule.add(edgeModule, pos=mod_pos) + del ss + betaModules.add(betaModule, pos=0) + + # Check if branch exceeds + if False in betas and True in betas: + isBetaLongBranch = False + def findInstCount(module, targetItem, count): + for inst in module.items(): + if isinstance(inst, Module): + count, found = findInstCount(inst, targetItem, count) + if found: + return count, True + elif (inst is targetItem): + return count, True + elif (not isinstance(inst, TextBlock)): + count += 1 + return count, False + count = 0 + count, found = findInstCount(betaModules, betaLabel, count) + if found: + if count >= 16384: + isBetaLongBranch = True + with self.allocTmpSgpr(1) as tmpSgprInfo: + module.add(self.checkIsBetaZero(kernel, tmpSgprInfo.idx, betaLabel, isBetaLongBranch, posNeg=1)) + module.appendModule(betaModules) + + if activationLabelList: + assert activationEnumStrList and activationSetPCStruct + for key, activationLabelModules in activationLabelList.items(): + gwvw = key + actModules = Module(getActivationFunctionModuleName(gwvw, \ + activationSetPCStruct.vgprActCopy, tmpVgpr, actTempSgpr)) + for index, activationLabelModule in enumerate(activationLabelModules): + actModule = Module(activationLabelModule.getLabelName()) + actModule.add(activationLabelModule) + activationTypeStr = activationEnumStrList[index] + actModule.appendModule(self.getActivationActivationComputeType(kernel, activation, \ + activationTypeStr, gwvw, activationSetPCStruct.vgprActCopy, (tmpVgpr + actPCGwvwVgpr), \ + actTempSgpr)) + actModule.add(SSetPCB64(src=sgpr(activationSetPCStruct.sgprOffsetBack,2))) + actModules.add(actModule) + module.add(actModules) + self.sgprPool.checkIn(activationSetPCStruct.sgprOffsetActivation) + self.sgprPool.checkIn(activationSetPCStruct.sgprOffsetBack) + + # End label + module.add(endLabel) + self.vgprPool.checkIn(tmpVgpr) + if bf16CVTVgpr is not None: + self.vgprPool.checkIn(bf16CVTVgpr) + return module + + ############################################################################## + # chooseGlobalRead : + # create the load instruction for requested vector width and other parms + # return an Inst class + # + # bpl = bytes per load op + ############################################################################## + def chooseGlobalRead(self, useBuffer, bpl, destVgpr, \ + addr0, addr1, soffset, offset, \ + glc=False, slc=False, lds=False, hi16=0, comment="load C"): + # rpv = regs per vector + rpv = bpl/4.0 + + if useBuffer: + rv = Module("Global Read") + mubuf = MUBUFModifiers(offen=True, offset12=offset, glc=glc, slc=slc, lds=lds) + + # Nested buffer load implementation function for easy branching for soffset + def bufferLoadImpl(soffset): + nonlocal rv + if bpl==1 and hi16: + rv.add(BufferLoadD16HIU8(dst=vgpr(destVgpr, rpv*4), vaddr=addr0, saddr=addr1, \ + soffset=soffset, mubuf=mubuf, comment=comment)) + return rv + elif bpl==1 and not hi16: + rv.add(BufferLoadD16U8(dst=vgpr(destVgpr, rpv*4), vaddr=addr0, saddr=addr1, \ + soffset=soffset, mubuf=mubuf, comment=comment)) + return rv + elif bpl==2 and hi16: + rv.add(BufferLoadD16HIB16(dst=vgpr(destVgpr, rpv*2), vaddr=addr0, saddr=addr1, \ + soffset=soffset, mubuf=mubuf, comment=comment)) + return rv + elif bpl==2 and not hi16: + rv.add(BufferLoadD16B16(dst=vgpr(destVgpr, rpv*2), vaddr=addr0, saddr=addr1, \ + soffset=soffset, mubuf=mubuf, comment=comment)) + return rv + elif bpl==4: + rv.add(BufferLoadB32(dst=vgpr(destVgpr, rpv), vaddr=addr0, saddr=addr1, \ + soffset=soffset, mubuf=mubuf, comment=comment)) + return rv + elif bpl==8: + rv.add(BufferLoadB64(dst=vgpr(destVgpr, rpv), vaddr=addr0, saddr=addr1, \ + soffset=soffset, mubuf=mubuf, comment=comment)) + return rv + elif bpl==16: + rv.add(BufferLoadB128(dst=vgpr(destVgpr, rpv), vaddr=addr0, saddr=addr1, \ + soffset=soffset, mubuf=mubuf, comment=comment)) + return rv + elif bpl==32: + # split into two dwordx4 loads. Second load offset is +0.5 bpl + mubuf.offset12 = (offset + bpl/2) + + rv = Module("emulated _buffer_load_b256") + rv.add(BufferLoadB128(dst=vgpr(destVgpr, rpv/2), vaddr=addr0, saddr=addr1, \ + soffset=soffset, mubuf=mubuf, comment=comment)) + rv.add(BufferLoadB128(dst=vgpr(int(destVgpr + rpv/2), rpv/2), vaddr=addr0, saddr=addr1, \ + soffset=soffset, mubuf=mubuf, comment=comment)) + return rv + else: + assert 0, "chooseGlobalRead: bad bpl" + + # buffer_load offset field is 12-bit. + # if offset >= 4096, use soffset instead + if offset >= 4096: + if soffset in (0, "0"): + mubuf.offset12 = 0 + with self.allocTmpSgpr(1) as tmpSgprInfo: + soffset = sgpr(tmpSgprInfo.idx) + rv.add(SMovB32(dst=soffset, src=offset, comment="large offset")) + bufferLoadImpl(soffset) + else: + assert 0, "offset too large and soffset set" + else: + bufferLoadImpl(soffset) + return rv + else: + flat = FLATModifiers(glc=glc, slc=slc, lds=lds) + if bpl==2 and hi16: + return FlatLoadD16HIB16(dst=vgpr(destVgpr, rpv*2), vaddr=addr0, flat=flat, comment=comment) + elif bpl==2 and not hi16: + return FlatLoadD16B16(dst=vgpr(destVgpr, rpv*2), vaddr=addr0, flat=flat, comment=comment) + elif bpl==4: + return FlatLoadB32(dst=vgpr(destVgpr, rpv), vaddr=addr0, flat=flat, comment=comment) + elif bpl==8: + return FlatLoadB64(dst=vgpr(destVgpr, rpv), vaddr=addr0, flat=flat, comment=comment) + elif bpl==16: + return FlatLoadB128(dst=vgpr(destVgpr, rpv), vaddr=addr0, flat=flat, comment=comment) + else: + assert 0, "chooseGlobalRead: bad bpl" + + ############################################################################## + def chooseGlobalWrite(self, useBuffer, bps, srcVgpr, rpv, \ + addr0, addr1, offset, glc=False, slc=False, hi16=0): + """ + create the store instruction for requested vector width and other parms + rpv = regs per vector + """ + + module = Module("chooseGlobalWrite %s -> %s (%s)"%(srcVgpr, addr0, addr1)) + + def bufferStoreImpl(tmpSgpr, mubuf): + if bps==2 and hi16: + module.add(BufferStoreD16HIB16(src=vgpr(srcVgpr, rpv*2), vaddr=addr0, \ + saddr=addr1, soffset=tmpSgpr, mubuf=mubuf, comment="store D")) + elif bps==2 and not hi16: + module.add(BufferStoreB16(src=vgpr(srcVgpr, rpv*2), vaddr=addr0, \ + saddr=addr1, soffset=tmpSgpr, mubuf=mubuf, comment="store D")) + elif bps==4: + module.add(BufferStoreB32(src=vgpr(srcVgpr, rpv), vaddr=addr0, \ + saddr=addr1, soffset=tmpSgpr, mubuf=mubuf, comment="store D")) + elif bps==8: + module.add(BufferStoreB64(src=vgpr(srcVgpr, rpv), vaddr=addr0, \ + saddr=addr1, soffset=tmpSgpr, mubuf=mubuf, comment="store D")) + elif bps==16: + module.add(BufferStoreB128(src=vgpr(srcVgpr, rpv), vaddr=addr0, \ + saddr=addr1, soffset=tmpSgpr, mubuf=mubuf, comment="store D")) + elif bps == 32: + # split into two dwordx4 loads. Offset the second by +0.5 bps + module.add(BufferStoreB128(src=vgpr(srcVgpr, rpv/2), vaddr=addr0, \ + saddr=addr1, soffset=tmpSgpr, mubuf=mubuf, comment="store D")) + + module.add(BufferStoreB128(src=vgpr(int(srcVgpr +rpv/2), rpv/2), vaddr=addr0, \ + saddr=addr1, soffset=tmpSgpr, mubuf=mubuf, comment="store D")) + else: + assert 0, "bad bps" + + if useBuffer: + mubuf = MUBUFModifiers(offen=True, offset12=offset, glc=glc, slc=slc) + # buffer_load offset field is 12-bit. + # if offset >= 4096, use soffset instead + if offset >= 4096: + with self.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr = sgpr(tmpSgprInfo.idx) + module.add(SMovB32(dst=tmpSgpr, src=offset, comment="large offset")) + mubuf.offen = False + mubuf.offset12 = 0 + bufferStoreImpl(tmpSgpr, mubuf) + else: + bufferStoreImpl(0, mubuf) + + else: + flat = FLATModifiers(glc=glc, slc=slc) + if bps==2 and hi16: + module.add(FlatStoreD16HIB16(vaddr=addr0, src=vgpr(srcVgpr*2), flat=flat, comment="store D")) + elif bps==2 and not hi16: + module.add(FlatStoreD16B16(vaddr=addr0, src=vgpr(srcVgpr, rpv*2), flat=flat, comment="store D")) + elif bps==4: + module.add(FlatStoreB32(vaddr=addr0, src=vgpr(srcVgpr, rpv), flat=flat, comment="store D")) + elif bps==8: + module.add(FlatStoreB64(vaddr=addr0, src=vgpr(srcVgpr, rpv), flat=flat, comment="store D")) + elif bps==16: + module.add(FlatStoreB128(vaddr=addr0, src=vgpr(srcVgpr, rpv), flat=flat, comment="store D")) + else: + assert 0, "bad bps" + + return module + + def addBiasLoad(self, kernel, ss, addrCalc, biasVgpr): + """ + Add bias for the element with addrCalc, elementIdx, and biasVgpr. + biasVgpr is one or more vgpr :temp vGPR ( = gwvw * numbytes // 4 + 1 if cvt is needed) + """ + # Add bias here + module = Module("addBias") + if kernel["ProblemType"]["UseBias"] and (kernel["GlobalSplitU"] == 1): + bps = kernel["ProblemType"]["DataType"].numBytes() * ss.cfg.gwvw + if kernel["BufferLoad"]: + addr0 = vgpr(addrCalc.addrBiasVgpr) + addr1 = sgpr("SrdBias", 4) + else: + addr0 = vgpr(addrCalc.addrBiasVgpr,2) + addr1 = "" + + useBuffer = kernel["BufferLoad"] + if kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16(): + module.add(self.chooseGlobalRead(useBuffer, bps, biasVgpr, \ + addr0, addr1, soffset=0, offset=addrCalc.biasOffset, hi16=0, comment="load bias")) + elif kernel["ProblemType"]["DataType"].isInt32() or kernel["ProblemType"]["DataType"].isSingle(): + module.add(self.chooseGlobalRead(useBuffer, bps, biasVgpr, \ + addr0, addr1, soffset=0, offset=addrCalc.biasOffset, comment="load bias")) + elif kernel["ProblemType"]["DataType"].isDouble() or kernel["ProblemType"]["DataType"].isSingleComplex() : + module.add(self.chooseGlobalRead(useBuffer, bps, biasVgpr, \ + addr0, addr1, soffset=0, offset=addrCalc.biasOffset, comment="load bias")) + else: + printExit("Unsupported bias type %s."%(str(kernel["ProblemType"]["DataType"]))) + return module + + ############################################################################## + def addStore(self, kernel, ss, addrCalc, sumIdx, tmpS01, edge): + """ + Add stores for the element with addrCalc and sumIdx. + tmpS01 is a single :temp sGPR + """ + module = Module("addStore sumIdx %s"%(str(sumIdx))) + if self.do["GlobalWrite"]: + # perform vector stores here, so no VI indexing. + # if GWVW > Vw, might need to support loops to + # implement wider stores + isGlc = False + isSlc = False + if kernel["NonTemporalD"]%2==1: + isGlc = True + if kernel["NonTemporalD"]//2==1: + isSlc = True + + bps = self.bpeCexternal * ss.cfg.gwvw + rpv = self.bpeCexternal * ss.cfg.gwvw / self.bpr + + if kernel["BufferStore"]: + addr0 = vgpr(addrCalc.addrDVgpr) + addr1 = sgpr("SrdD", 4) + else: + addr0 = vgpr(addrCalc.addrDVgpr,2) + addr1 = "" + + useBuffer = kernel["BufferStore"] + if ss.optSrdIncForRow and addrCalc.rowInc: + module.add(addrCalc.incrementToNextRow(kernel, "D", ss, tmpS01)) + if kernel["ProblemType"]["DestDataType"].isHalf() or kernel["ProblemType"]["DestDataType"].isBFloat16(): + + if not kernel["ProblemType"]["HighPrecisionAccumulate"]: + # (H,H,H,H,H,H), internal H + module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx//2, rpv, \ + addr0, addr1, addrCalc.globalOffset, isGlc, isSlc, hi16=sumIdx%2)) + else: + # (B,B,B,B,S,S), internal S + # (H,H,H,H,H,H), internal S + # (H,H,H,H,S,S), internal S + module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx, rpv, \ + addr0, addr1, addrCalc.globalOffset, isGlc, isSlc, hi16=0)) + elif kernel["ProblemType"]["DestDataType"].isInt32() or kernel["ProblemType"]["DestDataType"].isSingle(): + module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx, rpv, \ + addr0, addr1, addrCalc.globalOffset, isGlc, isSlc)) + elif kernel["ProblemType"]["DestDataType"].isDouble() or kernel["ProblemType"]["DestDataType"].isSingleComplex(): + module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx*2, rpv, \ + addr0, addr1, addrCalc.globalOffset, isGlc, isSlc)) + elif kernel["ProblemType"]["DestDataType"].isDoubleComplex(): + rps = kernel["ProblemType"]["DestDataType"].numRegisters() + module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx*rps, rpv, \ + addr0, addr1, addrCalc.globalOffset, isGlc, isSlc)) + elif kernel["ProblemType"]["DestDataType"].isInt8(): + if kernel["ProblemType"]["HighPrecisionAccumulate"]: + module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx, rpv, \ + addr0, addr1, addrCalc.globalOffset, isGlc, isSlc)) + return module + + ############################################################################## + # Global Read C Input + ############################################################################## + def readCInput(self, kernel, ss, addrCalc, vc0, data, gwvw, addr, tmpS01): + module = Module("readCInput") + bps = kernel["ProblemType"]["DestDataType"].numBytes() * gwvw + useBuffer = kernel["BufferStore"] + + if kernel["BufferStore"]: + addr0 = vgpr(addr) + addr1 = sgpr("SrdC", 4) + else: + addr0 = vgpr(addr,2) + addr1 = "" + + isGlc = True if kernel["NonTemporalC"]%2==1 else False + isSlc = True if kernel["NonTemporalC"]//2==1 else False + + if ss.optSrdIncForRow and addrCalc.rowInc: + module.add(addrCalc.incrementToNextRow(kernel, "C", ss, tmpS01)) + + if kernel["ProblemType"]["DestDataType"].isHalf(): + module.add(self.chooseGlobalRead(useBuffer, bps, data, \ + addr0, addr1, soffset=0, offset=addrCalc.globalOffset, \ + glc=isGlc, slc=isSlc, lds=False, hi16=vc0 % 2, + comment="load C for beta calc")) + elif kernel["ProblemType"]["DestDataType"].isInt8(): + module.add(self.chooseGlobalRead(useBuffer, bps, data, \ + addr0, addr1, soffset=0, offset=addrCalc.globalOffset, \ + glc=isGlc, slc=isSlc, lds=False, hi16=vc0 % 4, + comment="load C for beta calc")) + elif kernel["ProblemType"]["DestDataType"].isBFloat16() or \ + kernel["ProblemType"]["DestDataType"].isInt32() or \ + kernel["ProblemType"]["DestDataType"].isSingle() or \ + kernel["ProblemType"]["DestDataType"].isDouble() or \ + kernel["ProblemType"]["DestDataType"].isSingleComplex() or \ + kernel["ProblemType"]["DestDataType"].isDoubleComplex(): + module.add(self.chooseGlobalRead(useBuffer, bps, data, \ + addr0, addr1, soffset=0, offset=addrCalc.globalOffset, \ + glc=isGlc, slc=isSlc, lds=False, \ + comment="load C for beta calc")) + + return module + + ############################################################################## + # Global Write Batch + ############################################################################## + def globalWriteBatch(self, kernel, tPA, tPB, activation, ss: StoreState, batchIdx, \ + applyAlpha, beta, edge, atomic, gwvw, atomicW, \ + batchElements, addrD, addrC, addrBias, \ + tmpVgpr, bf16CVTVgprStruct, activationSetPCStruct, batchElementSgprs, tmpSgpr, codeAccVgprRead, codeMulAlpha) -> Module: + packdata = Component.PackData.find(self) + gwriter = Component.GlobalWriteComponents.find(self) + return gwriter(kernel, tPA, tPB, activation, ss, \ + batchIdx, applyAlpha, beta, edge, atomic, gwvw, atomicW, \ + batchElements, addrD, addrC, addrBias, tmpVgpr, bf16CVTVgprStruct, activationSetPCStruct, \ + batchElementSgprs, tmpSgpr, codeAccVgprRead, codeMulAlpha, packdata, self) + + ############################################################################## + def openPrefetchGlobalRead2(self, kernel): + imod = Module() + loopCounter = self.loopCounter(kernel, self.unrollIdx) + imod.add(SCmpEQU32(src0=loopCounter, src1=hex(1), comment="PGR=2 but only 1 loop")) + skipPGR2 = Label(self.labels.getName("skipPGR2"), "") + imod.add(SCBranchSCC1(labelName=skipPGR2.getLabelName(), comment="PGR=2 but only 1 loop")) + return imod + + def closePrefetchGlobalRead2(self): + imod = Module() + skipPGR2 = Label(self.labels.getName("skipPGR2"), "") + imod.add(skipPGR2) + return imod + + ######################################## + # Activation related + ######################################## + def insertActFunctionCallAddrCalc(self, sgprOffset, gwvw, \ + toActModuleList, activationEnumStrList, activationLabelList, \ + betaIdx = -1, edgeIdx = -1): + activationLabelModules = activationLabelList[gwvw] + module = Module(getActivationBranchModuleName()) + setAddrEndLabel = Label(self.labels.getNameInc("ActivationSetPCAddrEnd"), "") + toActModules = deepcopy(toActModuleList[gwvw]) + for index, toActModule in enumerate(toActModules): + if betaIdx >= 0 and edgeIdx >= 0: + toActModule.label = self.labels.getNameInc(toActModule.label + "_beta_%u_edge_%u"%(betaIdx, edgeIdx)) + if index != 0: + enumIndex = ActivationType.getEnumIndex(activationEnumStrList[index]) + module.add(SCmpKEQU32(sgpr("ActivationType"), enumIndex, "activationType == %u"%enumIndex)) + module.add(SCBranchSCC1(toActModule.getLabelName(), "Branch if true")) + for index, activationLabelModule in enumerate(activationLabelModules): + toActModule = toActModules[index] + module.add(toActModule) + with self.allocTmpSgpr(1) as tmpSgprInfo: + module.addModuleAsFlatItems(SGetPositivePCOffset(sgprOffset, activationLabelModule, tmpSgprInfo)) + module.add(SBranch(setAddrEndLabel.getLabelName())) + module.add(setAddrEndLabel) + return module + + def insertActivationAfterPacked(self, kernel, activationTypeStr): + result = False + if ((kernel["ProblemType"]["ActivationType"] != 'none') and \ + (kernel["GlobalSplitU"] == 1) and kernel["ActivationFused"]): + if kernel["ProblemType"]["ActivationHPA"]: + # Still use BFloat16 for abs. + if kernel["ProblemType"]["DestDataType"].isBFloat16() and (activationTypeStr == 'abs'): + result = True + elif kernel["ProblemType"]["DestDataType"].isHalf() and \ + ((activationTypeStr == 'abs') or (activationTypeStr == 'relu')): + result = True + else: + result = True + return result + + def getActivationDestDataType(self, kernel, activation, activationTypeStr: str, gwvw, \ + elementSumIdx, tmpVgpr, tmpSgpr): + module = Module("ActivationAfterPack") + for vi in range(0, gwvw): + sumIdxV = elementSumIdx + vi + if kernel["ProblemType"]["DestDataType"].isHalf() or \ + kernel["ProblemType"]["DestDataType"].isBFloat16(): + if kernel["ProblemType"]["HighPrecisionAccumulate"]: + # Generate single f16 code if edge is detected. + if ((vi + 1) == gwvw) and ((gwvw % 2) == 1): + activation.setUsePK(False) + # Original packed route + elif vi%2 == 1: + assert (gwvw % 2 == 0) + else: + continue + vgprIdx = elementSumIdx + vi//2 + else: + if (sumIdxV % 2 != 0): + continue + vgprIdx = sumIdxV // 2 + elif kernel["ProblemType"]["DestDataType"].isSingle(): + vgprIdx = sumIdxV + elif kernel["ProblemType"]["DestDataType"].isDouble(): + vgprIdx = sumIdxV * 2 + elif kernel["ProblemType"]["DestDataType"].isInt32(): + vgprIdx = sumIdxV + else: + raise RuntimeError("Unsupported data type %s for activation vgpr index."%str(self.kernel["ProblemType"]["DestDataType"])) + # Here we still use DestDataType cause the data is ready to be written to global + actModule = activation.getModule(self.kernel["ProblemType"]["DestDataType"], activationTypeStr, vgprIdx) + module.add(activation.assignGpr(actModule, tmpVgpr, tmpSgpr)) + activation.setUsePK(True) + return module + + def getActivationActivationComputeType(self, kernel, activation, activationTypeStr: str, gwvw, \ + elementSumIdx, tmpVgpr, tmpSgpr, satInt8=False): + module = Module("ActivationBeforePack") + if satInt8: + activation.setSaturationForInt8(True) + activation.setVgprPrefixFormat("ValuC+%u") + for vi in range(0, gwvw): + vgprIdx = elementSumIdx + vi - self.startVgprValuC + actModule = activation.getModule(kernel["ProblemType"]["ActivationComputeDataType"], activationTypeStr, vgprIdx) + module.add(activation.assignGpr(actModule, tmpVgpr, tmpSgpr)) + activation.setSaturationForInt8(False) + activation.setVgprPrefixFormat("") + return module + + ############################################################################## + # Function End + ############################################################################## + def functionEnd(self, addLabel=True): + imod = Module() + if addLabel: + imod.add(Label("KernelEnd", "")) + imod.add(SEndpgm(comment="Kernel End")) + return imod + + ############################################################################## + # Function Suffix + ############################################################################## + def functionSuffix(self, kernel): + if self.vgprPool.size() > self.maxVgprs: + self.overflowedResources = 1 + elif self.sgprPool.size() > self.maxSgprs: + self.overflowedResources = 2 + + if kernel["ScheduleIterAlg"] == 2 and \ + self.getOccupancy(kernel["NumThreads"], self.vgprPool.size(), \ + self.getLdsSize(kernel), self.agprPool.size(), self.unifiedVgprRegs) < 2: + self.overflowedResources = 6 + + vgprPerCU = 65536 + vgprPerThreadPerOccupancy = vgprPerCU // kernel["NumThreads"] + numWorkGroupsPerCU = vgprPerThreadPerOccupancy // max(self.vgprPool.size(), self.agprPool.size()) + if numWorkGroupsPerCU < 1: + self.overflowedResources = 4 + + module = Module("functionSuffix") + if self.overflowedResources: + module.add(ValueEndif("overflowed resources")) + + self.vgprPool.checkFinalState() + return module + + ############################################################################## + # openOddNoLoadLoopForDTV + # generate open code for DirectToVgpr + odd exit case in noLoadLoop code + ############################################################################## + def openOddNoLoadLoopForDTV(self, name): + module = Module("openOddNoLoadLoopForDTV") + evenStartLabelName = Label.getFormatting("EvenStart" + name) + # odd exit check code + # use OrigLoopCounter & 1 + with self.allocTmpSgpr(1) as tmpSgprInfo: + tmpSgpr = tmpSgprInfo.idx + #scc0or1 = 0 if isNGLL else 1 + #oddOrEven = "Even" if isNGLL else "Odd" + module.add(SAndB32(dst=sgpr(tmpSgpr), src0=sgpr("OrigLoopCounter"), src1=1, comment="test if OrigLoopCounter is Odd ?")) + module.add(SCBranchSCC0(labelName=evenStartLabelName, comment="Skip odd code if OrigLoopCounter is Even")) + + return module + + ############################################################################## + # closeOddNoLoadLoopForDTV + # generate close code for DirectToVgpr + odd exit case in noLoadLoop code + ############################################################################## + def closeOddNoLoadLoopForDTV(self, name): + module = Module("closeOddNoLoadLoopForDTV") + evenStartLabelName = Label.getFormatting("EvenStart" + name) + evenEndLabelName = Label.getFormatting("EvenEnd" + name) + # odd exit code + module.add(SBranch(labelName=evenEndLabelName, comment="Skip even code")) + # generate even start label + module.add(Label(evenStartLabelName, "")) + return module + + ############################################################################## + # generateEvenEndLabeNoLoadLoopForDTV + # generate even end label for DirectToVgpr + ############################################################################## + def generateEvenEndLabeNoLoadLoopForDTV(self, name): + module = Module("generateEvenEndLabeNoLoadLoopForDTV") + # generate even end label + module.add(Label("EvenEnd" + name, "")) + return module + + ############################################################################## + # generateOddEndVgprCopyForDTV + # generate odd end vgpr copy for DirectToVgpr + ############################################################################## + def generateOddEndVgprCopyForDTV(self, kernel): + module = Module("generateOddEndVgprCopyForDTV") + vregNameBase = "G2LA" if kernel["DirectToVgprA"] else "G2LB" + numVreg = self.numVgprG2LA//2 if kernel["DirectToVgprA"] else self.numVgprG2LB//2 + vregSet0 = vregNameBase + "0+" + vregSet1 = vregNameBase + "1+" + self.comment("copy Vreg set1 to Vreg set0 for DirectToVgpr + PrefetchAcrossPersistent") + for index in range(numVreg): + module.add(VMovB32(dst=vgpr(vregSet0+str(index)), src=vgpr(vregSet1+str(index)))) + return module + + ############################################################################## + # WaitCnt- DONE + # 3 components can contribute to the waitcnt: + # - Pending global reads. (skipGlobalRead) + # - Pending local write. (skipLocalWrite) + # - Pending local reads (skipLocalRead) + # If a skip* arg is -1, the associated component does not contribute to + # the expected lgkmcnt or vmcnt + ############################################################################## + def wait(self, kernel, tPA, tPB, skipGlobalRead, skipLocalWrite, \ + skipLocalRead, comment): + if not self.do["Wait"]: return Module("noWait") + # skip = -1 -> ignore + # skip = n -> waitcnt(n*num) + + lgkmcnt = 0 if skipLocalWrite > -1 or skipLocalRead > -1 else -1 + + if skipLocalWrite > -1 or skipLocalRead > -1: + if skipLocalWrite > -1: + numA = 0 if kernel["DirectToLdsA"] \ + else tPA["nrp"]*tPA["nrc"]*max(tPA["nwcv"],tPA["nwpv"])//tPA["nwcvpi"] + numB = 0 if kernel["DirectToLdsB"] \ + else tPB["nrp"]*tPB["nrc"]*max(tPB["nwcv"],tPB["nwpv"])//tPB["nwcvpi"] + lgkmcnt += skipLocalWrite * (numA + numB) + if skipLocalRead > -1: + readsPerIter = self.numReadsPerIterA + self.numReadsPerIterB + lgkmcnt += skipLocalRead * readsPerIter + + vmcnt = 0 if skipGlobalRead > -1 else -1 + if skipGlobalRead > -1: + numA = kernel["NumLoadsPerpendicularA"] * kernel["NumLoadsCoalescedA"] + numB = kernel["NumLoadsPerpendicularB"] * kernel["NumLoadsCoalescedB"] + vmcnt += skipGlobalRead * (numA + numB) + + # Unlike flat loads, BufferLoad do not increment the outstanding + # lgkmcnt + if lgkmcnt > -1 and not kernel["BufferLoad"]: + lgkmcnt += skipGlobalRead * (numA + numB) + + if (self.db["ConservativeWaitCnt"] & 0x2) and skipGlobalRead != -1 or \ + (self.db["ConservativeWaitCnt"] & 0x4) and skipLocalWrite != -1 or \ + (self.db["ConservativeWaitCnt"] & 0x8) and skipLocalRead != -1: + imod = Module("ConservativeWaitCnt") + imod.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="debug %s"%comment)) + imod.add(SBarrier(comment="debug")) + return imod + + maxLgkmcnt = globalParameters["AsmCaps"][self.version]["MaxLgkmcnt"] + lgkmcnt = min(lgkmcnt, maxLgkmcnt) + if lgkmcnt >= 0 and vmcnt >= 0: + vmcnt = -1 # preserve prior behavior of removing vmcnt here? + maxVmcnt = globalParameters["AsmCaps"][self.version]["MaxVmcnt"] + vmcnt = min(vmcnt, maxVmcnt) + # This line is added for backward compatibility + vscnt = vmcnt if lgkmcnt != -1 and vmcnt != -1 and self.archCaps["SeparateVscnt"] else -1 + + waitcnt = SWaitCnt(lgkmcnt,vmcnt, vscnt, comment) + if 0 and lgkmcnt == 0: + imod = Module("DebugWait") + imod.add(waitcnt) + imod.add(self.getBomb()) + return imod + return waitcnt + + ############################################################################## + # SyncThreads + ############################################################################## + def syncThreads(self, kernel, comment=""): + module = Module("syncThreads") + if kernel["NumThreads"] > self.kernel["WavefrontSize"] and self.do["Sync"]: + if self.archCaps["SeparateVscnt"]: + module.add(SWaitCnt(vscnt=0)) + elif kernel.enabledSplitLDS or kernel["ScheduleIterAlg"] == 2 \ + or kernel["PrefetchGlobalRead"] == 2: + module.addComment("Skip force waitcnt0") + elif self.archCaps["Waitcnt0Disabled"]: + # FIXME: should we add s_waitcnt_vscnt? + module.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=-1, comment="force waitcnt0")) + + module.add(SBarrier(comment=comment)) + else: + module.addComment("Skip barrier: NumThreads=%s"%(kernel["NumThreads"]) + \ + comment) + return module + + def AccVgprImagNumOffset(self, kernel): + acc2arch, _ = self.AccToArchMapper(kernel) + return len(acc2arch) * kernel["MIRegPerOut"] + + ############################################################################## + # AccToArchMapper + # Provides forward (acc2arch) and backward (arch2acc) index transformation + # - Forward transformation is currently used for acc->vgpr copying + # - Backward transformation is used in ShiftVectorComponent() to map logical + # C-tile index back to original acc index + ############################################################################## + def AccToArchMapper(self, kernel): + acc2arch = dict() + arch2acc = dict() + + matrixInstM = (kernel["MatrixInstM"] * kernel["MatrixInstBM"]) if (kernel["MatrixInstM"] == 4) else kernel["MatrixInstM"] + matrixInstN = (kernel["MatrixInstN"] * kernel["MatrixInstBN"]) if (kernel["MatrixInstN"] == 4) else kernel["MatrixInstN"] + matrixInstBM = 1 if (kernel["MatrixInstM"] == 4) else kernel["MatrixInstBM"] + matrixInstBN = 1 if (kernel["MatrixInstN"] == 4) else kernel["MatrixInstBN"] + + OutputsPerMFMA1B = matrixInstM * matrixInstN // self.kernel["WavefrontSize"] + VectorWidth0 = kernel["VectorWidth"] if kernel["SourceSwap"] else 1 + outerTT0 = kernel["MIWaveTile"][0] // VectorWidth0 + lrvwB = self.lrvwB if kernel["allowLRVWforTLUandMI"] else 1 + VectorWidth1 = lrvwB + outerTT1 = kernel["MIWaveTile"][1] // VectorWidth1 + + for wgIdx1 in range(0, outerTT1): + for lb in range(0, lrvwB): + for wgIdx0 in range(0, outerTT0): + for bIdx1 in range(0, matrixInstBN): + for bIdx0 in range(0, matrixInstBM): + for tIdx in range(0, OutputsPerMFMA1B): + for vw0 in range(0, VectorWidth0): + src, dst = 0, 0 + if kernel["SourceSwap"]: + src = tIdx + OutputsPerMFMA1B * (bIdx0 + matrixInstBM * (bIdx1 + matrixInstBN * (vw0 + VectorWidth0 * (wgIdx0 + outerTT0 * (wgIdx1 * lrvwB + lb))))) + dst = vw0 + VectorWidth0 * ((bIdx0 + matrixInstBM * (wgIdx0 + outerTT0 * ((tIdx + OutputsPerMFMA1B * (bIdx1 + matrixInstBN * wgIdx1)) * lrvwB + lb)))) + else: + src = tIdx + OutputsPerMFMA1B * (bIdx1 + matrixInstBN * (bIdx0 + matrixInstBM * (wgIdx0 + outerTT0 * wgIdx1))) + dst = tIdx + OutputsPerMFMA1B * (bIdx0 + matrixInstBM * (wgIdx0 + outerTT0 * (bIdx1 + matrixInstBN * wgIdx1))) + acc2arch[src] = dst + arch2acc[dst] = src + + return acc2arch, arch2acc + + ############################################################################## + # MapAcctoArch + # function to map MFMA Acc Registers to Arch VGPR register + ############################################################################## + def MapAcctoArchRegs(self, kernel): + module = Module("MapAcctoArchRegs") + module.addComment1("Mapping of Acc register -> C Vgpr register") + + acc2arch, _ = self.AccToArchMapper(kernel) + + complexMultiplier = 2 if kernel["ProblemType"]["DataType"].isComplex() else 1 + self.codeAccVgprRead = Module("AccVgprRead") + self.codeAccVgprRead.itemList = [None] * kernel["MIRegPerOut"] * complexMultiplier * len(acc2arch) + accImOffset = self.AccVgprImagNumOffset(kernel) + for i in range(len(acc2arch)): + for cm in range(complexMultiplier): + for r in range(kernel["MIRegPerOut"]): + destIdx = (acc2arch[i]*complexMultiplier + cm) * kernel["MIRegPerOut"] + r + srcIdx = ((i * kernel["MIRegPerOut"] + r) + (cm*accImOffset)) + if not kernel["MIArchVgpr"]: + accStr = accvgpr(srcIdx) + self.codeAccVgprRead.itemList[destIdx] = VAccvgprReadB32(dst=vgpr(Holder(name="ValuC")), + src=accStr, + comment="copy acc to vreg[%u]" % destIdx) + else: + self.codeAccVgprRead.itemList[destIdx] = VMovB32(dst=vgpr(Holder(name="ValuC")), + src=vgpr("ValuC+%u"%srcIdx), + comment="copy MI out reg to vreg[%u]" % destIdx) + + return module + + ############################################################################## + # MulMIoutAlphaToArch + # function to handle MFMA alpha*MIout to Arch VGPR register + ############################################################################## + def MulMIoutAlphaToArch(self, kernel): + module = Module("MulMIoutAlphaToArch") + module.addComment1("Multiply MI out register with Alpha -> C Vgpr register") + + acc2arch, _ = self.AccToArchMapper(kernel) + + self.codeMulAlpha = Module("MulAlpha") + self.codeMulAlpha.itemList = [None] * len(acc2arch) + for i in range(len(acc2arch)): + destIdx = acc2arch[i] + srcIdx = i * kernel["MIRegPerOut"] + if kernel["ProblemType"]["ComputeDataType"].isDouble(): + self.codeMulAlpha.itemList[destIdx] = VMulF64(dst=vgpr(Holder(name="ValuC"),2), + src0=sgpr("Alpha",2), src1=vgpr("ValuC+%u"%srcIdx,2), + comment="Multiply MI out reg with alpha") + elif kernel["ProblemType"]["ComputeDataType"].isSingle() or \ + (kernel["ProblemType"]["ComputeDataType"].isHalf() and kernel["ProblemType"]["HighPrecisionAccumulate"]): + self.codeMulAlpha.itemList[destIdx] = VMulF32(dst=vgpr(Holder(name="ValuC")), + src0=sgpr("Alpha"), src1=vgpr("ValuC+%u"%srcIdx), + comment="Multiply MI out reg with alpha") + elif kernel["ProblemType"]["ComputeDataType"].isInt32(): + self.codeMulAlpha.itemList[destIdx] = VMulLOU32(dst=vgpr(Holder(name="ValuC")), + src0=sgpr("Alpha"), src1=vgpr("ValuC+%u"%srcIdx), + comment="Multiply MI out reg with alpha") + elif kernel["ProblemType"]["ComputeDataType"].isDoubleComplex(): + accImOffset = self.AccVgprImagNumOffset(kernel) + imod = Module() + # cannot use tmp vgpr for write batch, use allocated vgpr instead + vtmp1 = self.startVgprAlphaTmp + vtmp2 = vtmp1 + 2 + # tmp1 = a.real * b.real + imod.add(VMulF64(dst=vgpr(vtmp1,2), src0=sgpr("Alpha+0",2), src1=vgpr("ValuC+%u"%srcIdx,2))) + # tmp2 = a.imag * b.real + imod.add(VMulF64(dst=vgpr(vtmp2,2), src0=sgpr("Alpha+2",2), src1=vgpr("ValuC+%u"%srcIdx,2))) + # c.real = a.real * b.real - a.imag * b.imag = tmp1 - a.imag * b.imag + imod.add(VFmaF64(dst=vgpr(Holder(name="ValuC"),2), src0=sgpr("Alpha+2",2), src1=vgpr("ValuC+%u"%(srcIdx+accImOffset),2), src2=vgpr(vtmp1,2))) + # c.imag = a.real * b.imag + a.imag * b.real = a.real * b.imag + tmp2 + imod.add(VFmaF64(dst=vgpr(Holder(name="ValuC+2"),2), src0=sgpr("Alpha+0",2), src1=vgpr("ValuC+%u"%(srcIdx+accImOffset),2), src2=vgpr(vtmp2,2))) + self.codeMulAlpha.itemList[destIdx] = imod + + return module + + ############################################################################## + # Wrappers + ############################################################################## + + ############################################################################## + # Compile Args + ############################################################################## + def getCompileArgs(self, sourceFileName, objectFileName, *moreArgs, isa=None, wavefrontSize=None): + if isa is None: + isa = self.version + if wavefrontSize is None: + wavefrontSize = self.kernel["WavefrontSize"] + return getAsmCompileArgs(globalParameters['AssemblerPath'], \ + globalParameters["CodeObjectVersion"], \ + globalParameters["AsmCaps"][isa]["HasCodeObjectV3"], \ + isa, wavefrontSize, sourceFileName, objectFileName, *moreArgs) + + def getLinkCodeObjectArgs(self, objectFileNames, coFileName, *moreArgs): + return getAsmLinkCodeObjectArgs(globalParameters['AssemblerPath'], \ + objectFileNames, coFileName, *moreArgs) + + ############################################################################## + # longBranchScc0 - 32 bit offset + # Conditional branch to label when SCC == 0 + # Use when erroring out "invalid operand due to label > SIMM16" + ############################################################################## + def longBranchScc0(self, label: Label, posNeg: int=0): + with self.allocTmpSgpr(3) as tmpSgprInfo: + return SCLongBranchScc0(label, tmpSgprInfo, \ + self.labels.getUniqueNamePrefix("NoBranch"), \ + self.labels.getUniqueNamePrefix("Positive"), + posNeg) + + ############################################################################## + # longBranchScc1 - 32 bit offset + # Conditional branch to label when SCC == 1 + # Use when erroring out "invalid operand due to label > SIMM16" + ############################################################################## + def longBranchScc1(self, label: Label, posNeg: int=0): + with self.allocTmpSgpr(3) as tmpSgprInfo: + return SCLongBranchScc1(label, tmpSgprInfo, \ + self.labels.getUniqueNamePrefix("NoBranch"), \ + self.labels.getUniqueNamePrefix("Positive"), + posNeg) + + def sMagicDivWrapper(self, dest, dividend, magicNumber, magicShift): + tmpVgpr = self.vgprPool.checkOut(2) + module = sMagicDiv(dest, globalParameters["AsmCaps"][self.version]["HasSMulHi"], dividend, magicNumber, magicShift, tmpVgpr) + self.vgprPool.checkIn(tmpVgpr) + return module + + def s_mul_u64_u32 (self, dst0, dst1, src0, src1, comment): + vtmp0 = self.vgprPool.checkOut(2) + module = SMulInt64to32(globalParameters["AsmCaps"][self.version]["HasSMulHi"], \ + dst0, dst1, src0, src1, False, vtmp0, comment) + self.vgprPool.checkIn(vtmp0) + return module + + def s_mul_i64_i32 (self, dst0, dst1, src0, src1, comment): + vtmp0 = self.vgprPool.checkOut(2) + module = SMulInt64to32(globalParameters["AsmCaps"][self.version]["HasSMulHi"], \ + dst0, dst1, src0, src1, True, vtmp0, comment) + self.vgprPool.checkIn(vtmp0) + return module + + def getBomb(self, cookie=None) -> Module: + scratchVgpr = self.vgprPool.checkOut(2) + bombCode = bomb(scratchVgpr, cookie) + self.vgprPool.checkIn(scratchVgpr) + return bombCode + + def getCmpAssert(self, function, val0, val1, cookie=-1): + scratchVgpr = self.vgprPool.checkOut(2) + function(val0, val1, scratchVgpr, cookie) + self.vgprPool.checkIn(scratchVgpr) + + def getMultipleB32Assert(self, sval, multiple2, cookie=-1): + scratchVgpr = self.vgprPool.checkOut(2) + self.asmAssert.multiple_b32(sval, multiple2, scratchVgpr, cookie) + self.vgprPool.checkIn(scratchVgpr) + + def getVectorDiffAssert(self, v0, v1, expectedScalarDiff, cookie=-1): + cmpvtmp = self.vgprPool.checkOut(1) + vtmp = self.vgprPool.checkOut(2) + self.asmAssert.assert_vector_diff(v0, v1, expectedScalarDiff, cmpvtmp, vtmp, cookie) + self.vgprPool.checkIn(vtmp) + self.vgprPool.checkIn(cmpvtmp) + + def dump(self, vgprStore): + return self.dumpData.dumpVgpr(vgprStore, self.labels.getUniqueName()) + + def dumpSgpr(self, sgprStore): + tmp = RegisterPoolResource(idx=self.vgprPool.checkOut(1,"tmp"), size=1) + module = self.dumpData.dumpSgpr(sgprStore, tmp, self.labels.getUniqueName()) + self.vgprPool.checkIn(tmp.idx) + return module + + def dumpLDS(self, kernel, startU, numU): + tmp = RegisterPoolResource(idx=self.vgprPool.checkOut(2), size=2) + module = self.dumpData.dumpLds(startU, numU, tmp, self.bpeAB, kernel["NumThreads"], \ + self.labels.getUniqueName()) + self.vgprPool.checkIn(tmp.idx) + return module + +def _growPool(pool: RegisterPool, rangeStart: int, rangeEnd: int, checkOutSize: int, comment: str=""): + tl = [] + for _ in range(rangeStart, rangeEnd): + tl.append(pool.checkOut(checkOutSize, comment)) + for t in tl: + pool.checkIn(t) diff --git a/tensilelite/Tensile/KernelWriterBase.py b/tensilelite/Tensile/KernelWriterBase.py new file mode 100644 index 0000000000..3e2bc5c05a --- /dev/null +++ b/tensilelite/Tensile/KernelWriterBase.py @@ -0,0 +1,119 @@ +################################################################################ +# Copyright 2020-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from abc import ABC +from abc import abstractmethod +from copy import deepcopy + +class KernelWriterBase(ABC): + + def __init__(self): + super().__init__() + + self.state = {} + + self.endLine = "\n" + self.getGroupIdStr = "hc_get_group_id" + self.getNumGroupsStr = "hc_get_num_groups" + self.getLocalIdStr = "hc_get_workitem_id" + self.getGlobalIdStr = "hc_get_workitem_absolute_id" + self.sharedDeclStr = "__shared__ " + self.sharedPtrStr = "" + self.globalPtrStr = "" + self.macFStr = "fmaf" + self.macDStr = "fma" + self.int64Str = "int64_t" + self.uint64Str = "uint64_t" + self.vectorComponents = ["x", "y", "z", "w"] + self.atomicCasStr = "atomicCAS" + self.volatileStr = "" + self.deviceFunctionStr = "__device__ " + + + def extractIndices(self, extractFrom, varPrefix, indices): + kStr = "" + for (i, index) in enumerate(indices): + kStr += " unsigned int " + varPrefix + self.indexChars[index] + " = ( " + extractFrom + for j in reversed(list(range(i+1, len(indices)))): + index2 = indices[j] + kStr += " / size" + self.indexChars[index2] + kStr += ")" + + if len(indices) > 1: + kStr += " % size" + self.indexChars[index] + kStr += ";" + self.endLine + + return kStr + + + @abstractmethod + def getKernelName(self): + pass + + + @abstractmethod + def getHeaderFileString(self): + pass + + + @abstractmethod + def getSourceFileString(self): + pass + + + ########################## + # make class look like dict + def keys(self): + return list(self.state.keys()) + + def __len__(self): + return len(self.state) + + def __iter__(self): + return iter(self.state) + + def __getitem__(self, key): + return self.state[key] + + def __setitem__(self, key, value): + self.state[key] = value + + def __str__(self): + return self.getKernelName() + + def __repr__(self): + return self.__str__() + + def getAttributes(self): + return deepcopy(self.state) + + def __hash__(self): + return hash(str(self)) + + def __eq__(self, other): + return isinstance(other, KernelWriterBase) and str(self) == str(other) + + def __ne__(self, other): + result = self.__eq__(other) + if result is NotImplemented: + return result + return not result + diff --git a/tensilelite/Tensile/KernelWriterBetaOnly.py b/tensilelite/Tensile/KernelWriterBetaOnly.py new file mode 100644 index 0000000000..c5189b82eb --- /dev/null +++ b/tensilelite/Tensile/KernelWriterBetaOnly.py @@ -0,0 +1,307 @@ +################################################################################ +# Copyright 2020-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from copy import deepcopy + +from .Common import globalParameters, CHeader +from .TensileInstructions import DataType +from .KernelWriterBase import KernelWriterBase + +class KernelWriterBetaOnly(KernelWriterBase): + + def __init__(self, state): + super().__init__() + + self.state["ProblemType"] = deepcopy(state["ProblemType"]) + self.state["_GlobalAccumulation"] = state["_GlobalAccumulation"] + + # derive parameter + self.language = "HIP" + self.kernelName = self.getKernelName() + self.datatype = self.state["ProblemType"]["ComputeDataType"].toDevice(self.language) + + # determine chars for fast access + self.indexChars = [] + for i in range(0, len(globalParameters["IndexChars"])): + self.indexChars.append(globalParameters["IndexChars"][i]) + self.indexChars[self.state["ProblemType"]["Index0"]] = "0" + self.indexChars[self.state["ProblemType"]["Index0"]] + self.indexChars[self.state["ProblemType"]["Index1"]] = "1" + self.indexChars[self.state["ProblemType"]["Index1"]] + self.tileChar0 = self.indexChars[self.state["ProblemType"]["Index0"]] + self.tileChar1 = self.indexChars[self.state["ProblemType"]["Index1"]] + + + def functionSignature(self): + kStr = "" + + # self.state name + kStr += self.endLine + kStr += "extern \"C\"" + self.endLine + kStr += "__global__ " + kStr += "void %s" % ( self.kernelName ) + kStr += "(" + self.endLine + + # pointers + ptrStr = self.state["ProblemType"]["DestDataType"].toDevice(self.language) + if self.state["_GlobalAccumulation"]: + ptrStr = self.state["ProblemType"]["ComputeDataType"].toDevice(self.language) + + isStridedBuffer = self.state["ProblemType"]["StridedBatched"] or self.state["_GlobalAccumulation"] + ptrStr += "" if isStridedBuffer else "*" + batch = "" if isStridedBuffer else "Batch" + kStr += " " + ptrStr + " * " + batch + "D," + self.endLine + + ptrStr = self.state["ProblemType"]["DestDataType"].toDevice(self.language) + isStridedBuffer = self.state["ProblemType"]["StridedBatched"] + ptrStr += "" if isStridedBuffer else "*" + batch = "" if isStridedBuffer else "Batch" + kStr += " " + ptrStr + " const * " + batch + "C," + self.endLine + + # bias + if self.state["ProblemType"]["BetaOnlyUseBias"]: + biasPtrStr = self.state["ProblemType"]["DataType"].toDevice(self.language) + kStr += " " + biasPtrStr + " const * " + "Bias," + self.endLine + + + # strides + firstStrideCD = 1 + if self.state["ProblemType"]["UseInitialStridesCD"]: + firstStrideCD = 0 + lastStrideC = self.state["ProblemType"]["NumIndicesC"] + for i in range(firstStrideCD, lastStrideC): + kStr += " unsigned int const strideD%s,%s" % (self.indexChars[i], self.endLine) + for i in range(firstStrideCD, lastStrideC): + kStr += " unsigned int const strideC%s,%s" % (self.indexChars[i], self.endLine) + + # sizes + for i in range(0, self.state["ProblemType"]["NumIndicesC"]): + kStr += " unsigned int const size%s,%s" % (self.indexChars[i], self.endLine) + + # offset + kStr += " unsigned int offsetD,%s" % self.endLine + kStr += " unsigned int offsetC,%s" % self.endLine + + # beta + kStr += " %s const beta)%s" % (self.state["ProblemType"]["ComputeDataType"].toDevice(self.language), self.endLine ) + + return kStr + + + ############################################################################## + # Kernel Body Beta-Only + ############################################################################## + def kernelBodyBetaOnly(self): + problemType = self.state["ProblemType"] + globalAccum = self.state["_GlobalAccumulation"] + + kStr = "" + kStr += "{%s" % self.endLine + + ######################################## + # defined initial strides + firstStride = 0 + if problemType["UseInitialStridesCD"]: + # no strides #defined + lastStrideC = 0 + assert 0 # need to fix beta-clear routine to pass initial stride parms + else: + # #define initial stride + kStr += "/* hard-coded initial strides */%s" % self.endLine + lastStrideC = 1 + for i in range(firstStride, lastStrideC): + kStr += "#define strideD" + self.indexChars[i] + " 1" + self.endLine + for i in range(firstStride, lastStrideC): + kStr += "#define strideC" + self.indexChars[i] + " 1" + self.endLine + + ######################################## + # GLOBAL_D() + kStr += "#define GLOBAL_D(IDX%s" % self.indexChars[0] + for i in range(1, problemType["NumIndicesC"]): + kStr += ", IDX%s" % self.indexChars[i] + indexChar = self.indexChars[0] + kStr += ") (( (IDX%s)*strideD%s" % (indexChar, indexChar) + for i in range(1, problemType["NumIndicesC"]): + indexChar = self.indexChars[i] + kStr += " + (IDX%s)*strideD%s" % (indexChar, indexChar) + kStr += " ))" + self.endLine + + # GLOBAL_C() + kStr += "#define GLOBAL_C(IDX%s" % self.indexChars[0] + for i in range(1, problemType["NumIndicesC"]): + kStr += ", IDX%s" % self.indexChars[i] + indexChar = self.indexChars[0] + kStr += ") (( (IDX%s)*strideC%s" % (indexChar, indexChar) + for i in range(1, problemType["NumIndicesC"]): + indexChar = self.indexChars[i] + kStr += " + (IDX%s)*strideC%s" % (indexChar, indexChar) + kStr += " ))" + self.endLine + + ######################################## + # multi buffers GSU: Accumulate all GSU buffer + indexChar = self.indexChars[0] + kStr += " uint64_t id = %s(0);%s" % (self.getGlobalIdStr, self.endLine) + kStr += " if (id >= (size%s" % self.indexChars[0] + for i in range(1, problemType["NumIndicesC"]): + kStr += "*size%s" % self.indexChars[i] + kStr += "))%s" % self.endLine + kStr += " return;%s" % self.endLine + + kStr += self.endLine + kStr += " uint64_t id0" + for i in range(1, problemType["NumIndicesC"]): + kStr += ", id%d" % i + kStr += ";%s" % self.endLine + + for i in range(0, problemType["NumIndicesC"]): + kStr += " id%d = id %% size%s;%s" % (i, self.indexChars[i], self.endLine) + kStr += " id = id / size%s;%s" % (self.indexChars[i], self.endLine) + + nonTileFreeIndices = [] + + # apply batch + if not self.state["ProblemType"]["StridedBatched"]: + nonTileFreeIndices = list(range(0, self.state["ProblemType"]["NumIndicesC"])) + nonTileFreeIndices.remove(self.state["ProblemType"]["Index0"]) + nonTileFreeIndices.remove(self.state["ProblemType"]["Index1"]) + + kStr += self.endLine + kStr += " uint64_t wg = 0" + batchStride = "1" + for i in nonTileFreeIndices: + kStr += " + id%d * %s " % (i, batchStride) + batchStride += " * size%s" % self.indexChars[i] + kStr += ";" + self.endLine + + if not self.state["_GlobalAccumulation"]: + ptrStr = self.state["ProblemType"]["DestDataType"].toDevice(self.language) + kStr += " " + ptrStr + " * D = BatchD[wg];" + self.endLine + ptrStr = self.state["ProblemType"]["DestDataType"].toDevice(self.language) + zeroStr = self.state["ProblemType"]["ComputeDataType"].zeroString(self.language, 1) + kStr += " " + ptrStr + f" const* C = (beta == {zeroStr}) ? nullptr : BatchC[wg];" + self.endLine + + # apply offset + kStr += self.endLine + if not self.state["_GlobalAccumulation"]: + kStr += " D = D + offsetD;" + self.endLine + kStr += " C = C + offsetC;" + self.endLine + + + kStr += self.endLine + ######################################## + # D index + kStr += " %s idxD = GLOBAL_D( (%s)" % (self.uint64Str, self.uint64Str) + for i in range(problemType["NumIndicesC"]): + tmpStr = '' + if self.state["_GlobalAccumulation"]: + tmpStr = 'id%d' % i + elif i in nonTileFreeIndices: + tmpStr = '0' + else: + tmpStr = 'id%d' % i + kStr += ', ' if i else '' + kStr += tmpStr + kStr += ");%s" % (self.endLine) + + # C index + kStr += " %s idxC = GLOBAL_C( (%s)" % (self.uint64Str, self.uint64Str) + for i in range(problemType["NumIndicesC"]): + kStr += ', ' if i else '' + kStr += '0' if i in nonTileFreeIndices else ('id%d' % i) + kStr += ");%s" % (self.endLine) + + ######################################## + # zero + if globalAccum: + ptrStr = problemType["ComputeDataType"].toDevice(self.language) + if problemType["DataType"].isHalf() and problemType["HighPrecisionAccumulate"]: + ptrStr = DataType('single').toDevice(self.language) + else: + ptrStr = problemType["DataType"].toDevice(self.language) + kStr += "#define SCALAR_ZERO ((%s)(0))%s" % (ptrStr, self.endLine ) + + biasStr = "" + if self.state["ProblemType"]["BetaOnlyUseBias"]: + biasStr = " + ((" + self.datatype + ")(Bias == 0 ? 0 : Bias[id0]))" + + ######################################## + # zero + kStr += " if( beta == (%s)0) {%s" % (self.datatype, self.endLine) + kStr += " D[idxD] = SCALAR_ZERO%s;%s" % (biasStr, self.endLine) + kStr += " } else {%s" % self.endLine + kStr += " D[idxD] = ((%s)(C[idxC])) * beta%s;%s" % (self.datatype, biasStr, self.endLine) + kStr += " }%s" % self.endLine + + ######################################## + # end + kStr += "}%s" % self.endLine + for i in range(firstStride, lastStrideC): + kStr += "#undef strideD" + self.indexChars[i] + self.endLine + for i in range(firstStride, lastStrideC): + kStr += "#undef strideC" + self.indexChars[i] + self.endLine + kStr += "#undef GLOBAL_D%s" % (self.endLine) + kStr += "#undef GLOBAL_C%s" % (self.endLine) + kStr += "#undef SCALAR_ZERO%s" % ( self.endLine) + + return kStr + + + def getKernelName(self): + indexChars = globalParameters["IndexChars"] + # C dimensions + name = "C" + for i in range(0, self.state["ProblemType"]["NumIndicesC"]): + name += indexChars[i].lower() + name += "_" + name += self.state["ProblemType"]["DestDataType"].toChar() + name += "" if self.state["ProblemType"]["StridedBatched"] else "_GB" # legacy + name += "_Bias" if self.state["ProblemType"]["BetaOnlyUseBias"] else "" + name += "_GA" if self.state["_GlobalAccumulation"] else "" + + return name + + + def getSourceFileString(self): + fileString = "" + + if not globalParameters["MergeFiles"]: + fileString += "\n" + fileString += "#include \"%s.h\"\n" % self.kernelName + fileString += "\n" + + fileString += self.functionSignature() + fileString += self.kernelBodyBetaOnly() + + return (0, fileString) + + def getHeaderFileString(self): + fileString = "" # CHeader + if not globalParameters["MergeFiles"]: + fileString += CHeader + fileString += "#pragma once\n\n" + fileString += "\n" + fileString += "#include \n\n" + fileString += "#include \n" + fileString += "#include \n" + fileString += "\n" + + fileString += self.functionSignature() + fileString += ";\n" + + return fileString diff --git a/tensilelite/Tensile/KernelWriterConversion.py b/tensilelite/Tensile/KernelWriterConversion.py new file mode 100644 index 0000000000..7028b9e7a3 --- /dev/null +++ b/tensilelite/Tensile/KernelWriterConversion.py @@ -0,0 +1,354 @@ +################################################################################ +# Copyright 2020-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from copy import deepcopy + +from .Common import globalParameters, CHeader +from .KernelWriterBase import KernelWriterBase + +class KernelWriterConversion(KernelWriterBase): + + def __init__(self, state): + super().__init__() + + self.state["ProblemType"] = deepcopy(state["ProblemType"]) + self.state["_GlobalAccumulation"] = state["_GlobalAccumulation"] + self.state["ActivationFused"] = state["ActivationFused"] + + # derive parameter + self.language = "HIP" + self.kernelName = self.getKernelName() + self.datatype = self.state["ProblemType"]["ComputeDataType"].toDevice(self.language) + + # determine chars for fast access + self.indexChars = [] + for i in range(0, len(globalParameters["IndexChars"])): + self.indexChars.append(globalParameters["IndexChars"][i]) + self.indexChars[self.state["ProblemType"]["Index0"]] = "0" + self.indexChars[self.state["ProblemType"]["Index0"]] + self.indexChars[self.state["ProblemType"]["Index1"]] = "1" + self.indexChars[self.state["ProblemType"]["Index1"]] + self.tileChar0 = self.indexChars[self.state["ProblemType"]["Index0"]] + self.tileChar1 = self.indexChars[self.state["ProblemType"]["Index1"]] + + + def functionSignature(self): + kStr = "" + + # kernel name + kStr += self.endLine + kStr += "extern \"C\"\n" + kStr += "__global__ " + kStr += "void %s" % ( self.kernelName ) + kStr += "(" + self.endLine + + # pointers + ptrStr = self.state["ProblemType"]["DestDataType"].toDevice(self.language) + ptrStr += '' if self.state["ProblemType"]["StridedBatched"] else '*' + bStr = '' if self.state["ProblemType"]["StridedBatched"] else 'Batch' + + kStr += " " + ptrStr + " * " + bStr + "D," + self.endLine + kStr += " " + self.datatype + " * W," + self.endLine + kStr += " " + ptrStr + " const * " + bStr + "C," + self.endLine + + # bias + if self.state["ProblemType"]["UseBias"]: + biasPtrStr = self.state["ProblemType"]["DataType"].toDevice(self.language) + kStr += " " + biasPtrStr + " const * " + "Bias," + self.endLine + + # alpha & beta + kStr += " %s const alpha,%s" % (self.state["ProblemType"]["ComputeDataType"].toDevice(self.language), self.endLine) + kStr += " %s const beta,%s" % (self.state["ProblemType"]["ComputeDataType"].toDevice(self.language), self.endLine) + + # activation + activationCDataType = self.state["ProblemType"]["ActivationComputeDataType"] + enumName = "Tensile::ActivationType_%s"%activationCDataType.toChar() + if ((self.state["ProblemType"]["ActivationType"] != 'none') and self.state["ActivationFused"]): + activationCDataType = self.state["ProblemType"]["ComputeDataType"] if self.state["ProblemType"]["ActivationHPA"] else \ + self.state["ProblemType"]["DestDataType"] + for name in self.state["ProblemType"]["ActivationType"].getAdditionalArgStringList(): + kStr += " %s const %s,%s" % (activationCDataType.toDevice(self.language), name, self.endLine) + if self.state["ProblemType"]["ActivationType"] == 'all': + kStr += " %s const activationType,%s" % (enumName, self.endLine) + + # strides + firstStrideCD = 1 + if self.state["ProblemType"]["UseInitialStridesCD"]: + firstStrideCD = 0 + lastStrideC = self.state["ProblemType"]["NumIndicesC"] + for i in range(firstStrideCD, lastStrideC): + kStr += " unsigned int const strideD%s,%s" % (self.indexChars[i], self.endLine) + for i in range(firstStrideCD, lastStrideC): + kStr += " unsigned int const strideW%s,%s" % (self.indexChars[i], self.endLine) + for i in range(firstStrideCD, lastStrideC): + kStr += " unsigned int const strideC%s,%s" % (self.indexChars[i], self.endLine) + + # sizes + for i in range(0, self.state["ProblemType"]["NumIndicesC"]): + kStr += " unsigned int const size%s,%s" % (self.indexChars[i], self.endLine) + + # offset + kStr += " unsigned int offsetD,%s" % self.endLine + kStr += " unsigned int offsetC,%s" % self.endLine + + # gsu + kStr += " unsigned int const gsu)%s" % self.endLine + + return kStr + + + def kernelBody(self): + kStr = "" + kStr += "{%s" % self.endLine + problemType = self.state["ProblemType"] + + ######################################## + # defined initial strides + firstStride = 0 + if problemType["UseInitialStridesCD"]: + # no strides #defined + lastStrideC = 0 + assert 0 # need to fix beta-clear routine to pass initial stride parms + else: + # #define initial stride + kStr += "/* hard-coded initial strides */%s" % self.endLine + lastStrideC = 1 + for i in range(firstStride, lastStrideC): + kStr += "#define strideD" + self.indexChars[i] + " 1" + self.endLine + for i in range(firstStride, lastStrideC): + kStr += "#define strideW" + self.indexChars[i] + " 1" + self.endLine + for i in range(firstStride, lastStrideC): + kStr += "#define strideC" + self.indexChars[i] + " 1" + self.endLine + + ######################################## + # GLOBAL_D() + kStr += "#define GLOBAL_D(IDX%s" % self.indexChars[0] + for i in range(1, problemType["NumIndicesC"]): + kStr += ", IDX%s" % self.indexChars[i] + indexChar = self.indexChars[0] + kStr += ") (( (IDX%s)*strideD%s" % (indexChar, indexChar) + for i in range(1, problemType["NumIndicesC"]): + indexChar = self.indexChars[i] + kStr += " + (IDX%s)*strideD%s" % (indexChar, indexChar) + kStr += " ))" + self.endLine + + # GLOBAL_W() + kStr += "#define GLOBAL_W(IDX%s" % self.indexChars[0] + for i in range(1, problemType["NumIndicesC"]): + kStr += ", IDX%s" % self.indexChars[i] + indexChar = self.indexChars[0] + kStr += ") (( (IDX%s)*strideW%s" % (indexChar, indexChar) + for i in range(1, problemType["NumIndicesC"]): + indexChar = self.indexChars[i] + kStr += " + (IDX%s)*strideW%s" % (indexChar, indexChar) + kStr += " ))" + self.endLine + + # GLOBAL_C() + kStr += "#define GLOBAL_C(IDX%s" % self.indexChars[0] + for i in range(1, problemType["NumIndicesC"]): + kStr += ", IDX%s" % self.indexChars[i] + indexChar = self.indexChars[0] + kStr += ") (( (IDX%s)*strideC%s" % (indexChar, indexChar) + for i in range(1, problemType["NumIndicesC"]): + indexChar = self.indexChars[i] + kStr += " + (IDX%s)*strideC%s" % (indexChar, indexChar) + kStr += " ))" + self.endLine + + ######################################## + # multi buffers GSU: Accumulate all GSU buffer + indexChar = self.indexChars[0] + kStr += " uint64_t id = %s(0);%s" % (self.getGlobalIdStr, self.endLine) + kStr += " if (id >= (size%s" % self.indexChars[0] + for i in range(1, problemType["NumIndicesC"]): + kStr += "*size%s" % self.indexChars[i] + kStr += "))%s" % self.endLine + kStr += " return;%s" % self.endLine + + kStr += self.endLine + kStr += " uint64_t id0" + for i in range(1, problemType["NumIndicesC"]): + kStr += ", id%d" % i + kStr += ";%s" % self.endLine + + for i in range(0, problemType["NumIndicesC"]): + kStr += " id%d = id %% size%s;%s" % (i, self.indexChars[i], self.endLine) + kStr += " id = id / size%s;%s" % (self.indexChars[i], self.endLine) + + nonTileFreeIndices = [] + + ######################################## + # apply batch + if not self.state["ProblemType"]["StridedBatched"]: + nonTileFreeIndices = list(range(0, self.state["ProblemType"]["NumIndicesC"])) + nonTileFreeIndices.remove(self.state["ProblemType"]["Index0"]) + nonTileFreeIndices.remove(self.state["ProblemType"]["Index1"]) + + kStr += self.endLine + kStr += " uint64_t wg = 0" + batchStride = "1" + for i in nonTileFreeIndices: + kStr += " + id%d * %s " % (i, batchStride) + batchStride += " * size%s" % self.indexChars[i] + kStr += ";" + self.endLine + + ptrStr = self.state["ProblemType"]["DestDataType"].toDevice(self.language) + kStr += " " + ptrStr + " * D = BatchD[wg];" + self.endLine + ptrStr = self.state["ProblemType"]["DestDataType"].toDevice(self.language) + zeroStr = self.state["ProblemType"]["ComputeDataType"].zeroString(self.language, 1) + kStr += " " + ptrStr + f" const* C = (beta == {zeroStr}) ? nullptr : BatchC[wg];" + self.endLine + + ######################################## + # apply offset + kStr += self.endLine + kStr += " D = D + offsetD;" + self.endLine + kStr += " C = C + offsetC;" + self.endLine + + ######################################## + # D index + kStr += self.endLine + kStr += " %s idxD = GLOBAL_D( (%s)" % (self.uint64Str, self.uint64Str) + for i in range(problemType["NumIndicesC"]): + kStr += ', ' if i else '' + kStr += '0' if i in nonTileFreeIndices else ('id%d' % i) + kStr += ");%s" % (self.endLine) + + # W index + kStr += " %s idxW = GLOBAL_W( (%s)" % (self.uint64Str, self.uint64Str) + for i in range(problemType["NumIndicesC"]): + kStr += ', ' if i else '' + kStr += 'id%d' % i + kStr += ");%s" % (self.endLine) + + # D index + kStr += " %s idxC = GLOBAL_C( (%s)" % (self.uint64Str, self.uint64Str) + for i in range(problemType["NumIndicesC"]): + kStr += ', ' if i else '' + kStr += '0' if i in nonTileFreeIndices else ('id%d' % i) + kStr += ");%s" % (self.endLine) + + ######################################## + # multi buffers GSU: Accumulate all GSU buffer + indexChar = self.indexChars[0] + kStr += " %s strideW = 1 + (size%s - 1) * strideW%s" % (self.uint64Str, indexChar, indexChar) + for i in range(1, problemType["NumIndicesC"]): + indexChar = self.indexChars[i] + kStr += " + (size%s - 1) * strideW%s" % (indexChar, indexChar) + kStr += ";" + self.endLine + + kStr += " " + self.datatype + " accum = 0;%s" % self.endLine + kStr += " for (int i=0; i 2: + if "ActivationArgs" in data[2]: + solutionStartIdxInData += 1 + + solutions = [] + for i in range(solutionStartIdxInData, len(data)): + solutionState = data[i] + # force redo the deriving of parameters, make sure old version logic yamls can be validated + solutionState["AssignedProblemIndependentDerivedParameters"] = False + solutionState["AssignedDerivedParameters"] = False + solutionObject = Solution(solutionState) + solutions.append(solutionObject) + problemType = solutions[0]["ProblemType"] + problemSizes = ProblemSizes(problemType, problemSizesConfig) + return (problemSizes, solutions) + +def parseLibraryLogicFile(filename): + """Wrapper function to read and parse a library logic file.""" + return parseLibraryLogicData(readYAML(filename), filename) + +def parseLibraryLogicData(data, srcFile="?"): + """Parses the data of a library logic file.""" + if len(data) < 9: + printExit("Library logic file {} is missing required fields (len = {} < 9)" \ + .format(srcFile, len(data))) + + versionString = data[0]["MinimumRequiredVersion"] + scheduleName = data[1] + architectureName = data[2] if isinstance(data[2], str) else data[2]["Architecture"] + deviceNames = data[3] + problemTypeState = data[4] + solutionStates = data[5] + indexOrder = data[6] + exactLogic = data[7] + rangeLogic = data[8] + + if not versionIsCompatible(versionString): + printWarning("Version = {} in library logic file {} does not match Tensile version = {}" \ + .format(srcFile, versionString, __version__) ) + + # unpack problemType + problemType = ProblemType(problemTypeState) + # unpack solutions + solutions = [] + for i in range(0, len(solutionStates)): + solutionState = solutionStates[i] + if solutionState["KernelLanguage"] == "Assembly": + solutionState["ISA"] = Common.gfxArch(architectureName) + else: + solutionState["ISA"] = (0, 0, 0) + # force redo the deriving of parameters, make sure old version logic yamls can be validated + solutionState["AssignedProblemIndependentDerivedParameters"] = False + solutionState["AssignedDerivedParameters"] = False + solutionObject = Solution(solutionState) + + if solutionObject["ProblemType"] != problemType: + printExit("ProblemType in library logic file {} doesn't match solution: {} != {}" \ + .format(srcFile, problemType, solutionObject["ProblemType"])) + solutions.append(solutionObject) + + newLibrary = SolutionLibrary.MasterSolutionLibrary.FromOriginalState(data, solutions) + + return (scheduleName, deviceNames, problemType, solutions, indexOrder, \ + exactLogic, rangeLogic, newLibrary, architectureName) + +def rawLibraryLogic(data): + """Returns a tuple of the data in a library logic file.""" + versionString = data[0] + scheduleName = data[1] + architectureName = data[2] + deviceNames = data[3] + problemTypeState = data[4] + solutionStates = data[5] + indexOrder = data[6] + exactLogic = data[7] + rangeLogic = data[8] + otherFields = [] + + dataLength = len(data) + if dataLength > 9: + for idx in range(9, dataLength): + otherFields.append(data[idx]) + + return (versionString, scheduleName, architectureName, deviceNames,\ + problemTypeState, solutionStates, indexOrder, exactLogic, rangeLogic, otherFields) + + +################# +# Other functions +################# +def createLibraryLogic(schedulePrefix, architectureName, deviceNames, logicTuple): + """Creates the data for a library logic file suitable for writing to YAML.""" + problemType = logicTuple[0] + solutions = logicTuple[1] + indexOrder = logicTuple[2] + exactLogic = logicTuple[3] + rangeLogic = logicTuple[4] + + tileSelection = False + if len(logicTuple) > 5 and logicTuple[5]: + tileSelection = True + + data = [] + # Tensile version + data.append({"MinimumRequiredVersion":__version__}) + # schedule name + data.append(schedulePrefix) # change from Tensile to vega10 + data.append(architectureName) + # schedule device names + data.append(deviceNames) + # problem type + problemTypeState = problemType.state + problemTypeState["DataType"] = \ + problemTypeState["DataType"].value + problemTypeState["DestDataType"] = \ + problemTypeState["DestDataType"].value + problemTypeState["ComputeDataType"] = \ + problemTypeState["ComputeDataType"].value + problemTypeState["ActivationComputeDataType"] = \ + problemTypeState["ActivationComputeDataType"].value + problemTypeState["ActivationType"] = \ + problemTypeState["ActivationType"].value + + data.append(problemTypeState) + # solutions + solutionList = [] + for solution in solutions: + solutionState = solution.getAttributes() + solutionState["ProblemType"] = solutionState["ProblemType"].state + solutionState["ProblemType"]["DataType"] = \ + solutionState["ProblemType"]["DataType"].value + solutionState["ProblemType"]["DestDataType"] = \ + solutionState["ProblemType"]["DestDataType"].value + solutionState["ProblemType"]["ComputeDataType"] = \ + solutionState["ProblemType"]["ComputeDataType"].value + solutionState["ProblemType"]["ActivationComputeDataType"] = \ + solutionState["ProblemType"]["ActivationComputeDataType"].value + solutionState["ProblemType"]["ActivationType"] = \ + solutionState["ProblemType"]["ActivationType"].value + solutionList.append(solutionState) + + if tileSelection: + tileSolutions = logicTuple[5] + for solution in tileSolutions: + solutionState = solution.getAttributes() + solutionState["ProblemType"] = solutionState["ProblemType"].state + solutionState["ProblemType"]["DataType"] = \ + solutionState["ProblemType"]["DataType"].value + solutionState["ProblemType"]["DestDataType"] = \ + solutionState["ProblemType"]["DestDataType"].value + solutionState["ProblemType"]["ComputeDataType"] = \ + solutionState["ProblemType"]["ComputeDataType"].value + solutionState["ProblemType"]["ActivationComputeDataType"] = \ + solutionState["ProblemType"]["ActivationComputeDataType"].value + solutionState["ProblemType"]["ActivationType"] = \ + solutionState["ProblemType"]["ActivationType"].value + solutionList.append(solutionState) + + data.append(solutionList) + # index order + data.append(indexOrder) + + # exactLogic + exactLogicList = [] + for key in exactLogic: + exactLogicList.append([list(key), exactLogic[key]]) + data.append(exactLogicList) + + # rangeLogic + data.append(rangeLogic) + + if tileSelection: + tileSelectionLogic = {} + tileSelectionIndices = logicTuple[6] + tileSelectionLogic["TileSelectionIndices"] = tileSelectionIndices + data.append(tileSelectionLogic) + else: + data.append(None) + + data.append(logicTuple[7]) + return data diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py new file mode 100644 index 0000000000..601d4e1390 --- /dev/null +++ b/tensilelite/Tensile/LibraryLogic.py @@ -0,0 +1,1482 @@ +################################################################################ +# Copyright 2016-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ + setWorkingPath, popWorkingPath, assignParameterWithDefault, startTime, ProgressBar, printWarning +from .SolutionStructs import Solution +from . import LibraryIO +from . import SolutionSelectionLibrary + +from copy import deepcopy +from sys import stdout + +import array +import csv +import os +import time + +################################################################################ +# Analyze Problem Type +################################################################################ +def analyzeProblemType( problemType, problemSizeGroups, inputParameters ): + print2(HR) + print1("# Analyzing: %s" % problemType) + + enableTileSelection = problemType["TileAwareSelection"] + solutionsList = [] + problemSizesList = [] + dataFileNameList = [] + selectionFileNameList = [] + + for problemSizeGroup in problemSizeGroups: + problemSizes = problemSizeGroup[0] + dataFileName = problemSizeGroup[1] + dataFileNameList.append(dataFileName) + solutionsFileName = problemSizeGroup[2] + #print " problemSizes:", problemSizes + #print "# DataFileName:", dataFileName + #print " solutionsFileName:", solutionsFileName + if enableTileSelection: + selectionFileName = problemSizeGroup[3] + selectionFileNameList.append(selectionFileName) + + ###################################### + # Read Solutions + # (problemSizes, solutions) are already read and kept in problemSizeGroups, no need to call LibraryIO.readSolutions(solutionsFileName) again + solutions = problemSizeGroup[4] + problemSizesList.append(problemSizes) + solutionsList.append(solutions) + solutionMinNaming = Solution.getMinNaming(solutions) + print1("# Read: %s" % (solutionsFileName)) + print2("# ProblemSizes: %s" % problemSizes) + print2("# Solutions:") + solutionIdx = 0 + for solution in solutions: + print2("# (%u) %s" % (solutionIdx, Solution.getNameMin(solution, \ + solutionMinNaming))) + solutionIdx += 1 + print2(HR) + + ###################################### + # Create Logic Analyzer + logicAnalyzer = LogicAnalyzer( problemType, problemSizesList, solutionsList, \ + dataFileNameList, inputParameters) + + selectionSolutionsIdsList = None + selectionSolutions = None + + validSelectionSolutions = [] + + ###################################### + # Remove invalid solutions + logicAnalyzer.removeInvalidSolutions() + + ###################################### + # Remove least important solutions + if globalParameters["SolutionSelectionAlg"] == 0: + logicAnalyzer.removeLeastImportantSolutions() + elif globalParameters["SolutionSelectionAlg"] == 1: + logicAnalyzer.keepWinnerSolutions() + else: + printExit("Bad KeepLogic=%u"%globalParameters["KeepLogic"]) + + # print raw data + if globalParameters["PrintLevel"] >= 2: + line = "After Removals:\n" + numOther = 1 + for size in logicAnalyzer.numProblemSizes: + numOther *= size + numCols = logicAnalyzer.numProblemSizes[1] + if numCols == 0: numCols = 1 + numOther //= numCols + for row in range(0, numOther): + for col in range(0, numCols): + for sol in range(0, logicAnalyzer.numSolutions): + line += "% 5.0f" % logicAnalyzer.data[sol + logicAnalyzer.numSolutions*(col + row*numCols)] + line += "; " + line += "\n" + print(line) + + for i in range(0, len(logicAnalyzer.solutions)): + s = logicAnalyzer.solutions[i] + s["SolutionIndex"] = i + s["SolutionNameMin"] = Solution.getNameMin(s, solutionMinNaming) + print1("(%2u) %s : %s" % (i, Solution.getNameMin(s, solutionMinNaming), Solution.getNameFull(s))) + + if enableTileSelection: + validSelectionSolutions = SolutionSelectionLibrary.analyzeSolutionSelection(problemType, selectionFileNameList, \ + logicAnalyzer.numSolutionsPerGroup, logicAnalyzer.solutionGroupMap, solutionsList) + + validSelectionSolutionsIncluded = [] + validSelectionSolutionsRemainder = [] + selectionSolutionsIds = set([]) + for validSelectionSolution in validSelectionSolutions: + (validSolution, validSolutionInfo) = validSelectionSolution + if validSolution in logicAnalyzer.solutions: + validExactSolutionIndex = logicAnalyzer.solutions.index(validSolution) + selectionSolutionsIds.add(validExactSolutionIndex) + validExactSolution = logicAnalyzer.solutions[validExactSolutionIndex] + validSelectionSolutionsIncluded.append((validExactSolution, validSolutionInfo)) + else: + validSelectionSolutionsRemainder.append(validSelectionSolution) + + selectionSolutions = [] + for i in range(0 ,len(validSelectionSolutionsIncluded)): + validSelectionSolution = validSelectionSolutionsIncluded[i] + (validSolution, validSolutionInfo) = validSelectionSolution + validSolution["Ideals"] = validSolutionInfo + + solutionsStartIndex = len(logicAnalyzer.solutions) + + for i in range(0, len(validSelectionSolutionsRemainder)): + validSelectionSolution = validSelectionSolutionsRemainder[i] + (validSolution, validSolutionInfo) = validSelectionSolution + selectionSolutionIndex = solutionsStartIndex + i + selectionSolutionsIds.add(selectionSolutionIndex) + validSolution["SolutionNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming) + validSolution["Ideals"] = validSolutionInfo + selectionSolutions.append(validSolution) + + selectionSolutionsIdsList = list(selectionSolutionsIds) + + ###################################### + # Correct outliers + """ + if inputParameters["SmoothOutliers"]: + logicAnalyzer.smoothOutliers() + """ + numProblemSizes = logicAnalyzer.numProblemSizes + # print all 2D + numPermutations = 1 + permutations = [] + for i in range(0, logicAnalyzer.numIndices): + if i != logicAnalyzer.idx0 and i != logicAnalyzer.idx1: + numPermutations *= numProblemSizes[i] + #print numPermutations + for j in range(0, numPermutations): + pIdx = j + permutation = [] + for i in range(0, logicAnalyzer.numIndices): + if i != logicAnalyzer.idx0 and i != logicAnalyzer.idx1: + npsi = numProblemSizes[i] + permutation.append(pIdx%npsi) + pIdx /= numProblemSizes[i] + permutations.append(permutation) + #print permutations + for permutation in permutations: + logicAnalyzer.print2D(permutation) + + ###################################### + # Range Logic + rangeLogic = logicAnalyzer.enRule(0, logicAnalyzer.globalIndexRange) + print2("# Final Range Logic:") + print2(rangeLogic) + logicComplexity = [0]*logicAnalyzer.numIndices + logicAnalyzer.scoreLogicComplexity(rangeLogic, logicComplexity) + print2("# Range Logic Complexity: %s" % logicComplexity) + score = logicAnalyzer.scoreRangeForLogic( \ + logicAnalyzer.globalIndexRange, rangeLogic) + print1("\n# Score: %.0f ms" % (score/1000)) + logicAnalyzer.prepareLogic(rangeLogic) # convert indices to sizes, -1 + + ###################################### + # Range Logic + exactLogic = logicAnalyzer.exactWinners + print1("# Exact Logic:\n") + print1("%s"%exactLogic) + + #selectionSolutionsIdsList = list(selectionSolutionsIds) + return (problemType, logicAnalyzer.solutions, logicAnalyzer.indexOrder, \ + exactLogic, rangeLogic, selectionSolutions, selectionSolutionsIdsList, logicAnalyzer.perfMetric) + + + +################################################################################ +# LogicAnalyzer +################################################################################ +class LogicAnalyzer: + + ############################################################################## + ############################################################################## + ### + ### Entry / Top-Level Functions + ### + ############################################################################## + ############################################################################## + + ############################################################################## + # ENTRY: Init + ############################################################################## + def __init__(self, problemType, problemSizesList, solutionsList, \ + dataFileNameList, inputParameters): + + # parameters + self.parameters = inputParameters + + # problem type + self.problemType = problemType + self.idx0 = self.problemType["Index0"] + self.idx1 = self.problemType["Index1"] + self.idxU = self.problemType["IndexUnroll"] + + # merge solutions from size groups + # solutions needs to be a set, and offset needs to be mapping + print1("# Merging Solutions:") + self.numSolutionsPerGroup = [] + self.solutionGroupMap = [] + self.solutions = [] + solutionsHash = {} # for accelerating lookups + + totalSolutions = 0 + for solutionGroupIdx in range(0, len(solutionsList)): + solutionGroup = solutionsList[solutionGroupIdx] + totalSolutions += len(solutionGroup) + progressBar = ProgressBar(totalSolutions) + for solutionGroupIdx in range(0, len(solutionsList)): + solutionGroup = solutionsList[solutionGroupIdx] + self.numSolutionsPerGroup.append(len(solutionGroup)) + self.solutionGroupMap.append({}) + for solutionIdx in range(0, len(solutionGroup)): + solution = solutionGroup[solutionIdx] + if not solution in solutionsHash: + sIdx = len(self.solutions) # the one we are about to add + self.solutions.append(solution) + solutionsHash[solution] = sIdx + else: + sIdx = solutionsHash[solution] + + self.solutionGroupMap[solutionGroupIdx][solutionIdx] = sIdx + progressBar.increment() + self.numSolutions = len(self.solutions) + self.solutionMinNaming = Solution.getMinNaming(self.solutions) + self.solutionNames = [] + self.solutionTiles = [] + for solution in self.solutions: + self.solutionNames.append(Solution.getNameMin(solution, \ + self.solutionMinNaming)) + self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ + solution["MacroTile1"])) + self.flopsPerMac = self.problemType["DataType"].flopsPerMac() + + # merge problem sizes from size groups + #self.numIndices = len(problemSizesList[0].numProblemSizes) + self.numIndices = self.problemType["TotalIndices"] + problemType["NumIndicesLD"] + unifiedProblemSizes = [] + for i in range(0, self.numIndices): + unifiedProblemSizes.append(set()) + self.exactProblemSizes = set() + self.rangeProblemSizes = set() + for problemSizes in problemSizesList: + # add exacts + for problem in problemSizes.exacts: + self.exactProblemSizes.add(tuple(problem.sizes)) + + # add ranges + #print "ProblemSizes", problemSizes.sizes + #FIXME-problem + self.rangeProblemSizes.update([tuple(problem.sizes) for problem in problemSizes.problems]) + for rangeSize in problemSizes.ranges: + + if globalParameters["ExpandRanges"]: + # Treat ranges as pile of exacts: + for rsize in rangeSize.problemSizes: + self.exactProblemSizes.add(tuple(rsize)) + else: + # Create the ranges info in the logic file + #print "RangeSize", rangeSize + sizedIdx = 0 + mappedIdx = 0 + for i in range(0, self.numIndices): + if rangeSize.indexIsSized[i]: + index = rangeSize.indicesSized[sizedIdx] + sizedIdx += 1 + else: + index = rangeSize.indicesSized[ \ + rangeSize.indicesMapped[mappedIdx]] + mappedIdx += 1 + currentSize = index[0] + currentStride = index[1] + while currentSize <= index[3]: + unifiedProblemSizes[i].add(currentSize) + currentSize += currentStride + currentStride += index[2] + for i in range(0, len(unifiedProblemSizes)): + unifiedProblemSizes[i] = sorted(list(unifiedProblemSizes[i])) + print2("UnifiedProblemSizes: %s" % unifiedProblemSizes) + print2("ExactProblemSizes: %s" % self.exactProblemSizes) + print2("RangeProblemSizes: %s" % self.rangeProblemSizes) + + # problem size index <-> size + self.problemSizeToIndex = [] + self.problemIndexToSize = [] + self.numProblemSizes = [] + for i in range(0, self.numIndices): + self.problemSizeToIndex.append({}) + self.problemIndexToSize.append([]) + for j in range(0, len(unifiedProblemSizes[i])): + size = unifiedProblemSizes[i][j] + self.problemSizeToIndex[i][size] = j + self.problemIndexToSize[i].append(size) + self.numProblemSizes.append(len(unifiedProblemSizes[i])) + print1("# NumProblemSizes: %s" % self.numProblemSizes) + + # total size of data array + self.totalProblems = 1 + for numProblems in self.numProblemSizes: + self.totalProblems *= numProblems + self.totalSize = self.totalProblems * self.numSolutions + print2("TotalProblems: %u" % self.totalProblems) + print2("TotalSolutions: %u" % self.numSolutions) + print2("TotalSize: %u" % self.totalSize) + # data is a 2D array [problemIdx][solutionIdx] which stores perf data in gflops for + # the specified solution + self.data = array.array('f', [-2]*self.totalSize) + + # Each entry in exactWinners is a 2D array [solutionIdx, perf] + self.exactWinners = {} + + """ + # map problem sizes -> index + self.problemSizeToIndex = [] + self.problemIndexToSize = [] + sizedIdx = 0 + mappedIdx = 0 + for i in range(0, self.numIndices): + self.problemSizeToIndex.append({}) + self.problemIndexToSize.append([]) + if self.problemSizes.indexIsSized[i]: + index = self.problemSizes.indicesSized[sizedIdx] + sizedIdx += 1 + else: + index = self.problemSizes.indicesSized[ \ + self.problemSizes.indicesMapped[mappedIdx]] + mappedIdx += 1 + currentSize = index[0] + currentStride = index[1] + idx = 0 + while currentSize <= index[3]: + self.problemSizeToIndex[i][currentSize] = idx + self.problemIndexToSize[i].append(currentSize) + currentSize += currentStride + currentStride += index[2] + idx += 1 + """ + #self.rangeIndicesFree = range(0, self.problemType["NumIndicesC"]) + #self.rangeIndicesSummation = range(self.problemType["NumIndicesC"], \ + # self.problemType["TotalIndices"]) + self.indexOrder = self.recommendedIndexOrder() + print2("IndexOrder: %s" % self.indexOrder) + self.globalIndexRange = [] + for i in range(0, self.numIndices): + self.globalIndexRange.append([0, self.numProblemSizes[i]]) + self.problemIndicesForGlobalRange \ + = self.problemIndicesForRange(self.globalIndexRange) + self.tab = [""]*self.numIndices + + ###################################### + # Read Data From CSV + for fileIdx in range(0, len(dataFileNameList)): + dataFileName = dataFileNameList[fileIdx] + self.addFromCSV(dataFileName, self.numSolutionsPerGroup[fileIdx], \ + self.solutionGroupMap[fileIdx]) + + + + #print self.data + # map exact problem sizes to solutions + print1("# ExactWinners: %s" % self.exactWinners) + + + ############################################################################## + # ENTRY: Add From CSV + ############################################################################## + def addFromCSV(self, dataFileName, numSolutions, solutionMap): + + # open file + print("reading datafile", dataFileName) + try: + dataFile = open(dataFileName, "r") + except IOError: + printExit("Can't open \"%s\" to get data" % dataFileName ) + + # column indices + csvFile = csv.reader(dataFile) + problemSizeStartIdx = 1 + # notice that for OperationType != GEMM, the numIndices = 0 + totalSizeIdx = problemSizeStartIdx + self.numIndices + + # need to take care if the loaded csv is the export-winner-version + csvHasWinner = "_CSVWinner" in dataFileName + if csvHasWinner: + # the column of the two are fixed (GFlops, SizeI/J/K/L, LDD/C/A/B, TotalFlops, WinnerGFlops, WinnerTimeUs, WinnerIdx, WinnerName) + # the order are implemented in ResultFileReporter.cpp + columnOfWinnerGFlops = 10 + columnOfWinnerIdx = 12 + + # iterate over rows + rowIdx = 0 + for row in csvFile: + rowIdx+=1 + if rowIdx == 1: + # get unit (gflops or gflops/cu) of benchmark data + perfUnit = row[0] + if perfUnit == "GFlops": + self.perfMetric = "DeviceEfficiency" + elif perfUnit == "GFlopsPerCU": + self.perfMetric = "CUEfficiency" + else: + printWarning("Performance unit %s in %s is unrecognized: assuming GFlops (device efficiency)" % (perfUnit, dataFileName)) + self.perfMetric = "DeviceEfficiency" + + # get the length of each row, and derive the first column of the solution instead of using wrong "solutionStartIdx = totalSizeIdx + 1" + rowLength = len(row) + solutionStartIdx = rowLength - numSolutions + continue + else: + #if len(row) < rowLength: + # printWarning("CSV File %s row %u doesn't have %u elements; ignoring remainer of file." \ + # % (dataFileName, rowIdx, rowLength) ) + # break + + # get problem size + problemSize = [] + for i in range(problemSizeStartIdx, totalSizeIdx): + problemSize.append(int(row[i])) + problemSize = tuple(problemSize) + + # Exact Problem Size + if problemSize in self.exactProblemSizes: + + if csvHasWinner: + # Faster. Get the winner info from csv directly, avoid an extra loop + winnerGFlops = float(row[columnOfWinnerGFlops]) + winnerIdx = int(row[columnOfWinnerIdx]) + else: + # Old code. TODO - Can we get rid of this in the future? + # solution gflops + solutionIdx = 0 + winnerIdx = -1 + winnerGFlops = -1 + for i in range(solutionStartIdx, rowLength): + gflops = float(row[i]) + if gflops > winnerGFlops: + winnerIdx = solutionIdx + winnerGFlops = gflops + solutionIdx += 1 + + if winnerIdx != -1: + if problemSize in self.exactWinners: + if winnerGFlops > self.exactWinners[problemSize][1]: + #print "update exact", problemSize, "CSV index=", winnerIdx, self.exactWinners[problemSize], "->", solutionMap[winnerIdx], winnerGFlops + self.exactWinners[problemSize] = [solutionMap[winnerIdx], winnerGFlops] + else: + self.exactWinners[problemSize] = [solutionMap[winnerIdx], winnerGFlops] + #print "new exact", problemSize, "CSV index=", winnerIdx, self.exactWinners[problemSize] + + # Range Problem Size + elif problemSize in self.rangeProblemSizes: + problemIndices = [] + for i in range(0, self.numIndices): + problemIndices.append(self.problemSizeToIndex[i][problemSize[i]]) + serialIdx = self.indicesToSerial(0, problemIndices) + # solution gflops + solutionIdx = 0 + for i in range(solutionStartIdx, rowLength): + gflops = float(row[i]) + self.data[serialIdx+solutionMap[solutionIdx]] = gflops + solutionIdx += 1 + + # Unknown Problem Size + else: + printExit("Huh? %s has ProblemSize %s which isn't in its yaml" \ + % ( dataFileName, list(problemSize)) ) + #print self.data + + + ############################################################################## + # ENTRY: Remove Invalid Solutions + ############################################################################## + def removeInvalidSolutions(self): + #problemIndices = [0]*self.numIndices + allSolutionValid = False + while not allSolutionValid: + invalidIdx = -1 + for problemIndices in self.problemIndicesForGlobalRange: + problemSerial = self.indicesToSerial(0, problemIndices) + for solutionIdx in range(0, self.numSolutions): + gflops = self.data[problemSerial+solutionIdx] + if gflops == 0: + invalidIdx = solutionIdx + break + if invalidIdx >= 0: + print1("# Removing Invalid Solution: %u %s" \ + % (invalidIdx, self.solutionNames[invalidIdx]) ) + self.removeSolution(invalidIdx) + else: + allSolutionValid = True + + + ############################################################################## + # ENTRY: Original KeepLogic algorithm: Remove Least Important Solutions, + # one at a time. Stop when leastImportantSolution indicates no more + # solutions can be removed, which appears to be when the solution + # is used by an exact problem or is the only possible solution for some + # problem or doesn't improve the a solution by > SolutionImportanceMin% + ############################################################################## + def removeLeastImportantSolutions(self): + # Remove least important solutions + start = time.time() + while len(self.solutions) > 1: + lisTuple = self.leastImportantSolution() + if lisTuple != None: + lisIdx = lisTuple[0] + lisPercSaved = lisTuple[1] + lisPercWins = lisTuple[2] + lisPercTime = lisTuple[3] + if lisPercSaved < self.parameters["SolutionImportanceMin"] or lisPercWins == 0: + print1("# Removing Unimportant Solution %u/%u: %s ( %f%% wins, %f%% ms time, %f%% ms saved" \ + % (lisIdx, self.numSolutions, self.solutionNames[lisIdx], 100*lisPercWins, 100*lisPercTime, 100*lisPercSaved) ) + self.removeSolution(lisIdx) + continue + else: + break + else: # no more lis, remainders are exact winner + break + stop = time.time() + print("removeLeastImportantSolutions elapsed time = %.1f secs" % (stop - start)) + + + ############################################################################## + # ENTRY: Alternate KeepLogic algorithm that keeps the fastest for each + # exact and range. Other solutions are removed. + ############################################################################## + def keepWinnerSolutions(self): + + # solution indexes for the winners: + winners = set() + + solutionImportance = [] + for i in range(0, self.numSolutions): + solutionImportance.append([i, 0, 0, 0, False]) + problemSizes = [0]*self.numIndices + print("problemIndicesForGlobalRange", self.problemIndicesForGlobalRange) + for problemIndices in self.problemIndicesForGlobalRange: + for i in range(0, self.numIndices): + problemSizes[i] = self.problemIndexToSize[i][problemIndices[i]] + totalFlops = self.flopsPerMac + for size in problemSizes: + totalFlops *= size + + problemSerial = self.indicesToSerial(0, problemIndices) + winnerIdx = -1 + winnerGFlops = -1e6 + for solutionIdx in range(0, self.numSolutions): + solutionSerialIdx = problemSerial + solutionIdx + solutionGFlops = self.data[solutionSerialIdx] + if solutionGFlops > winnerGFlops: + winnerIdx = solutionIdx + winnerGFlops = solutionGFlops + + winners.add(winnerIdx) + + # Always keep the exact sizes: + for exactProblem in self.exactWinners: + winnerIdx = self.exactWinners[exactProblem][0] + #print "keepWinnerSolution adding exact", exactProblem, winnerIdx + winners.add(winnerIdx) + + print("Winners", winners) + self.pruneSolutions(winners) + + + + ############################################################################## + # ENTRY: En Rule + # currentIndexIndex = 0, 1, 2, 3... + # currentIndexRange will have only 1 size for prior indices (unless initial) + # + # Rule: + # [128, [ + # [64, [ + # [16, 0], + # [2880,1] + # ] + # ], + # [96, [ + # [16, 0], + # [64, 1] + # ] + # ] + # ] + # ], another + # + # + ############################################################################## + def enRule(self, currentIndexIndex, currentIndexRange): + cii = currentIndexIndex + if currentIndexIndex == 0: + self.tab[cii] = "[] " + elif currentIndexIndex == 1: + self.tab[cii] = "[%2u] " % ( \ + currentIndexRange[self.indexOrder[0]][0]) + elif currentIndexIndex == 2: + self.tab[cii] = "[%2u,%2u] " % ( \ + currentIndexRange[self.indexOrder[0]][0], \ + currentIndexRange[self.indexOrder[1]][0]) + elif currentIndexIndex == 3: + self.tab[cii] = "[%2u,%2u,%2u] " % ( \ + currentIndexRange[self.indexOrder[0]][0], \ + currentIndexRange[self.indexOrder[1]][0], \ + currentIndexRange[self.indexOrder[2]][0]) + elif currentIndexIndex == 4: + self.tab[cii] = "[%2u,%2u,%2u,%2u] " % ( \ + currentIndexRange[self.indexOrder[0]][0], \ + currentIndexRange[self.indexOrder[1]][0], \ + currentIndexRange[self.indexOrder[2]][0], \ + currentIndexRange[self.indexOrder[3]][0]) + tab = self.tab[cii] + if globalParameters["PrintLevel"] == 1: + stdout.write("\n%s"%tab) + currentIndex = self.indexOrder[currentIndexIndex] + print2("%senRule(%s)" % (tab, currentIndexRange)) + nextIndexIndex = currentIndexIndex+1 + nextIndexRange = deepcopy(currentIndexRange) + isLastIndex = currentIndexIndex == self.numIndices-1 + ruleList = [] + + ######################################## + # SingleProblem + ######################################## + if currentIndexRange[currentIndex][1] \ + - currentIndexRange[currentIndex][0] == 1: + + ######################################## + # SingleProblem & LastIndex + # this is last index, so just return fastest solution + ######################################## + if isLastIndex: + print2("%sSingleProblem & LastIndex" % tab) + winnerIdx = self.winnerForRange(currentIndexRange) + if winnerIdx < 0: + print2("%sSingleProblem & LastIndex :: winnerIdx<0; returning" % (tab) ) + return None + ruleList.append([-1, winnerIdx]) + if globalParameters["PrintLevel"] == 1: + stdout.write("%") + + ######################################## + # SingleProblem & NotLastIndex + # this isn't last index, so just recursively return next index + ######################################## + else: + print2("%sSingleProblem & NotLastIndex" % tab) + # % (tab, nextIndexRange) ) + nextRule = self.enRule(nextIndexIndex, nextIndexRange) + if nextRule == None: + print2("%sSingleProblem & NotLastIndex :: nextRule==None; returning" % (tab) ) + return None + rule = [ -1, nextRule ] + ruleList.append(rule) + if globalParameters["PrintLevel"] == 1: + stdout.write("%") + + else: + ######################################## + # MultiProblem + # Create Initial Rule + ######################################## + + if isLastIndex: + ######################################## + # MultiProblem & LastIndex + # InitialRule using winnerForRange() + ######################################## + print2("%sMultiProblem & LastIndex" % tab) + winnerIdx = -1 + for problemIndex in range(currentIndexRange[currentIndex][0], \ + currentIndexRange[currentIndex][1]): + nextIndexRange[currentIndex][0] = problemIndex + nextIndexRange[currentIndex][1] = problemIndex+1 + winnerIdx = self.winnerForRange(nextIndexRange) + initialRule = [ currentIndexRange[currentIndex][0], winnerIdx] + if winnerIdx >= 0: + break + if winnerIdx < 0: + print2("%sMultiProblem & LastIndex :: winnerIdx<0; returning" % (tab) ) + return None + + else: + ######################################## + # MultiProblem & NotLastIndex + # InitialRule using enRule() + ######################################## + print2("%sMultiProblem & NotLastIndex" % tab) + + # create initial rule + winnerIdx = -1 + nextRule = None + for problemIndex in range(currentIndexRange[currentIndex][0], \ + currentIndexRange[currentIndex][1]): + nextIndexRange[currentIndex][0] = problemIndex + nextIndexRange[currentIndex][1] = problemIndex+1 + nextRule = self.enRule(nextIndexIndex, nextIndexRange) + # break when found initial rule + if nextRule != None: + break + + if nextRule == None: + printWarning("%sMultiProblem & NotLastIndex :: nextRule==None; returning" % (tab) ) + return None + initialRule = [ currentIndexRange[currentIndex][0], nextRule ] + ruleList.append(initialRule) + print2("%sMultiProblem::InitialRuleList=%s" % (tab, ruleList)) + if globalParameters["PrintLevel"] == 1: + stdout.write("#") + + ######################################## + # MultiProblem + # Append Rules to Initial Rule + ######################################## + print2("%sMultiProblem::Improving Rule" % tab) + for problemIndex in range(currentIndexRange[currentIndex][0]+1, \ + currentIndexRange[currentIndex][1]): + nextIndexRange[currentIndex][0] = problemIndex + nextIndexRange[currentIndex][1] = problemIndex+1 + priorRule = ruleList[len(ruleList)-1] + priorRuleForSize = deepcopy(priorRule) + priorRuleForSize[0] = problemIndex + + if isLastIndex: + ######################################## + # nextRule using winnersForRange() + winnerIdx = self.winnerForRange(nextIndexRange) + print2("%sMultiProblem::ImproveRule[%u]::LastIndex::WinnerIdx=%u for %s" % (tab, problemIndex, winnerIdx, nextIndexRange)) + # if no solutions benchmarked for this problem size, continue + if winnerIdx < 0: + ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE + print2("%sUpdating range b/c None" % tab) + if globalParameters["PrintLevel"] == 1: + stdout.write(" ") + continue + else: + candidateRule = [ problemIndex, winnerIdx] + else: + ######################################## + # nextRule using enRule() + nextRule = self.enRule(nextIndexIndex, nextIndexRange) + print2("%sMultiProblem::ImproveRule[%u]::NotLastIndex::NextRule=%s for %s; %s" % (tab, problemIndex, nextRule, nextIndexIndex, nextIndexRange)) + if nextRule == None: + ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE + print2("%sUpdating b/c None" % tab) + if globalParameters["PrintLevel"] == 1: + stdout.write(" ") + continue + else: + candidateRule = [ problemIndex, nextRule ] + + ######################################## + # candidate same as prior + if candidateRule[1] == priorRule[1]: + print2("%sCandidateRule==PriorRule; just updating prior" % (tab)) + ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE + if globalParameters["PrintLevel"] == 1: + stdout.write(" ") + continue + + ######################################## + # compare candidate vs prior + else: + print2("%sCandidateRule!=PriorRule; appending rule assuming its better" % (tab)) + + """ + priorRuleScore = self.scoreRangeForLogic(nextIndexRange, \ + [priorRuleForSize]) + logicComplexity = [0]*self.numIndices + self.scoreLogicComplexity( \ + [priorRuleForSize], logicComplexity) + priorRuleScore += self.parameters["BranchPenalty"] \ + * sum(logicComplexity) + # score candidate + candidateRuleScore = self.scoreRangeForLogic(nextIndexRange, \ + [candidateRule]) + #print "CRS", candidateRuleScore + logicComplexity = [0]*self.numIndices + self.scoreLogicComplexity( \ + [candidateRule], logicComplexity) + candidateRuleScore += self.parameters["BranchPenalty"] \ + * sum(logicComplexity) + candidateRuleScore += self.parameters["BranchPenalty"] # penalize + candidateFaster = candidateRuleScore < priorRuleScore + print2("%sP[%2u]: %s %s~%.0fus < %s~%.0fus" % (tab, problemIndex, \ + "wins" if candidateFaster else "same", \ + candidateRule, candidateRuleScore, priorRuleForSize, \ + priorRuleScore )) + """ + + ######################################## + # candidate wins + #print candidateRuleScore, priorRuleScore + if True: # or candidateRuleScore < priorRuleScore: + ruleList.append(candidateRule) + print2("%sAppending b/c Different" % tab) + if globalParameters["PrintLevel"] == 1: + stdout.write("#") + + ######################################## + # prior wins + else: + print2("%sPrior Rule Wins" % tab) + if globalParameters["PrintLevel"] == 1: + stdout.write(".") + ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE + + print2("%sReturning RuleList: %s" % (tab, ruleList)) + return ruleList + + + + ############################################################################## + ############################################################################## + ### + ### Mid-Level Functions + ### + ############################################################################## + ############################################################################## + + + + ############################################################################## + # Prepare Logic + # convert threshold indices to sizes + # last threshold = -1 + ############################################################################## + def prepareLogic(self, logic): + depth = self.getLogicDepth(logic) + if depth == 0: return + indexIndex = self.numIndices - depth + index = self.indexOrder[indexIndex] + for i in range(0, len(logic)): + if i == len(logic)-1: + logic[i][0] = -1 + else: + logic[i][0] = self.problemIndexToSize[index][logic[i][0]] + self.prepareLogic(logic[i][1]) + + + ############################################################################## + # Print2D + ############################################################################## + def print2D(self, indices ): + indicesIdx = 0 + problemIndices = [] + for i in range(0, self.numIndices): + if i == self.idx0: + problemIndices.append(-1) + elif i == self.idx1: + problemIndices.append(-1) + else: + problemIndices.append(indices[indicesIdx]) + indicesIdx += 1 + + winnerIndices = [] + w = "winner" + g = "gflops" + f = "faster" + s = "second" + sss = [] + for sIdx in range(0, self.numSolutions): + sss.append("Sol[%u]" % sIdx) + for j in range(0, self.numProblemSizes[1]): + w += ",%4u" % self.problemIndexToSize[1][j] + g += ",%4u" % self.problemIndexToSize[1][j] + f += ",%4u" % self.problemIndexToSize[1][j] + s += ",%4u" % self.problemIndexToSize[1][j] + for sIdx in range(0, self.numSolutions): + sss[sIdx] += ",%4u" % self.problemIndexToSize[1][j] + w += "\n" + g += "\n" + f += "\n" + s += "\n" + for sIdx in range(0, self.numSolutions): + sss[sIdx] += "\n" + for i in range(0, self.numProblemSizes[0]): + problemIndices[self.idx0] = i + w += "%4u" % self.problemIndexToSize[0][i] + g += "%4u" % self.problemIndexToSize[0][i] + f += "%4u" % self.problemIndexToSize[0][i] + s += "%4u" % self.problemIndexToSize[0][i] + for sIdx in range(0, self.numSolutions): + sss[sIdx] += "%4u" % self.problemIndexToSize[0][i] + for j in range(0, self.numProblemSizes[1]): + problemIndices[self.idx1] = j + problemSerial = self.indicesToSerial(0, problemIndices) + for sIdx in range(0, self.numSolutions): + sss[sIdx] += ",%f" % self.data[problemSerial+sIdx] + winnerIdx = 0 + secondIdx = 1 + winnerGFlops = self.data[problemSerial+0] + secondGFlops = 1e-9 + for solutionIdx in range(1, self.numSolutions): + solutionSerialIdx = problemSerial + solutionIdx + solutionGFlops = self.data[solutionSerialIdx] + if solutionGFlops > winnerGFlops: + secondIdx = winnerIdx + secondGFlops = winnerGFlops + winnerIdx = solutionIdx + winnerGFlops = solutionGFlops + elif solutionGFlops > secondGFlops: + secondIdx = solutionIdx + secondGFlops = solutionGFlops + + if winnerIdx not in winnerIndices: + winnerIndices.append(winnerIdx) + w += ",%4u" % winnerIdx + g += ",%f" % winnerGFlops + f += ",%f" % (winnerGFlops/secondGFlops) + s += ",%4u" % (secondIdx) + w += "\n" + g += "\n" + f += "\n" + s += "\n" + for sIdx in range(0, self.numSolutions): + sss[sIdx] += "\n" + + w += "\n\n" + g += "\n\n" + f += "\n\n" + s += "\n\n" + for sIdx in range(0, self.numSolutions): + sss[sIdx] += "\n\n" + w += "Winners:\n" + for winnerIdx in winnerIndices: + w += "%4u, %s, %s\n" % (winnerIdx, self.solutionTiles[winnerIdx], self.solutionNames[winnerIdx]) + + printFileName = "Winner2D" + for idx in indices: + printFileName += "_%u" % idx + printFileName += ".csv" + printFile = open(os.path.join(globalParameters["WorkingPath"], printFileName), "w") + printFile.write( w ) + printFile.write( g ) + printFile.write( f ) + printFile.write( s ) + for sIdx in range(0, self.numSolutions): + printFile.write( sss[sIdx] ) + printFile.close() + + + ############################################################################## + # Least Important Solution + ############################################################################## + def leastImportantSolution(self): + solutionImportance = [] + for i in range(0, self.numSolutions): + solutionImportance.append([i, 0, 0, 0, False]) + problemSizes = [0]*self.numIndices + totalSavedMs = 0 + totalExecMs = 0 + totalWins = 0 + for problemIndices in self.problemIndicesForGlobalRange: + for i in range(0, self.numIndices): + problemSizes[i] = self.problemIndexToSize[i][problemIndices[i]] + totalFlops = self.flopsPerMac + for size in problemSizes: + totalFlops *= size + + problemSerial = self.indicesToSerial(0, problemIndices) + winnerIdx = -1 + winnerGFlops = -1e6 + secondGFlops = -1e9 + for solutionIdx in range(0, self.numSolutions): + solutionSerialIdx = problemSerial + solutionIdx + solutionGFlops = self.data[solutionSerialIdx] + if solutionGFlops > winnerGFlops: + secondGFlops = winnerGFlops + winnerIdx = solutionIdx + winnerGFlops = solutionGFlops + elif solutionGFlops > secondGFlops: + secondGFlops = solutionGFlops + + winnerTimeMs = totalFlops / winnerGFlops / 1000000.0 + secondTimeMs = totalFlops / secondGFlops / 1000000.0 + if winnerGFlops > 0 and secondGFlops > 0: + solutionImportance[winnerIdx][1] += (secondTimeMs - winnerTimeMs) + totalSavedMs += secondTimeMs - winnerTimeMs + if winnerGFlops > 0: + solutionImportance[winnerIdx][2] += 1 + solutionImportance[winnerIdx][3] += winnerTimeMs + totalExecMs += winnerTimeMs + totalWins += 1 + if secondGFlops <= 0: + solutionImportance[winnerIdx][4] = True # this is only valid solution for this problem size, keep it + + + # print data before sorting + for i in range(0, self.numSolutions): + print2("[%2u] %s: %e saved, %u wins, %u time, %s" \ + % (solutionImportance[i][0], \ + self.solutionNames[solutionImportance[i][0]], \ + solutionImportance[i][1], \ + solutionImportance[i][2], \ + solutionImportance[i][3], \ + "singular" if solutionImportance[i][4] else "" ) ) + + totalSavedMs = max(1, totalSavedMs) + solutionImportance.sort(key=lambda x: x[1]) + for i in range(0, self.numSolutions): + solutionIdx = solutionImportance[i][0] + canRemove = not solutionImportance[i][4] # don't remove if is only win for any size + for exactProblem in self.exactWinners: + winnerIdx = self.exactWinners[exactProblem][0] + if solutionIdx == winnerIdx: # exact winners are important + canRemove = False + break + if canRemove: + idx = solutionImportance[i][0] + if totalSavedMs > 0: + percSaved = 1.0 * solutionImportance[i][1] / totalSavedMs + else: + percSaved = 0 + if totalWins > 0: + percWins = 1.0 * solutionImportance[i][2] / totalWins + else: + percWins = 0 + if totalExecMs > 0: + percTime = 1.0 * solutionImportance[i][3] / totalExecMs + else: + percTime = 0 + return ( idx, percSaved, percWins, percTime ) + return None + + + ############################################################################## + # Remove Solution + ############################################################################## + def removeSolution(self, removeSolutionIdx): + + # temporarily move current to old + oldSolutions = deepcopy(self.solutions) + oldNumSolutions = self.numSolutions + oldData = deepcopy(self.data) + + # update solutions + self.solutions = [] + for i in range(0, oldNumSolutions): + if i != removeSolutionIdx: + self.solutions.append(oldSolutions[i]) + self.solutionMinNaming = Solution.getMinNaming(self.solutions) + self.solutionNames = [] + self.solutionTiles = [] + for solution in self.solutions: + self.solutionNames.append(Solution.getNameMin(solution, \ + self.solutionMinNaming)) + self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ + solution["MacroTile1"])) + self.numSolutions = len(self.solutions) + + # update data + self.totalSize = self.totalProblems * self.numSolutions + self.data = array.array('f', [0]*self.totalSize) + for problemIndex in range(0, self.totalProblems): + newSolutionIdx = 0 + for oldSolutionIdx in range(0, oldNumSolutions): + if oldSolutionIdx != removeSolutionIdx: + self.data[problemIndex*self.numSolutions+newSolutionIdx] \ + = oldData[problemIndex*oldNumSolutions+oldSolutionIdx] + newSolutionIdx += 1 + + # update exact Winners + for problemSize in self.exactWinners: + if self.exactWinners[problemSize][0] >= removeSolutionIdx: + self.exactWinners[problemSize][0] -= 1 + + + ############################################################################## + # Prune a list of solutions, keeping only the indices specified in + # keepSolutions. keepSolutions is a set not a list + ############################################################################## + def pruneSolutions(self, keepSolutions): + + removeSolutionIdxList = [] + solutionMapNewToOld = [] # dense mapping + solutionMapOldToNew = [-1] * self.numSolutions + + # temporarily move current to old + oldSolutions = deepcopy(self.solutions) + oldNumSolutions = self.numSolutions + oldData = deepcopy(self.data) + # update solutions + self.solutions = [] + for i in range(0, oldNumSolutions): + if i in keepSolutions: + solutionMapNewToOld.append(i) + solutionMapOldToNew[i] = len(self.solutions) + self.solutions.append(oldSolutions[i]) + else: + removeSolutionIdxList.append(i) + + self.solutionMinNaming = Solution.getMinNaming(self.solutions) + self.solutionNames = [] + self.solutionTiles = [] + for solution in self.solutions: + self.solutionNames.append(Solution.getNameMin(solution, \ + self.solutionMinNaming)) + self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ + solution["MacroTile1"])) + self.numSolutions = len(self.solutions) + + # update data + self.totalSize = self.totalProblems * self.numSolutions + self.data = array.array('f', [0]*self.totalSize) + for problemIndex in range(0, self.totalProblems): + for newSolutionIdx in range(0, self.numSolutions): + oldSolutionIdx = solutionMapNewToOld[newSolutionIdx] + self.data[problemIndex*self.numSolutions+newSolutionIdx] \ + = oldData[problemIndex*oldNumSolutions+oldSolutionIdx] + + # update exact Winners + for problemSize in self.exactWinners: + #print "prune updating exacWinner", problemSize, \ + # "from ", self.exactWinners[problemSize][0], \ + # "to ", solutionMapOldToNew[self.exactWinners[problemSize][0]] + self.exactWinners[problemSize][0] = \ + solutionMapOldToNew[self.exactWinners[problemSize][0]] + if self.exactWinners[problemSize][0] == -1: + print(("warning: exactWinner[", problemSize, "] == -1")) + if self.exactWinners[problemSize][0] >= self.numSolutions: + print(("warning: exactWinner[", problemSize, "] ")) + + + ############################################################################## + # Score Range For Logic + ############################################################################## + def scoreRangeForLogic(self, indexRange, logic): + depth = self.getLogicDepth(logic) + depth = self.numIndices - depth + fullLogic = deepcopy(logic) + for i in range(0, depth): + fullLogic = [[-1, fullLogic]] + return self.scoreRangeForFullLogic(depth, indexRange, fullLogic) + + ############################################################################## + # Score Range For Full Logic + ############################################################################## + def scoreRangeForFullLogic(self, depth, indexRange, logic): + score = 0 + for problemIndices in self.problemIndicesForRange(indexRange): + problemSerial = self.indicesToSerial(0, problemIndices) + totalFlops = self.totalFlopsForProblemIndices(problemIndices) + solutionIdx = self.getSolutionForProblemIndicesUsingLogic( \ + problemIndices, logic) + if solutionIdx == None: + printWarning("SolutionIdx = None. This should never happen.") + continue + solutionGFlops = self.data[problemSerial + solutionIdx] + solutionGFlops = max(1E-9, solutionGFlops) + timeUs = totalFlops / solutionGFlops / 1000 + score += timeUs + return score + + ############################################################################## + # Get Solution For Problem Indices Using Logic + ############################################################################## + def getSolutionForProblemIndicesUsingLogic(self, problemIndices, logic): + currentProblemIndices = self.toIndexOrder(problemIndices) + currentLogic = logic + for i in range(0, self.numIndices): + currentSizeIndex = currentProblemIndices[0] + for j in range(0, len(currentLogic)): + if currentLogic[j][0] < 0: + currentProblemIndices = currentProblemIndices[1:] + currentLogic = currentLogic[j][1] + break + if currentLogic[j][0] >= 0: + if currentSizeIndex <= currentLogic[j][0]: + currentProblemIndices = currentProblemIndices[1:] + currentLogic = currentLogic[j][1] + break + return currentLogic + + + ############################################################################## + ############################################################################## + ### + ### Helper / Low-Level Functions + ### + ############################################################################## + ############################################################################## + + + ############################################################################## + # Get Winner For Problem + def getWinnerForProblem(self, problemIndices): + problemSerial = self.indicesToSerial(0, problemIndices) + winnerIdx = -1 + winnerGFlops = -1 + for solutionIdx in range(0, self.numSolutions): + solutionSerialIdx = problemSerial + solutionIdx + solutionGFlops = self.data[solutionSerialIdx] + solutionGFlops = max(1E-9, solutionGFlops) + if solutionGFlops > winnerGFlops: + winnerIdx = solutionIdx + winnerGFlops = solutionGFlops + # print "Winner %u %f" % (winnerIdx, winnerGFlops) + return (winnerIdx, winnerGFlops) + + + ############################################################################## + # Winner For Range, -1 if nothing benchmarked + def winnerForRange(self, indexRange): + if self.numSolutions == 1: + return 0 + else: + scores = self.scoreRangeForSolutions(indexRange) + winnerIdx = 0 + hasWinner = False + for solutionIdx in range(1, self.numSolutions): + if scores[solutionIdx] < scores[winnerIdx]: + winnerIdx = solutionIdx + hasWinner = True + elif scores[solutionIdx] > scores[winnerIdx]: + hasWinner = True + else: + pass # still no winner + + return winnerIdx if hasWinner else -1 + + + ############################################################################## + # Score (microseconds) Range For Solutions + def scoreRangeForSolutions(self, indexRange): + scores = [0]*self.numSolutions + for problemIndices in self.problemIndicesForRange(indexRange): + problemSerial = self.indicesToSerial(0, problemIndices) + totalFlops = self.totalFlopsForProblemIndices(problemIndices) + for solutionIdx in range(0, self.numSolutions): + gflops = self.data[problemSerial+solutionIdx] + if gflops > 0: + timeUs = totalFlops / gflops / 1000 + else: # this solution not benchmarked for this size, therefore set + # its score to +inf so that its automatically disqualified + timeUs = float("inf") + scores[solutionIdx] += timeUs + return scores + + + ############################################################################## + # Score Logic Complexity + def scoreLogicComplexity(self, logic, logicComplexity): + depth = self.getLogicDepth(logic) + if depth == 0: return + depth = self.numIndices - depth + for i in range(0, len(logic)): + logicComplexity[depth] += 1 + self.scoreLogicComplexity(logic[i][1], logicComplexity) + + + ############################################################################## + # Get Logic Depth + def getLogicDepth(self, logic): + obj = logic + depth = 0 + while isinstance(obj, list): + obj = obj[0][1] + depth += 1 + return depth + + + ############################################################################## + # To Index Order + def toIndexOrder(self, problemIndices): + ordered = [] + for i in self.indexOrder: + ordered.append(problemIndices[i]) + return ordered + + + ############################################################################## + # Total Flops For Problem Indices + def totalFlopsForProblemIndices(self, problemIndices): + totalFlops = self.flopsPerMac + for i in range(0, self.numIndices): + totalFlops *= self.problemIndexToSize[i][problemIndices[i]] + return totalFlops + + + ############################################################################## + # Recommended Index Order + # TODO, this may depend on transposes + def recommendedIndexOrder(self): + order = [] + for i in range(0, self.problemType["TotalIndices"]): + if i != self.idxU and i != self.idx1 and i != self.idx0: + order.append(i) + order.append(self.idxU) + order.append(self.idx0) + order.append(self.idx1) + return order + + ############################################################################## + # Problem Indices For Range + def problemIndicesForRange(self, indexRange): + problemIndexList = [] + # early return for empty set + for i in range(0, self.numIndices): + if indexRange[i][0] == indexRange[i][1]: + return [] + problemIndices = [] + for idx in indexRange: + problemIndices.append(idx[0]) + moreProblems = True + while moreProblems: + problemIndexList.append(deepcopy(problemIndices)) + # next problem + problemIndices[0] += 1 + for i in range(0, self.numIndices): + if problemIndices[i] >= indexRange[i][1]: + if i == self.numIndices-1: + moreProblems = False + break + else: + problemIndices[i] = indexRange[i][0] + problemIndices[i+1] += 1 + else: + break + return problemIndexList + + + ############################################################################## + # Get Item + def __getitem__(self, indexTuple): + indices = indexTuple[0] # in analysis order + solutionIdx = indexTuple[1] + serial = self.indicesToSerial(solutionIdx, indices) + return self.data[serial] + + + ############################################################################## + # Set Item + def __setitem__(self, indexTuple, value): + indices = indexTuple[0] # in analysis order + solutionIdx = indexTuple[1] + serial = self.indicesToSerial(solutionIdx, indices ) + self.data[serial] = value + + + ############################################################################## + # Indices -> Serial + def indicesToSerial(self, solutionIdx, indices ): + serial = 0 + stride = 1 + serial += solutionIdx * stride + stride *= self.numSolutions + for i in range(0, self.numIndices): + serial += indices[i] * stride + stride *= self.numProblemSizes[i] + return serial + + + +def generateLogic(config, benchmarkDataPath, libraryLogicPath): + print2("# LibraryLogic config: %s" % config) + print2("# DefaultAnalysisParameters: " % defaultAnalysisParameters) + + setWorkingPath(libraryLogicPath) + + # Assign Defaults + analysisParameters = {} + for parameter in defaultAnalysisParameters: + assignParameterWithDefault(analysisParameters, parameter, config, \ + defaultAnalysisParameters) + + print1("") + print1(HR) + currentTime = time.time() + elapsedTime = currentTime - startTime + print1("# Analysing data in %s - %.3fs" % (globalParameters["BenchmarkDataPath"], elapsedTime) ) + for parameter in analysisParameters: + print2("# %s: %s" % (parameter, analysisParameters[parameter])) + print1(HR) + print1("") + + ############################################################################## + # Determine Which Problem Types + ############################################################################## + problemTypes = {} + if not os.path.exists(benchmarkDataPath): + printExit("Path doesn't exist: %s" % benchmarkDataPath) + fileNames = os.listdir(benchmarkDataPath) + fileNames = sorted(fileNames) + for fileName in fileNames: + if os.path.splitext(fileName)[1] == ".csv": + fileBase = os.path.splitext( \ + os.path.join(benchmarkDataPath, \ + fileName))[0] + dataFileName = fileBase + ".csv" + solutionsFileName = fileBase + ".yaml" + selectionFileName = fileBase + ".gsp" + if not os.path.exists(dataFileName): + printExit("%s doesn't exist for %s" % (dataFileName, fileBase) ) + if not os.path.exists(solutionsFileName): + printExit("%s doesn't exist for %s" % (solutionsFileName, fileBase) ) + (problemSizes, solutions) = LibraryIO.parseSolutionsFile(solutionsFileName) + if len(solutions) == 0: + printExit("%s doesn't contains any solutions." % (solutionsFileName) ) + problemType = solutions[0]["ProblemType"] + if problemType not in problemTypes: + problemTypes[problemType] = [] + problemTypes[problemType].append( (problemSizes, \ + dataFileName, solutionsFileName, selectionFileName, solutions) ) + + for problemType in problemTypes: + logicTuple = analyzeProblemType(problemType, problemTypes[problemType], analysisParameters) + + filename = os.path.join(globalParameters["WorkingPath"], \ + "{}_{}".format(analysisParameters["ScheduleName"], str(problemType) + ".yaml")) + + print2("# writing library logic YAML {}".format(filename)) + data = LibraryIO.createLibraryLogic(analysisParameters["ScheduleName"], \ + analysisParameters["ArchitectureName"], analysisParameters["DeviceNames"], logicTuple) + + LibraryIO.writeYAML(filename, data, explicit_start=False, explicit_end=False) + + currentTime = time.time() + elapsedTime = currentTime - startTime + print1("%s\n# Finish Analysing data to %s in %.3fs\n%s" % (HR, os.path.split(libraryLogicPath)[0], elapsedTime, HR) ) + popWorkingPath() + + +################################################################################ +################################################################################ +### +### Main +### +################################################################################ +################################################################################ +def main( config ): + + benchmarkDataPath = os.path.join(globalParameters["WorkingPath"], \ + globalParameters["BenchmarkDataPath"]) + + libraryLogicPath = os.path.join(globalParameters["WorkingPath"], \ + globalParameters["LibraryLogicPath"]) + + generateLogic(config, benchmarkDataPath, libraryLogicPath) diff --git a/tensilelite/Tensile/Parallel.py b/tensilelite/Tensile/Parallel.py new file mode 100644 index 0000000000..7dee9de955 --- /dev/null +++ b/tensilelite/Tensile/Parallel.py @@ -0,0 +1,135 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +import os +import itertools +import sys + + +def CPUThreadCount(enable=True): + from .Common import globalParameters + if not enable or globalParameters["CpuThreads"] == 0: + return 0 + else: + if os.name == "nt": + cpu_count = os.cpu_count() + else: + cpu_count = len(os.sched_getaffinity(0)) + cpuThreads = globalParameters["CpuThreads"] + if cpuThreads < 0: + return cpu_count*abs(cpuThreads) + return min(cpu_count, cpuThreads) + +def starmap_apply(item): + func, item = item + return func(*item) + +def apply_print_exception(item, *args): + #print(item, args) + try: + if len(args) > 0: + func = item + args = args[0] + return func(*args) + else: + func, item = item + return func(item) + except Exception: + import traceback + traceback.print_exc() + raise + finally: + sys.stdout.flush() + sys.stderr.flush() + +def OverwriteGlobalParameters(newGlobalParameters): + from . import Common + Common.globalParameters.clear() + Common.globalParameters.update(newGlobalParameters) + +def ProcessingPool(enable=True, maxTasksPerChild=None): + import multiprocessing + import multiprocessing.dummy + + threadCount = CPUThreadCount() + + if (not enable) or threadCount <= 1: + return multiprocessing.dummy.Pool(1) + + if multiprocessing.get_start_method() == "spawn": + from . import Common + return multiprocessing.Pool(threadCount, initializer=OverwriteGlobalParameters, maxtasksperchild=maxTasksPerChild, initargs=(Common.globalParameters,)) + else: + return multiprocessing.Pool(threadCount, maxtasksperchild=maxTasksPerChild) + +def ParallelMap(function, objects, message="", enable=True, method=None, maxTasksPerChild=None): + """ + Generally equivalent to list(map(function, objects)), possibly executing in parallel. + + message: A message describing the operation to be performed. + enable: May be set to false to disable parallelism. + method: A function which can fetch the mapping function from a processing pool object. + Leave blank to use .map(), other possiblities: + - `lambda x: x.starmap` - useful if `function` takes multiple parameters. + - `lambda x: x.imap` - lazy evaluation + - `lambda x: x.imap_unordered` - lazy evaluation, does not preserve order of return value. + """ + from .Common import globalParameters + threadCount = CPUThreadCount(enable) + pool = ProcessingPool(enable, maxTasksPerChild) + + if threadCount <= 1 and globalParameters["ShowProgressBar"]: + # Provide a progress bar for single-threaded operation. + # This works for method=None, and for starmap. + mapFunc = map + if method is not None: + # itertools provides starmap which can fill in for pool.starmap. It provides imap on Python 2.7. + # If this works, we will use it, otherwise we will fallback to the "dummy" pool for single threaded + # operation. + try: + mapFunc = method(itertools) + except NameError: + mapFunc = None + + if mapFunc is not None: + from . import Utils + return list(mapFunc(function, Utils.tqdm(objects, message))) + + mapFunc = pool.map + if method: mapFunc = method(pool) + + objects = zip(itertools.repeat(function), objects) + function = apply_print_exception + + countMessage = "" + try: + countMessage = " for {} tasks".format(len(objects)) + except TypeError: pass + + if message != "": message += ": " + + print("{0}Launching {1} threads{2}...".format(message, threadCount, countMessage)) + sys.stdout.flush() + rv = mapFunc(function, objects) + print("{0}Done.".format(message)) + sys.stdout.flush() + pool.close() + return rv diff --git a/tensilelite/Tensile/Properties.py b/tensilelite/Tensile/Properties.py new file mode 100644 index 0000000000..c0320e0a31 --- /dev/null +++ b/tensilelite/Tensile/Properties.py @@ -0,0 +1,89 @@ +################################################################################ +# Copyright 2019-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Utils import hash_objs, state + +class Property: + def __init__(self, tag=None, index=None, value=None): + self._tag = tag + self._index = index + self._value = value + + @property + def tag(self): return self._tag + @property + def index(self): return self._index + @property + def value(self): return self._value + + def state(self): + rv = {'type': self.tag} + if self.index is not None: rv['index'] = state(self.index) + if self.value is not None: rv['value'] = state(self.value) + return rv + + def __eq__(self, other): + return self.__class__ == other.__class__ and \ + self.tag == other.tag and \ + self.value == other.value and \ + self.index == other.index + + def __hash__(self): + #return hash(self.tag) ^ hash(self.value) ^ hash(self.index) + return hash_objs(self.tag, self.value, self.index) + + +class Predicate(Property): + @classmethod + def FromOriginalState(cls, d, morePreds=None): + if morePreds is None: morePreds = [] + predicates = [p for p in map(cls.FromOriginalKeyPair, d.items()) if p is not None] + morePreds + return cls.And(predicates) + + @classmethod + def And(cls, predicates): + predicates = tuple(predicates) + if len(predicates) == 0: + return cls('TruePred') + if len(predicates) == 1: + return predicates[0] + return cls('And', value=predicates) + + @classmethod + def Or(cls, predicates): + predicates = tuple(predicates) + if len(predicates) == 0: + return cls('TruePred') + if len(predicates) == 1: + return predicates[0] + return cls('Or', value=predicates) + + def __lt__(self, other): + # Ensure TruePred appears last. + if other.tag == 'TruePred': + if self.tag == 'TruePred': + return False + return True + if self.tag == 'TruePred': + return False + + # If neither is a TruePred then just use the default comparison. + return (self.tag, self.index, self.value) < (other.tag, other.index, other.value) diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py new file mode 100644 index 0000000000..dd821e5b0a --- /dev/null +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -0,0 +1,397 @@ +################################################################################ +# Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +import itertools + +from . import Properties +from . import Hardware +from . import Common +from . import Contractions +from .SolutionStructs import Solution as OriginalSolution +from .Utils import state + +class SingleSolutionLibrary: + Tag = 'Single' + + def __init__(self, solution): + self.solution = solution + + @property + def tag(self): + return self.__class__.Tag + + def state(self): + return {'type': self.tag, 'index': self.solution.index} + + def remapSolutionIndices(self,indexMap): + pass + +class GranularitySelectionLibrary: + Tag = 'GranularitySelection' + StateKeys = [('type', 'tag'), 'indices', 'exact'] + + @classmethod + def FromOriginalState(cls, d, indices): + origTable = d[1] + #indices = d[9]['TileSelectionIndices'] + # entry = {'key': key, 'value': value, 'speed': row[1][1]} + entries = [] + for row in origTable: + try: + index = row[1][0] + key = list(row[0][0:4]) + entry = {'key': key, 'value': index } + entries.append(entry) + except KeyError: + pass + + + return cls(indices, entries) + + @property + def tag(self): + return self.__class__.Tag + + def merge(self, other): + assert self.__class__ == other.__class__ + self.indices = list(set().union(self.indices, other.indices)) + self.exact = list(set().union(self.exact, other.exact)) + + def __init__(self, indices, exact): + self.indices = indices + self.exact = exact + + def remapSolutionIndices(self,indexMap): + for i in range(0, len(self.indices)): + index = self.indices[i] + if index in indexMap: + self.indices[i] = indexMap[index] + +class MatchingLibrary: + Tag = 'Matching' + StateKeys = [('type', 'tag'), 'properties', 'table', 'distance'] + + @classmethod + def FromOriginalState(cls, d, solutions, distance='Euclidean'): + indices = d[0] + origTable = d[1] + + propertyKeys = { + 2:lambda: Properties.Property('FreeSizeA', index=0), + 3:lambda: Properties.Property('FreeSizeB', index=0), + #0:lambda: Properties.Property('BatchSize', index=0), + 1:lambda: Properties.Property('BoundSize', index=0) + } + + properties = list([propertyKeys[i]() for i in indices if i in propertyKeys]) + keyOrder = [i for i,j in enumerate(indices) if j in propertyKeys] + + table = [] + + for row in origTable: + try: + index = row[1][0] + value = SingleSolutionLibrary(solutions[index]) + key = list([row[0][i] for i in keyOrder]) + #key = list(row[0][0:len(properties)]) + entry = {'key': key, 'value': value, 'speed': row[1][1]} + table.append(entry) + except KeyError: + pass + + table.sort(key=lambda r: r['key']) + + return cls(properties, table, distance) + + @property + def tag(self): + return self.__class__.Tag + + def merge(self, other): + assert self.__class__ == other.__class__ \ + and self.properties == other.properties \ + and self.distance == other.distance + + self.table += other.table + + self.table.sort(key=lambda r: r['key']) + + def remapSolutionIndices(self,indexMap): + pass + + def __init__(self, properties, table, distance): + self.properties = properties + self.table = table + self.distance = distance + +class ProblemMapLibrary: + Tag = 'ProblemMap' + StateKeys = [('type', 'tag'), ('property', 'mappingProperty'), ('map', 'mapping')] + + def __init__(self, mappingProperty=None, mapping=None): + self.mappingProperty = mappingProperty + self.mapping = mapping + + @property + def tag(self): + return self.__class__.Tag + + def merge(self, other): + assert self.__class__ == other.__class__ and self.tag == other.tag and self.mappingProperty == other.mappingProperty + + for key,value in list(other.mapping.items()): + if key in self.mapping: + self.mapping[key].merge(value) + else: + self.mapping[key] = value + + def remapSolutionIndices(self,indexMap): + for key,value in list(self.mapping.items()): + value.remapSolutionIndices(indexMap) + +class PredicateLibrary: + StateKeys = [('type', 'tag'), 'rows'] + + def __init__(self, tag=None, rows=None): + self.tag = tag + if rows is None: rows = [] + self.rows = rows + + def merge(self, other): + assert self.__class__ == other.__class__ and self.tag == other.tag + + rowdict = {r['predicate']: i for i,r in enumerate(self.rows)} + + for row in other.rows: + if row['predicate'] in rowdict: + myRownum = rowdict[row['predicate']] + self.rows[myRownum]['library'].merge(row['library']) + else: + self.rows.append(row) + + # Sort to ensure consistent fallback logic. + self.rows.sort(key=lambda x: x['predicate']) + + def remapSolutionIndices(self,indexMap): + for row in self.rows: + row['library'].remapSolutionIndices(indexMap) + + +class MasterSolutionLibrary: + StateKeys = ['solutions', 'library'] + + @classmethod + def FixSolutionIndices(cls, solutions): + # fix missing and duplicate solution indices. + try: + maxSolutionIdx = max([s.index for s in solutions if s.index is not None]) + except ValueError: + maxSolutionIdx = -1 + + solutionsSoFar = set() + for solution in solutions: + if solution.index is None or solution.index in solutionsSoFar: + maxSolutionIdx += 1 + solution.index = maxSolutionIdx + else: + solutionsSoFar.add(solution.index) + @classmethod + def FromOriginalState(cls, d, origSolutions, solutionClass=Contractions.Solution, libraryOrder = None): + if libraryOrder is None: + libraryOrder = ['Hardware', 'OperationIdentifier', 'PerformanceMetric', 'Fp16AltImpl', 'Predicates', 'Matching'] + + deviceSection = d[1:4] + origProblemType = d[4] + #origSolutions = d[5] + origLibrary = d[6:8] + + if len(d) > 9 and d[9]: + # It's a Granularity library + assert libraryOrder[-1] == 'Matching' + libraryOrder[-1] = 'Granularity' + + perfMetric = 'DeviceEfficiency' + if len(d) > 10 and d[10]: + perfMetric = d[10] + + fp16AltImpl = False + if len(d) > 11 and d[11]: + fp16AltImpl = True + + matching = 'Euclidean' + if len(d) > 12 and d[12]: + matching = d[12] + + problemType = Contractions.ProblemType.FromOriginalState(origProblemType) + + allSolutions = [solutionClass.FromSolutionStruct(s, deviceSection) for s in origSolutions] + cls.FixSolutionIndices(allSolutions) + + solutions = {s.index: s for s in allSolutions} + + for libName in reversed(libraryOrder): + if libName == 'Matching': + if matching == 'Equality': + predicate = Properties.Predicate(tag='EqualityMatching') + else: + predicate = Properties.Predicate(tag='TruePred') + + matchingLib = MatchingLibrary.FromOriginalState( \ + origLibrary, allSolutions, matching) + library = PredicateLibrary(tag='Problem') + library.rows.append({'predicate': predicate, 'library': matchingLib}) + + elif libName == 'Granularity': + selectionIndices = d[9]['TileSelectionIndices'] + library = GranularitySelectionLibrary.FromOriginalState(origLibrary, selectionIndices) + + elif libName == 'Hardware': + + if isinstance(deviceSection[1], dict): + architectureProps = deviceSection[1] + assert 'Architecture' in architectureProps, 'Invalid device section [1]' + assert 'CUCount' in architectureProps, 'Invalid device section [1]' + devicePart = architectureProps['Architecture'] + cuCount = architectureProps['CUCount'] + else: + devicePart = deviceSection[1] + cuCount = None + + newLib = PredicateLibrary(tag='Hardware') + if devicePart == 'fallback': + pred = Hardware.HardwarePredicate('TruePred') + else: + pred = Hardware.HardwarePredicate.FromHardware(Common.gfxArch(devicePart), cuCount) + + newLib.rows.append({'predicate': pred, 'library': library}) + library = newLib + + elif libName == 'Predicates': + predicates = problemType.predicates(includeBatch=True, includeType=True) + predicate = Contractions.ProblemPredicate.And(predicates) + + newLib = PredicateLibrary(tag='Problem') + newLib.rows.append({'predicate': predicate, 'library': library}) + library = newLib + + elif libName == 'OperationIdentifier': + operationID = problemType.operationIdentifier + prop = Properties.Property('OperationIdentifier') + mapping = {operationID: library} + + newLib = ProblemMapLibrary(prop, mapping) + library = newLib + + elif libName == 'PerformanceMetric': + if perfMetric != 'DeviceEfficiency': + predicate = Properties.Predicate(tag=perfMetric) + else: + predicate = Properties.Predicate(tag='TruePred') + newLib = PredicateLibrary(tag='Problem') + newLib.rows.append({'predicate': predicate, 'library': library}) + library = newLib + + elif libName == 'Fp16AltImpl': + if fp16AltImpl: + predicate = Properties.Predicate(tag='Fp16AltImpl') + else: + predicate = Properties.Predicate(tag='TruePred') + newLib = PredicateLibrary(tag='Problem') + newLib.rows.append({'predicate': predicate, 'library': library}) + library = newLib + + else: + raise ValueError('Unknown value ' + libName) + + rv = cls(solutions, library) + return rv + + @classmethod + def BenchmarkingLibrary(cls, solutions): + solutionObjs = list([Contractions.Solution.FromOriginalState(s._state) for s in solutions]) + cls.FixSolutionIndices(solutionObjs) + + predRows = list([{'predicate': s.problemPredicate, 'library': SingleSolutionLibrary(s)} for s in solutionObjs]) + library = PredicateLibrary(tag='Problem', rows=predRows) + + solutionMap = {s.index: s for s in solutionObjs} + + return cls(solutionMap, library) + + def __init__(self, solutions, library, version=None): + self.solutions = solutions + self.library = library + self.version = version + + def state(self): + rv = {'solutions': state(iter(list(self.solutions.values()))), + 'library': state(self.library)} + + if self.version is not None: + rv['version'] = self.version + return rv + + def applyNaming(self, naming=None): + if naming is None: + #allSolutions = itertools.chain(iter(list(self.solutions.values())), iter(list(self.sourceSolutions.values()))) + kernels = list(itertools.chain(*[s.originalSolution.getKernels() for s in self.solutions.values()])) + naming = OriginalSolution.getMinNaming(kernels) + + for s in list(self.solutions.values()): + s.name = OriginalSolution.getNameMin(s.originalSolution.getKernels()[0], naming) + + def remapSolutionIndicesStartingFrom(self, curIndex): + reIndexMap = {} + solutionCopy = self.solutions + self.solutions = dict() + for k,s in solutionCopy.items(): + reIndexMap[s.index] = curIndex + s.index = curIndex + self.solutions[curIndex] = s + curIndex += 1 + + self.library.remapSolutionIndices(reIndexMap) + + def merge(self, other, startIndex=0): + assert self.__class__ == other.__class__ + + curIndex = max(startIndex, max(self.solutions.keys()) + 1) + + reIndexMap = {} + for k,s in other.solutions.items(): + reIndexMap[s.index] = curIndex + s.index = curIndex + self.solutions[curIndex] = s + curIndex += 1 + + other.library.remapSolutionIndices(reIndexMap) + + self.library.merge(other.library) + + return curIndex #Next unused index + + @property + def cpp_base_class(self): + return 'SolutionLibrary' + + @property + def cpp_class(self): + return 'MasterSolutionLibrary' diff --git a/tensilelite/Tensile/SolutionSelectionLibrary.py b/tensilelite/Tensile/SolutionSelectionLibrary.py new file mode 100644 index 0000000000..bc88e9427c --- /dev/null +++ b/tensilelite/Tensile/SolutionSelectionLibrary.py @@ -0,0 +1,162 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .SolutionStructs import Solution + +import csv + +def getSummationKeys(header): + keys=[] + for i in range(7, len(header)): + keystr = header[i].split("=")[1].strip() + key = int(keystr) + keys.append(key) + return keys + +def makeKey(row): + key=row[3] + for i in range(4, 7): + key += "_%s" % row[i].strip() + return key + +def getSolutionBaseKey (solution): + + macroTile0 = solution["MacroTile0"] + macroTile1 = solution["MacroTile1"] + globalSplitU = solution["GlobalSplitU"] + localSplitU = solution["WorkGroup"][2] + + key = "%s_%s_%s_%s" % (macroTile0, macroTile1, localSplitU, globalSplitU) + + return key + +def updateIfGT(theDictionary, theKey, theValue): + if not theKey in theDictionary: + theDictionary[theKey] = theValue + else: + theOldValue = theDictionary[theKey] + if theValue > theOldValue: + theDictionary[theKey] = theValue + + +def updateValidSolutions(validSolutions, analyzerSolutions, solutionMinNaming): + solutionsStartIndex = len(analyzerSolutions) + validSelectionSolutionsIncluded = [] + validSelectionSolutionsRemainder = [] + selectionSolutionsIds = set([]) + for validSelectionSolution in validSolutions: + (validSolution, validSolutionInfo) = validSelectionSolution + if validSolution in analyzerSolutions: + validExactSolutionIndex = analyzerSolutions.index(validSolution) + selectionSolutionsIds.add(validExactSolutionIndex) + validExactSolution = analyzerSolutions[validExactSolutionIndex] + validSelectionSolutionsIncluded.append((validExactSolution, validSolutionInfo)) + else: + validSelectionSolutionsRemainder.append(validSelectionSolution) + + selectionSolutions = [] + for i in range(0 ,len(validSelectionSolutionsIncluded)): + validSelectionSolution = validSelectionSolutionsIncluded[i] + (validSolution, validSolutionInfo) = validSelectionSolution + validSolution["Ideals"] = validSolutionInfo + analyzerSolutions.append(validSolution) + + solutionsStartIndex = len(analyzerSolutions) + + for i in range(0, len(validSelectionSolutionsRemainder)): + validSelectionSolution = validSelectionSolutionsRemainder[i] + (validSolution, validSolutionInfo) = validSelectionSolution + selectionSolutionIndex = solutionsStartIndex + i + selectionSolutionsIds.add(selectionSolutionIndex) + validSolution["SolutionNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming) + validSolution["Ideals"] = validSolutionInfo + selectionSolutions.append(validSolution) + + selectionSolutionsIdsList = list(selectionSolutionsIds) + + return selectionSolutionsIdsList + + +def analyzeSolutionSelection(problemType, selectionFileNameList, numSolutionsPerGroup, solutionGroupMap, solutionsList): + + performanceMap = {} + solutionsHash = {} + + totalIndices = problemType["TotalIndices"] + summationIndex = totalIndices + numIndices = totalIndices + problemType["NumIndicesLD"] + problemSizeStartIdx = 1 + totalSizeIdx = problemSizeStartIdx + numIndices + solutionStartIdx = totalSizeIdx + 1 + for fileIdx in range(0, len(selectionFileNameList)): + solutions = solutionsList[fileIdx] + selectionFileName = selectionFileNameList[fileIdx] + numSolutions = numSolutionsPerGroup[fileIdx] + rowLength = solutionStartIdx + numSolutions + solutionBaseKeys = [] + + for solution in solutions: + baseKey = getSolutionBaseKey(solution) + solutionBaseKeys.append(baseKey) + + selectionfFile = open(selectionFileName, "r") + csvFile = csv.reader(selectionfFile) + + firstRow = 0 + for row in csvFile: + if firstRow == 0: + firstRow += 1 + else: + sumationId = row[summationIndex].strip() + + solutionIndex = 0 + for i in range(solutionStartIdx, rowLength): + baseKey = solutionBaseKeys[solutionIndex] + key = "%s_%s" % (baseKey, sumationId) + solution = solutions[solutionIndex] + solutionIndex += 1 + value = float(row[i]) + if not solution in solutionsHash: + dataMap = {} + solutionsHash[solution] = dataMap + + updateIfGT(solutionsHash[solution], sumationId, value) + if not key in performanceMap: + performanceMap[key] = (solution, value) + else: + _,valueOld = performanceMap[key] + if value > valueOld: + performanceMap[key] = (solution, value) + + + validSolutions = [] + validSolutionSet = set([]) + + for key in performanceMap: + solution, _ = performanceMap[key] + validSolutionSet.add(solution) + + for validSolution in validSolutionSet: + dataMap = solutionsHash[validSolution] + validSolutions.append((validSolution,dataMap)) + + return validSolutions + diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py new file mode 100644 index 0000000000..5c9f582422 --- /dev/null +++ b/tensilelite/Tensile/SolutionStructs.py @@ -0,0 +1,3198 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Common import assignParameterWithDefault, \ + defaultProblemType, defaultSolution, \ + globalParameters, \ + print2, printExit, printWarning, \ + validMFMA, validParameters, \ + validGEMMTypes, HPATypes +from .TensileInstructions import DataType, roundUpToNearestMultiple + +from .KernelWriterBetaOnly import KernelWriterBetaOnly +from .KernelWriterConversion import KernelWriterConversion +from .KernelWriterActivationEnumHeader import KernelWriterActivationEnumHeader +from .KernelWriterActivationFunction import KernelWriterActivationFunction +from .KernelWriterActivationOnly import KernelWriterActivationOnly + +from .Activation import ActivationType + +from .CustomKernels import isCustomKernelConfig + +from collections import namedtuple,OrderedDict +from collections.abc import Mapping +from copy import deepcopy +from enum import Enum + +import collections +import math +import operator +import sys + +######################################## +# Print a reject message : +def reject(state, *args): + if state and "NoReject" in state and state["NoReject"]: + return + + if globalParameters["PrintSolutionRejectionReason"]: + sys.stdout.write("\nreject: ") + for a in args: + print(a) + #traceback.print_stack(None, 2) + solutionIndex = state["SolutionIndex"] if (state != None and "SolutionIndex" in state) else -1 + if solutionIndex != -1: + # If we have valid solutionIndex, this means we are during TensileCreateLibrary stage + # In this stage, all solutions in the logic should be valid + # So if any rejection happens, print the warning for further check + # This will be done only when --global-parameters=PrintSolutionRejectionReason=True + solutionNameMin = state["SolutionNameMin"] if ("SolutionNameMin" in state) else None + # if we don't have SolutionNameMin, we simply use the problemTypeName + solutionNameMin = str(state["ProblemType"]) if (solutionNameMin == None) else solutionNameMin + print("!! Warning: Any rejection of a LibraryLogic is not expected, please check. \ + SolutionIndex: %d (or SolutionName/ProblemType: %s)"%(solutionIndex, solutionNameMin)) + if state != None: + state["Valid"] = False + +# print a labled variable +def pvar(state, field): + return field + "=" + str(state[field]) + +def roundupRatio(dividend, divisor): + return int(math.ceil(float(dividend) / float(divisor))) + +class Fbs(Enum): + Free=0 # Expect to be free dimension + Batch=1 # Expect to be batch dimension + Sum=2 # Expect to be summation dimension + +################################################################################ +# ProblemType +# name of solution should begin with name of problemType, and arguments can be listed out explicitly +class ProblemType(Mapping): + ######################################## + def __init__(self, config): + self.state = {} + + for key in defaultProblemType: + assignParameterWithDefault(self.state, key, config, defaultProblemType) + + # adjusting all data types + if "DataType" in config: + self["DataType"] = DataType(config["DataType"]) + else: + printExit("NO data type specified") + self["DataType"] = DataType(0) + + if "DestDataType" in config: + self["DestDataType"] = DataType(config["DestDataType"]) + else: + if "DataType" in config: + self["DestDataType"] = DataType(config["DataType"]) + else: + printExit("NO dest data type or data type specified") + self["DataType"] = DataType(0) + + if "ComputeDataType" in config: + self["ComputeDataType"] = DataType(config["ComputeDataType"]) + else: + if "DestDataType" in config: + self["ComputeDataType"] = DataType(config["DestDataType"]) + else: + if "DataType" in config: + self["ComputeDataType"] = DataType(config["DataType"]) + else: + printExit("NO compute data type, or dest data type, or data type specified") + self["DataType"] = DataType(0) + + # Modifying ComputeDataType for HHH+HPA: if (HHH+HPA), convert it to HHS_BH by setting ComputeDataType to S. + if self["ComputeDataType"].isHalf() and self["DataType"].isHalf() and self["HighPrecisionAccumulate"]: + printWarning("Inconsistent DataTypes: DataType == f16, DestType == f16, ComputeDataType == f16, but HPA == True (HHH+HPA, no such a type); Converting HHH+HPA to HHS_BH by setting compute data type to f32.") + self["ComputeDataType"] = DataType('s') + + # Modifying ComputeDataType for BBB+HPA: if (BBB+HPA), convert it to BBS_BH by setting ComputeDataType to S. + if self["ComputeDataType"].isBFloat16() and self["DataType"].isBFloat16() and self["HighPrecisionAccumulate"]: + printWarning("Inconsistent DataTypes: DataType == bf16, DestType == bf16, ComputeDataType == bf16, but HPA == True (BBB+HPA, no such a type); Converting BBB+HPA to BBS_BH by setting compute data type to f32.") + self["ComputeDataType"] = DataType('s') + + # Modifying ComputeDataType for I8I8I_BH: if (I8I8I8+HPA), convert it to I8I8I_BH by setting ComputeDataType to i. + if self["ComputeDataType"].isInt8() and DataType(config["DataType"]).isInt8() and self["HighPrecisionAccumulate"]: + print2("DataType == i8 and HPA == True; setting compute data type to int32") + self["ComputeDataType"] = DataType('i') + + if self["OperationType"] == "GEMM": + self.checkIfSupportedGEMMType() + self.initGEMM() + else: + printExit("Unsupported OperationType = %s" % self["OperationType"]) + + self.state["AssignedDerivedParameters"] = False + ProblemType.assignDerivedParameters(self.state) + + for tc in ('A', 'B'): + for sc in self["SetConstStride%s"%tc] : + (anchorDim, stride) = sc[:2] + if anchorDim not in self.state["IndexAssignments%s"%tc]: + printExit("SetConstStride%s=%s anchorDim=%u is not in IndexAssignments%s"%(tc, sc, anchorDim, tc)) + + # Bias + # If compute data type is not equal to dest data type, tensile will run conversion kernel. + # In this case we don't need to apply bias in beta only kernel. + if "UseBias" in config: + if self["ComputeDataType"] != self["DestDataType"]: + self["BetaOnlyUseBias"] = False + else: + self["BetaOnlyUseBias"] = self["UseBias"] + else: + self["BetaOnlyUseBias"] = False + + # Activation + if "Activation" in config: + typeStr = 'all' if config["Activation"] else 'none' + self["ActivationType"] = ActivationType(typeStr) + else: + self["ActivationType"] = ActivationType('none') + if "ActivationHPA" in config: + self["ActivationHPA"] = config["ActivationHPA"] + else: + self["ActivationHPA"] = False + self["ActivationComputeDataType"] = self["ComputeDataType"] if self["ActivationHPA"] else \ + self["DestDataType"] + + if self["ActivationType"] != 'none': + if ((not self["HighPrecisionAccumulate"]) and self["ActivationHPA"]): + printExit("Must enable HighPrecisionAccumulate to use ActivationHPA.") + if ((self["HighPrecisionAccumulate"]) and (not self["ActivationHPA"]) and \ + (self["DestDataType"].isBFloat16() or self["DestDataType"].isInt8())): + printWarning("%s only supports ActivationHPA = True if HighPrecisionAccumulate = True. \ + ActivationHPA will be set to True automatically."%str(self["DestDataType"])) + self["ActivationHPA"] = True + if self["ActivationHPA"] and (self["DestDataType"].isSingle() or self["DestDataType"].isDouble()): + printWarning("Single and Double does not support ActivationHPA. ActivationHPA will be set to False automatically.") + self["ActivationHPA"] = False + + + ################################################################################ + # Function checkIfSupportedGEMMType: + # Assures 3 data-types are valid, supported and well-assigned + # See the discussion on Common.py for validGEMMTypes + ################################################################################ + def checkIfSupportedGEMMType(self): + inType = self["DataType"] + outType = self["DestDataType"] + computeType = self["ComputeDataType"] + + gemmType = ( inType.toChar(), outType.toChar(), computeType.toChar() ) + if gemmType not in validGEMMTypes: + printExit("This typed-GEMM (Ti, To, Tc) = (%s, %s, %s) is not supported yet."%(gemmType[0],gemmType[1],gemmType[2])) + + ######################################## + def initGEMM(self): + sumIdx = 3 if self["Batched"] else 2 + self["IndexAssignmentsA"] = [0, sumIdx] # N + self["IndexAssignmentsB"] = [sumIdx, 1] # N + if self["TransposeA"]: + self["IndexAssignmentsA"] = [sumIdx, 0] # T + if self["TransposeB"]: + self["IndexAssignmentsB"] = [1, sumIdx] # T + if self["Batched"]: + self["IndexAssignmentsA"].append(2) + self["IndexAssignmentsB"].append(2) + self["NumIndicesC"] = 3 + else: + self["NumIndicesC"] = 2 + + self["NumIndicesLD"] = 4 + self["IndexAssignmentsLD"][0] = self["NumIndicesC"] + 1 + for i in range(1, len(self["IndexAssignmentsLD"])): + self["IndexAssignmentsLD"][i] = self["IndexAssignmentsLD"][i-1] + 1 + + ######################################## + def isGEMM(self): + return self.operationType == 0 + + ######################################## + # determine d0, d1, dU + @staticmethod + def assignDerivedParameters(state): + if "AssignedDerivedParameters" in state: + if state["AssignedDerivedParameters"]: + return + state["AssignedDerivedParameters"] = False + + state["TotalIndices"] = max(max(state["IndexAssignmentsA"])+1, \ + max(state["IndexAssignmentsB"])+1) + + # determine num free, batch + state["IndicesFree"] = [] + state["IndicesBatch"] = [] + state["IndicesSummation"] = [] + + for i in range(0, state["NumIndicesC"]): + inA = i in state["IndexAssignmentsA"] + inB = i in state["IndexAssignmentsB"] + if inA and inB: + state["IndicesBatch"].append(i) + + elif inA or inB: + state["IndicesFree"].append(i) + else: + printExit("invalid index %u (inC but not (inA or inB))" % i) + + # determine num summation + for i in range(state["NumIndicesC"], state["TotalIndices"]): + inA = i in state["IndexAssignmentsA"] + inB = i in state["IndexAssignmentsB"] + if inA and inB: + state["IndicesSummation"].append(i) + else: + printExit("invalid index %u (expected summation but not (inA and inB))" % i) + # print index assignments + if globalParameters["PrintIndexAssignments"]: + print("IndicesFree: %s" % state["IndicesFree"]) + print("IndicesBatch: %s" % state["IndicesBatch"]) + print("IndicesSum: %s" % state["IndicesSummation"]) + print("IndexAssignmentsA: %s" % state["IndexAssignmentsA"]) + print("IndexAssignmentsB: %s" % state["IndexAssignmentsB"]) + print("NumIndicesC: %s" % state["NumIndicesC"]) + + for k in ('IndexAssignmentsA','IndexAssignmentsB'): + if len(state[k]) != len(set(state[k])): + printExit("duplicate index in %s=%s"% (k,state[k])) + + state["NumIndicesFree"] = len(state["IndicesFree"]) + state["NumIndicesBatch"] = len(state["IndicesBatch"]) + state["NumIndicesSummation"] = len(state["IndicesSummation"]) + if not state["AllowNoFreeDims"] and state["NumIndicesFree"] < 2 : + printExit("Tensile requires >= 2 free indices or set AllowNoFreeDims; FreeIndices=%s."% state["IndicesFree"]) + + # by default, unroll index will be the last/inner summation index + state["IndexUnroll"] = state["IndicesSummation"][len(state["IndicesSummation"])-1] + for i in range(0, len(state["IndexAssignmentsA"])): + if state["IndexAssignmentsA"][i] == state["IndexUnroll"]: + state["IndexUnrollA"] = i + break + for i in range(0, len(state["IndexAssignmentsB"])): + if state["IndexAssignmentsB"][i] == state["IndexUnroll"]: + state["IndexUnrollB"] = i + break + #print2("IndexUnrollA: %u" % state["IndexUnrollA"]) + #print2("IndexUnrollB: %u" % state["IndexUnrollB"]) + + # assign d0, d1 + if state["AllowNoFreeDims"]: + dimList = state["IndicesFree"] + state["IndicesBatch"] + else: + dimList = state["IndicesFree"] + state["Index01A"] = [i for i in state["IndexAssignmentsA"] if i in dimList][0] + state["Index01B"] = [i for i in state["IndexAssignmentsB"] if i in dimList][0] + #print2("Index01A: %u" % state["Index01A"]) + #print2("Index01B: %u" % state["Index01B"]) + # Store code is optimized for 0 as the fastest-moving in memory + # whichever has lower stride in C (lower value), is 0, other is 1 + if state["Index01A"] < state["Index01B"]: + state["Index0"] = state["Index01A"] + state["Index1"] = state["Index01B"] + state["Tensor0"] = 0 + state["Tensor1"] = 1 + state["TileA"] = 0 + state["TileB"] = 1 + else: + state["Index0"] = state["Index01B"] + state["Index1"] = state["Index01A"] + state["Tensor0"] = 1 + state["Tensor1"] = 0 + state["TileA"] = 1 + state["TileB"] = 0 + + # generalize transpose + strideIdxA = state["IndexAssignmentsA"].index(state["Index01A"]) + strideIdxB = state["IndexAssignmentsB"].index(state["Index01B"]) + unrollIdxA = state["IndexAssignmentsA"].index(state["IndexUnroll"]) + unrollIdxB = state["IndexAssignmentsB"].index(state["IndexUnroll"]) + state["TLUA"] = strideIdxA < unrollIdxA + state["TLUB"] = strideIdxB < unrollIdxB + #state["TLUB"] = True # hack + + if globalParameters["PrintIndexAssignments"]: + print("TLUA: %s (stridePosA(%d) self.indicesSized[i-1][3]: + #/ reset prior index + currentSizedIndexSizes[i-1] = self.indicesSized[i-1][0] + currentSizedIndexIncrements[i-1] = self.indicesSized[i-1][1] + # increment next index + if i >= len(self.indicesSized): + moreProblemSizes = False + else: + currentSizedIndexSizes[i] += currentSizedIndexIncrements[i] + currentSizedIndexIncrements[i] += self.indicesSized[i][2] + + problemIdx+=1 + + ######################################## + # YAML format + def __str__(self): + state = "[ " + sizedIdx = 0 + mappedIdx = 0 + for i in range(0, len(self.indexIsSized)): + if self.indexIsSized[i]: + indices = self.indicesSized[sizedIdx] + state += "[ %u, %u, %u, %u ]" \ + % (indices[0], indices[1], indices[2], indices[3]) + sizedIdx += 1 + else: + indices = self.indicesSized[self.indicesMapped[mappedIdx]] + state += str(self.indicesMapped[mappedIdx]) + mappedIdx += 1 + if i < len(self.indexIsSized)-1: + state += ", " + state += " ]" + return state + +class Problem: + """ Problem sizes, strides, padding and other info""" + def __init__(self, sizes=None, stridesA=None, stridesB=None, stridesC=None, stridesD=None, count=None): + self.sizes = tuple(sizes) if sizes else None + self.stridesA = tuple(stridesA) if stridesA else None + self.stridesB = tuple(stridesB) if stridesB else None + self.stridesC = tuple(stridesC) if stridesC else None + self.stridesD = tuple(stridesD) if stridesD else None + + self.count = count + + def __str__(self): + rv= "{ sizes:" + str(list(self.sizes)) + if self.stridesA: + rv += ", stridesA:" + str(list(self.stridesA)) + if self.stridesB: + rv += ", stridesB:" + str(list(self.stridesB)) + if self.stridesC: + rv += ", stridesC:" + str(list(self.stridesC)) + if self.stridesD: + rv += ", stridesD:" + str(list(self.stridesD)) + rv += " }" + return rv + +class ExactList(Problem): + def __init__(self, e, problemType): + if len(e) == problemType["TotalIndices"]: + if -1 in e: + printExit("ExactSize %s contains -1" % (e)) + if problemType["OperationType"] == "GEMM": + e += [-1, -1, -1, -1] + e = ExactList.convertLeadingDims(problemType, tuple(e)) + sizes=e + + elif len(e) == (problemType["TotalIndices"] + problemType["NumIndicesLD"]): + sizes = ExactList.convertLeadingDims(problemType, tuple(e)) + else: + printExit("ExactSize %s doesn't match indices of ProblemType %s, totalIndices=%d" \ + % (e, problemType, problemType["TotalIndices"]) ) + + # TODO- pass strides here, remove calls to convertLeadingDims + Problem.__init__(self, sizes=sizes) + + def __str__(self): + return str(list(self.sizes)) + + @staticmethod + def convertLeadingDims(problemType, problemSize, stridesA = None, stridesB = None, stridesC = None, stridesD = None): + # FIXME-problem: refactor to eliminate max, pass strides in strideB parm rather than hacked + # onto the end of the sizes list + predStridesD = stridesD is not None and stridesD[1] != -1 + predStridesC = stridesC is not None and stridesC[1] != -1 + predStridesA = stridesA is not None and stridesA[1] != -1 + predStridesB = stridesB is not None and stridesB[1] != -1 + return problemSize[:problemType["NumIndicesC"]+1] + \ + (max(problemSize[0], problemSize[problemType["IndexAssignmentsLD"][0]]) if not predStridesD else stridesD[1], ) + \ + (max(problemSize[0], problemSize[problemType["IndexAssignmentsLD"][1]]) if not predStridesC else stridesC[1], ) + \ + (max(problemSize[problemType["IndexAssignmentsLD"][2]], + problemSize[problemType["IndexAssignmentsA"][0]]) if not predStridesA else stridesA[1], ) + \ + (max(problemSize[problemType["IndexAssignmentsLD"][3]], + problemSize[problemType["IndexAssignmentsB"][0]]) if not predStridesB else stridesB[1], ) + + +class ExactDict(Problem): + AllowedFields = [ 'count', 'sizes', 'stridesA', 'stridesB', 'stridesC', 'stridesD' ] + + def __init__(self, e, problemType): + Problem.__init__(self) + + for f in e: + if f in ExactDict.AllowedFields: + setattr(self, f, e[f]) + else: + raise RuntimeError ("specified field '%s' is not a valid Exact dict field"%f) + + if problemType: + if "OperationType" in problemType and problemType["OperationType"] == "GEMM": + sizesTuple = tuple(self.sizes + [-1, -1, -1, -1]) + self.sizes = ExactList.convertLeadingDims(problemType, sizesTuple, self.stridesA, self.stridesB, self.stridesC, self.stridesD) + + if problemType: + if "OperationType" in problemType and problemType["OperationType"] == "GEMM": + if len(self.sizes) != (problemType["TotalIndices"] + problemType["NumIndicesLD"]): + # FIXME-ExactDict size descriptor still (but preferrably not so) uses 8-tuple for GEMM problems + raise RuntimeError ("specified size=%s does not have enough indices for problem (expected %d, got %d)" \ + % (self.sizes, problemType["TotalIndices"]+problemType["NumIndicesLD"], len(self.sizes))) + elif len(self.sizes) != problemType["TotalIndices"]: + raise RuntimeError ("specified size=%s does not have enough indices for problem (expected %d, got %d)" \ + % (self.sizes, problemType["TotalIndices"], len(self.sizes))) + + +################################################################################ +# ProblemSizes +################################################################################ +""" +Adapter class for class `ProblemSizes`. It satisfies the implicit usage requirement +of ClientWriter.writeClientConfig() by converting ExactLogic to list of `Problem` objects +""" +class ProblemSizesMock: + def __init__(self, exactLogic): + self.problems = [Problem(problem) for problem, solution in exactLogic] + +class ProblemSizes: + + ######################################## + def __init__(self, problemType, config): + self.problemType = problemType + self.ranges = [] + self.exacts = [] + self.minStrides = None + if config: + for dictionary in config: + for sizeTypeKey in dictionary: + #print ("PROBLEM parsed:", sizeTypeKey, dictionary[sizeTypeKey]) + if sizeTypeKey == "Range": + psr = ProblemSizeRange(problemType, dictionary[sizeTypeKey]) + self.ranges.append( psr ) + elif sizeTypeKey == "Exact": + e= dictionary[sizeTypeKey] + if isinstance(e,list): + self.exacts.append(ExactList(e, problemType)) + elif isinstance(e,dict): + self.exacts.append(ExactDict(e, problemType)) + else: + printExit("Unsupported Exact type==%s"%type(e)) + elif sizeTypeKey == "MinStride": + e = dictionary[sizeTypeKey] + if len(e) != problemType["TotalIndices"]: + printExit("MinStride %s doesn't match indices of ProblemType %s" \ + % (e, problemType) ) + if self.minStrides: + printExit("Only one MinStride command is allowed in a ProblemsSizes definition. Previous minStrides:%s, New minstride:%s" \ + % (self.minStrides, e) ) + + self.minStrides=(tuple(e)) + else: + printExit("ProblemSize Type %s not supported"%sizeTypeKey) + + if not self.minStrides: + # set harmless default mins of 0 + self.minStrides = ([0]* problemType["TotalIndices"]) + + # not the ideal spot, but convert leading dims that are below the minimum size + if problemType["OperationType"] == "GEMM": + for i in range(0, len(self.ranges)): + self.ranges[i].problemSizes[:] = \ + [ExactList.convertLeadingDims(self.problemType, problemSize) for problemSize in self.ranges[i].problemSizes] + + self.problems = OrderedDict() + for e in self.exacts: + self.problems.update({e : 1}) + if globalParameters["SortProblems"]: + self.problems = sorted(list( self.problems.keys()), key=operator.attrgetter("sizes")) + else: + self.problems = list(self.problems.keys()) + self.totalProblemSizes = len(self.problems) + + # max sizes + self.maxD = 0 + self.maxC = 0 + self.maxA = 0 + self.maxB = 0 + for problem in self.problems: + problemSize = problem.sizes # FIXME-problem. This should use problem.strides* + + sizeLdd = problemSize[self.problemType["IndexAssignmentsLD"][0]] if problemType["OperationType"] == "GEMM" else problemSize[0] + sizeD = max(self.minStrides[0], sizeLdd) + for i in range(1, problemType["NumIndicesC"]): + sizeD *= max(self.minStrides[i], problemSize[i]) + + sizeLdc = problemSize[self.problemType["IndexAssignmentsLD"][1]] if problemType["OperationType"] == "GEMM" else problemSize[0] + sizeC = max(self.minStrides[0], sizeLdc) + for i in range(1, problemType["NumIndicesC"]): + sizeC *= max(self.minStrides[i], problemSize[i]) + + sizeLda = problemSize[self.problemType["IndexAssignmentsLD"][2]] \ + if problemType["OperationType"] == "GEMM" \ + else problemSize[self.problemType["IndexAssignmentsA"][0]] + sizeA = max(self.minStrides[self.problemType["IndexAssignmentsA"][0]], sizeLda) + for i in self.problemType["IndexAssignmentsA"][1:]: + sizeA *= max(self.minStrides[i], problemSize[i]) + + sizeLdb = problemSize[self.problemType["IndexAssignmentsLD"][3]] \ + if problemType["OperationType"] == "GEMM" \ + else problemSize[self.problemType["IndexAssignmentsB"][0]] + sizeB = max(self.minStrides[self.problemType["IndexAssignmentsB"][0]], sizeLdb) + for i in self.problemType["IndexAssignmentsB"][1:]: + sizeB *= max(self.minStrides[i], problemSize[i]) + + self.maxD = max(self.maxD, sizeD) + self.maxC = max(self.maxC, sizeC) + self.maxA = max(self.maxA, sizeA) + self.maxB = max(self.maxB, sizeB) + + def __str__(self): + s = "ProblemSizes\n" + for sizeRange in self.ranges: + s += " %s" % sizeRange + return s + +################################################################################ +# Activation +################################################################################ + +class activationSetting: + def __init__(self): + self.activationEnum = "" +class ActivationArgs: + + ######################################## + def __init__(self, problemType, config): + self.settingList = [] + self.totalProblemSizes = 0 + if problemType["ActivationType"] == 'none': + return + if config: + for settings in config: + actSetting = activationSetting() + for dictionary in settings: + for sizeTypeKey in dictionary: + if sizeTypeKey == "Enum": + actSetting.activationEnum = ActivationType(dictionary[sizeTypeKey]) + if problemType["ActivationType"] == 'all': + if (not actSetting.activationEnum): + printExit("Must provide an activation enum if Activation is set to True.") + else: + actSetting.activationEnum = problemType["ActivationType"] + self.settingList.append(actSetting) + self.totalProblemSizes += 1 + if (problemType["ActivationType"] == 'all') and (not self.settingList): + printExit("Must provide an activation enum in benchmark parameters if Activation is set to True.") + def __str__(self): + s = "ActivationArgs\n" + return s + +# kds is class Solution or class Kernel +# All free dims are packed +def isPackedIndex(ks, index): + problemType = ks["ProblemType"] + return index in problemType["IndicesFree"] + +def isExtractableIndex(ks, index, tc='x'): + xA = index in ks['PackedC0IndicesX'][:-1] + xB = index in ks['PackedC1IndicesX'][:-1] + if tc=='A': + return xA + elif tc=='B': + return xB + else: + return xA or xB + +################################################################################ +# Solution +################################################################################ +class Solution(collections.abc.Mapping): + + ######################################## + def __init__(self, config): + self._name = None + config = deepcopy(config) + + self._state = {} + # problem type + if "ProblemType" in config: + self["ProblemType"] = ProblemType(config["ProblemType"]) + else: + self["ProblemType"] = ProblemType(defaultProblemType) + + # assign parameters with defaults + for key in defaultSolution: + assignParameterWithDefault(self._state, key, config, defaultSolution) + + if 'ISA' not in self._state: + if 'ISA' in config: + if not globalParameters["AsmCaps"][tuple(config['ISA'])]["SupportedISA"]: + defaultIsa = [9,0,0] + print("warning: ISA:", config['ISA'], " is not supported; overriding with ", defaultIsa) + self._state['ISA'] = defaultIsa + else: + self._state['ISA'] = config['ISA'] + else: + # Assembly by default + self._state['ISA'] = list(globalParameters["CurrentISA"]) + if 'KernelLanguage' in config: + if config['KernelLanguage'] != 'Assembly': + self._state['ISA'] = [0,0,0] + + if "CodeObjectVersion" not in self._state: + if "CodeObjectVersion" in config: + self._state["CodeObjectVersion"] = config["CodeObjectVersion"] + else: + self._state["CodeObjectVersion"] = globalParameters["CodeObjectVersion"] + + # assign parameters without defaults + for key in config: + if key != "ProblemType" and key not in self._state: + self._state[key] = config[key] + self["Valid"] = True + # this could prevent OriginalSolution from re-assigning the parameters, save lots of time + if "AssignedProblemIndependentDerivedParameters" not in self._state: + self["AssignedProblemIndependentDerivedParameters"] = False + if "AssignedDerivedParameters" not in self._state: + self["AssignedDerivedParameters"] = False + + Solution.assignDerivedParameters(self._state) + self._name = config["CustomKernelName"] if isCustomKernelConfig(config) else None + self.initHelperKernelObjects() + + # these keys are copied from ProblemType to internal that may be overridden + InternalKeys = ["UseSgprForGRO","VectorStore"] + + + ######################################## + # get a list of kernel parameters for this solution + def getKernels(self): + kernel = deepcopy(self) + kernel._state.update({"Kernel": True}) + kernels = [] + kernels.append(kernel) + return kernels + + + ######################################## + # create Helper Kernels + def initHelperKernelObjects(self): + self.initBetaOnlyKernelObjects() + self.initConversionKernelObjects() + self.initActivationEnumHeaderObjects() + self.initActivationFunctionObjects() + self.initActivationOnlyKernelObjects() + + + ######################################## + # create BetaONly Kernels + def initBetaOnlyKernelObjects(self): + self.betaOnlyKernelObjects = [] + if self["GlobalSplitU"] > 1: + state = {} + state["ProblemType"] = deepcopy(self["ProblemType"]) + state["KernelLanguage"] = "Source" + state["_GlobalAccumulation"] = self["_GlobalAccumulation"] + self.betaOnlyKernelObjects.append(KernelWriterBetaOnly(state)) + + + ######################################## + # create Conversion Kernels + def initConversionKernelObjects(self): + self.conversionKernelObjects = [] + if (self["GlobalSplitU"] > 1) and self["_GlobalAccumulation"]: + state = {} + state["ProblemType"] = deepcopy(self["ProblemType"]) + state["KernelLanguage"] = "Source" + state["_GlobalAccumulation"] = self["_GlobalAccumulation"] + state["ActivationFused"] = self["ActivationFused"] + self.conversionKernelObjects.append(KernelWriterConversion(state)) + + def initActivationEnumHeaderObjects(self): + self.activationEnumHeaderObjects = [] + if ((self["ProblemType"]["ActivationType"] != 'none') and (self["ProblemType"]["ActivationType"] == 'all')) : + state = {} + state["ProblemType"] = deepcopy(self["ProblemType"]) + state["KernelLanguage"] = "Source" + self.activationEnumHeaderObjects.append(KernelWriterActivationEnumHeader(state)) + + def initActivationFunctionObjects(self): + self.activationFunctionObjects = [] + if ((self["ProblemType"]["ActivationType"] != 'none') and (self["ProblemType"]["ActivationType"] == 'all') and \ + ((self["GlobalSplitU"] > 1) or (self["ActivationFused"] == False))) : + state = {} + state["ProblemType"] = deepcopy(self["ProblemType"]) + state["KernelLanguage"] = "Source" + state["Kernel"] = {"WavefrontSize": self["WavefrontSize"], "ISA": tuple(self["ISA"])} + self.activationFunctionObjects.append(KernelWriterActivationFunction(state)) + + def initActivationOnlyKernelObjects(self): + self.activationOnlyKernelObjects = [] + if (((self["GlobalSplitU"] > 1) and (not self["_GlobalAccumulation"])) or (self["ActivationFused"] == False)) \ + and (self["ProblemType"]["ActivationType"] != 'none') : + state = {} + state["ProblemType"] = deepcopy(self["ProblemType"]) + state["KernelLanguage"] = "Source" + state["_GlobalAccumulation"] = self["_GlobalAccumulation"] + state["ActivationFused"] = self["ActivationFused"] + self.activationOnlyKernelObjects.append(KernelWriterActivationOnly(state)) + + ######################################## + # get Helper Kernels + def getHelperKernelObjects(self): + return self.activationEnumHeaderObjects + self.activationFunctionObjects + \ + self.betaOnlyKernelObjects + self.conversionKernelObjects + \ + self.activationOnlyKernelObjects + + + ######################################## + # get Helper Kernels + def getKernelBetaOlnyObjects(self): + return self.betaOnlyKernelObjects + + + ######################################## + # get Helper Kernels + def getKernelConversionObjects(self): + return self.conversionKernelObjects + + + ######################################## + # assign tile sizes + @staticmethod + def assignProblemIndependentDerivedParameters(state): + + if globalParameters["NewClient"] != 2: + print("WARNING: Old client deprecated, NewClient parameter being set to 2.") + globalParameters["NewClient"] = 2 + + if "AssignedProblemIndependentDerivedParameters" in state: + if state["AssignedProblemIndependentDerivedParameters"]: + return + state["AssignedProblemIndependentDerivedParameters"] = False + if "Valid" not in state: + state["Valid"] = True + + if (not state["ProblemType"]["StridedBatched"]) and (not state["ProblemType"]['Batched']): + reject(state, "General Batched GEMM only support Batched Problem") + + if (not state["ProblemType"]["StridedBatched"]) and (state["ProblemType"]["OperationType"] != 'GEMM'): + reject(state, "General Batched GEMM only support GEMM OperationType") + + Solution.MatrixInstructionToMIParameters(state) + EnableMatrixInstruction = state["EnableMatrixInstruction"] if "EnableMatrixInstruction" in state else None + if EnableMatrixInstruction == None: + if ("MIBlock" in state and len(state["MIBlock"]) == 6) \ + and ("MIWaveGroup" in state and len(state["MIWaveGroup"]) == 2) \ + and ("MIWaveTile" in state and len(state["MIWaveTile"]) == 2): + EnableMatrixInstruction = True + elif ("WorkGroup" in state and len(state["WorkGroup"]) == 3) \ + and ("ThreadTile" in state and len(state["ThreadTile"]) == 2) : + EnableMatrixInstruction = False + else: + reject(state, "EnableMatrixInstruction undetermined") + + if EnableMatrixInstruction == True: + state["MatrixInstM"] = state["MIBlock"][0] + state["MatrixInstN"] = state["MIBlock"][1] + state["MatrixInstK"] = state["MIBlock"][2] + state["MatrixInstB"] = state["MIBlock"][3] + state["MatrixInstBM"] = state["MIBlock"][4] + state["MatrixInstBN"] = state["MIBlock"][5] + + state["LocalSplitU"] = 1 + state["MIOutputVectorWidth"] = 1 if (state["ProblemType"]["DataType"].MIOutputTypeNameAbbrev() == 'f64') else 4 + state["MIRegPerOut"] = 2 if (state["ProblemType"]["DataType"].MIOutputTypeNameAbbrev() == 'f64') else 1 + + if state["MatrixInstM"] == 4: + state["ThreadTile0"] = state["MIWaveTile"][0] * state["MIOutputVectorWidth"] + state["ThreadTile1"] = state["MIWaveTile"][1] + state["SubGroup0"] = state["MIWaveGroup"][0] * state["MatrixInstM"] * state["MatrixInstBM"] // state["MIOutputVectorWidth"] + state["SubGroup1"] = state["MIWaveGroup"][1] * state["MatrixInstN"] * state["MatrixInstBN"] + else: + state["ThreadTile0"] = state["MatrixInstBM"] * state["MIWaveTile"][0] * (state["MatrixInstM"] * state["MatrixInstN"] // state["WavefrontSize"]) + state["ThreadTile1"] = state["MatrixInstBN"] * state["MIWaveTile"][1] + state["SubGroup0"] = state["MIWaveGroup"][0] * (state["WavefrontSize"] // state["MatrixInstN"]) + state["SubGroup1"] = state["MIWaveGroup"][1] * state["MatrixInstN"] + + elif EnableMatrixInstruction == False: + state["ThreadTile0"] = state["ThreadTile"][0] + state["ThreadTile1"] = state["ThreadTile"][1] + + state["SubGroup0"] = state["WorkGroup"][0] + state["SubGroup1"] = state["WorkGroup"][1] + state["LocalSplitU"] = state["WorkGroup"][2] + + if "SubGroup0" in state and "SubGroup1" in state and "LocalSplitU" in state: + state["NumThreads"] = state["SubGroup0"] * state["SubGroup1"] * state["LocalSplitU"] + + # macro tile sizes + if "SubGroup0" in state and "ThreadTile0" in state: + state["MacroTile0"] = state["SubGroup0"]*state["ThreadTile0"] + if "SubGroup1" in state and "ThreadTile1" in state: + state["MacroTile1"] = state["SubGroup1"]*state["ThreadTile1"] + if "MacroTile" in state: + if state["MacroTile0"] != state["MacroTile"][0] \ + or state["MacroTile1"] != state["MacroTile"][1]: + reject(state, "MacroTile mismatch") + + # done + state["AssignedProblemIndependentDerivedParameters"] = True + + ######################################## + # This is the "classic" algorithm which requires that each threads load the same number of bytes + # Called with tc=A and then with tc=B + # totalVectors is totalElements/GRVW, this is #vectors loaded by the LoadTile + # Reduces the GlobalLoadVectorWidth if necessary if each thread has a small amount of work to do. + # Output from this function: + # state[GlobalLoadVectorWidth*] + # state[NumLoads*] # only used in SolutionStructs, with classic alg + @staticmethod + def setGlobalLoadVectorWidth(state, tc, totalVectors, grvw): + validDepthU = True + if totalVectors < state["NumThreads"]: + # Try to reduce size of vector so every thread has a load to do + pv = state["NumThreads"]//totalVectors + if state["NumThreads"] % totalVectors != 0: + reject(None, "NumThreads %u %% totalVectors %u != 0" \ + % (state["NumThreads"], totalVectors)) + validDepthU = False + if pv * totalVectors != state["NumThreads"]: + reject(None, "pv %u * totalVectors %u != NumThreads %u" \ + % (pv, totalVectors, state["NumThreads"])) + validDepthU = False + if grvw % pv != 0: + reject(None, "GlobalReadVectorWidth %u %% pv %u != 0" \ + % (grvw, pv)) + validDepthU = False + else: + pv = 1 # no partial vector required + if totalVectors % state["NumThreads"] != 0: + reject(None, "totalVectors %u %% NumThreads %u != 0" \ + % (totalVectors, state["NumThreads"])) + validDepthU = False + + state["GlobalLoadVectorWidth%s"%tc] = grvw//pv + + # NumLoads is NOT used on the fractional path + # NumLoads is number of vector loads per-thread + state["NumLoads%s"%tc] = totalVectors * pv // state["NumThreads"] + #print "result: ", pvar(state, "GlobalLoadVectorWidth%s"%tc), \ + # pvar(state, "NumLoads%s"%tc) + + return validDepthU + + ######################################## + # Sets the Global Read Tile dims (para, perp) + # This information controls which threads read which addresses from global mem) + # Output from this function: + # state[NumLoadsCoalescedA] + # state[NumLoadsPerpendicularA] + # state[LSCA] + # state[LSPA] + @staticmethod + def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, totalElementsPerp): + + if state["WaveSeparateGlobalRead%s"%tc]: + totalElementsPerp = roundupRatio(totalElementsPerp, state["NumThreads"] // state["WavefrontSize"]) + + # nlc = 1 + if state["NumLoadsCoalesced%s"%tc] == 1 : + foundValid = False + nlcStart = 1 + if state["DirectToVgpr%s"%tc]: + # adjust nlc for DirectToVgpr + if state["ProblemType"]["TLU%s"%tc]: + nlcStart = roundupRatio(state["MIWaveTile%s"%tc], state["GlobalLoadVectorWidth%s"%tc]) + else: + nlcStart = roundupRatio(state["DepthU"], state["MatrixInstK"] * state["GlobalLoadVectorWidth%s"%tc]) + for nlc in range(nlcStart, int(state["NumLoads%s"%tc]+1)): + nlp = state["NumLoads%s"%tc] // nlc + if state["NumLoads%s"%tc] % nlc == 0 \ + and totalVectorsCoalesced % nlc == 0 \ + and totalElementsPerp % nlp == 0: + state["NumLoadsCoalesced%s"%tc] = nlc + state["NumLoadsPerpendicular%s"%tc] = nlp + #print("NumLoadsCoalesced",state["NumLoadsCoalesced%s"%tc]) + #print("NumLoadsPerpendicular",state["NumLoadsPerpendicular%s"%tc]) + foundValid = True + break + if not foundValid: + reject(state, "%s: No NumLoadsCoalesced=1 found"%tc) + return False + + # nlc = -1 + elif state["NumLoadsCoalesced%s"%tc] == -1: + foundValid = False + for nlc in range(state["NumLoads%s"%tc], 0, -1): + nlp = state["NumLoads%s"%tc] // nlc + if state["NumLoads%s"%tc] % nlc == 0 \ + and totalVectorsCoalesced % nlc == 0 \ + and totalElementsPerp % nlp == 0: + state["NumLoadsCoalesced%s"%tc] = nlc + state["NumLoadsPerpendicular%s"%tc] = nlp + foundValid = True + break + if not foundValid: + reject(state, "%s: No NumLoadsCoalesced=-1 found"%tc) + return False + + # nlc = other + else: + if state["NumLoadsCoalesced%s"%tc] > state["NumLoads%s"%tc]: + reject(state, "%s nlc > numLoads"%tc) + return False + + state["NumLoadsPerpendicular%s"%tc] = state["NumLoads%s"%tc] \ + // state["NumLoadsCoalesced%s"%tc] + + if state["NumLoads%s"%tc] % state["NumLoadsCoalesced%s"%tc] != 0: + reject(state, "%s: numLoads %u %% numLoadsCoalesced %u != 0" \ + % (tc, state["NumLoads%s"%tc], state["NumLoadsCoalesced%s"%tc])) + return False + + if totalVectorsCoalesced % state["NumLoadsCoalesced%s"%tc] != 0 : + reject(state, "%s: totalVectorsCoalesced %u %% numLoadsPara %u != 0" \ + % (tc, totalVectorsCoalesced, state["NumLoadsCoalesced%s"%tc])) + return False + if totalElementsPerp % state["NumLoadsPerpendicular%s"%tc] != 0: + reject(state, "%s: totalElementsPerp %u %% numLoadsPerp %u != 0" \ + % (tc, totalElementsPerp, state["NumLoadsPerpendicular%s"%tc])) + return False + + if state["ProblemType"]["TLU%s"%tc]: + state["LSC%s"%tc] = state["MacroTile%s"%tc] // state["NumLoadsCoalesced%s"%tc] + state["LSP%s"%tc] = int(math.ceil(float(state["DepthU"]) / state["NumLoadsPerpendicular%s"%tc])) + else: + state["LSC%s"%tc] = int(math.ceil(float(state["DepthU"]) / state["NumLoadsCoalesced%s"%tc])) + state["LSP%s"%tc] = state["MacroTile%s"%tc] // state["NumLoadsPerpendicular%s"%tc] + + if state["WaveSeparateGlobalRead%s"%tc]: + state["LSP%s"%tc] = roundupRatio(state["LSP%s"%tc], state["NumThreads"] // state["WavefrontSize"]) + + return True + + + ######################################## + # Sets the Global Read Tile dims (para, perp) + # This information controls which threads read which addresses from global mem) + # Output from this function: + # state[NumLoadsCoalesced*] + # state[NumLoadsPerpendicular*] + # state[LSC*] + # state[LSP*] + # state[GlobalReadVectorWidth] + # + # LSC and LSP define the shape of the PerLoadTile, measured in elements. + # LSC*LSP is the elements loaded by a single instruction across all + # threads in the group. + # LSC is the number of elements loaded in the para(coalesced) dimension + # LSP is the number of elements loaded in the perp(noncoalesced) dimension + # PerLoadTile is always rectangular. + # When BufferLoad=1, the area (LSC*LSP) can be larger than NumThreads. + # In this case, some threads will generate a dummy OOB GRO. + # Related fields: + # LVC = LSC/GRVW (LVCA = LSCA/GLVWA) + # LVP = LSP/GRVW (LVPA = LSPA/GLVWA) + # + # NumLoadsCoalesced and NumLoadsPerpendicular define the number of times the + # PerLoadTile is loaded in each dimension to fetch the LoadTile + # LoadTile = (LSC * NumLoadsCoalesced) * (LSP * NumLoadsPerpendicular). + # For Fractional, the LoadTile can be larger than the MacroTile. Buffer + # loads will clip any OOB references to 0 and will also avoid writing these + # into LDS. + + # Fractional load algorithm: + # - Each load instruction loads one or more (complete) rows of the load tile. + # - Each row is LSC elements wide + # - Rows are complete and do not wrap. This allows a single base GRO VGPR + # to be used for all loads in the tile. + # - Some work-items in the load may not perform useful work. These WI will + # set their GRO to a large OOB number so as to do no harm + # - Some G2L registers space may be unused as well. + # - The 'used' message at the bottom of this routine computes and prints the + # wasted register space. + # - The wasted space is removed when the data is written to LDS- the LWO + # for work-items beyond the valid ones are set to safely write to OOB locations. + + # - In cases where each load is loading multiple rows (multiple lines of lsc + # elements), the last load is allowed to load fewer lines than the others. + # The KernelWriterAssembly will modify the LWO for the last load. This allows + # flexibility in the unroll factors for example. + @staticmethod + def setGlobalLoadTileDimFractional(state, tc, depthU): + + assert(depthU > 0) + dbFract = 0 + + # parDim, perpDim define the LoadTile and are measured in elements + if state["ProblemType"]["TLU%s"%tc]: + parDim = state["MacroTile%s"%tc] + perpDim = depthU + else: + parDim = depthU + perpDim = state["MacroTile%s"%tc] + + if dbFract: + print("\ninfo: %s Fractional MT%u_%u_%u Par=%u Perp=%u WG%02u_%02u_%02u NumThreads=%u GRWV=%u" \ + % (tc, state["MacroTile0"], state["MacroTile1"], depthU, \ + parDim, perpDim, \ + state["WorkGroup"][0], state["WorkGroup"][1], state["LocalSplitU"], \ + state["NumThreads"], state["GlobalReadVectorWidth"])) + + # Try to find a GRVW which is smaller than the LSC and also does not force + # the LSC to wrap - both of these conditions can be tested with lsc % grvw ==0. + # Each iteration divides GRWV by 2 which provides finer granularity + # and a possible opportunity to handle the lsc + grvw = state["GlobalReadVectorWidth"] + minGrvw = 2 if state["ProblemType"]["DataType"].isHalf() and \ + globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"] else 1 + # TODO- check this for int8 and fractional load + # minGrvw = 4 if state["ProblemType"]["DataType"].isInt8() and \ + # globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"] else 1 + bestVw = -1 + while grvw >= minGrvw: + # Per instruction across the entire group: + elementsLoadedPerInst = state["NumThreads"]*grvw + if (state["DirectToVgpr%s"%tc] and state["ProblemType"]["TLU%s"%tc]): + elementsLoadedPerInst //= state["MatrixInstK"] + # LSC, LSP - #elements loaded along specified dim with each load + if parDim >= elementsLoadedPerInst: + # entire work-group can work on (part) of the same row + # DirectToVgpr case, LSC is limited to elementsLoadedPerInst // state["MatrixInstK"] + state["LSC%s"%tc] = elementsLoadedPerInst + state["LSP%s"%tc] = 1 if not (state["DirectToVgpr%s"%tc] and state["ProblemType"]["TLU%s"%tc]) else state["MatrixInstK"] + state["NumLoadsCoalesced%s"%tc] = roundupRatio(parDim , state["LSC%s"%tc]) + state["NumLoadsPerpendicular%s"%tc] = 1 + else: + # work-group exceeds read dimension so wraps to multiple rows + state["LSC%s"%tc] = parDim + state["LSP%s"%tc] = min(perpDim, elementsLoadedPerInst // parDim) + state["NumLoadsCoalesced%s"%tc] = 1 + state["NumLoadsPerpendicular%s"%tc] = roundupRatio(perpDim , state["LSP%s"%tc]) + + # Vector loads can't wrap to next P dim, so LSC must be divisible by vector elements; + if dbFract: + print(" lsc search : lsc(%u) %% grvw(%u) = %u (?0)" % (state["LSC%s"%tc], grvw, state["LSC%s"%tc] % grvw)) + if state["LSC%s"%tc] % grvw == 0: + bestVw = grvw + # Try to shrink GRVW if possible while keeping same LSC and LSP: + # For example, avoid cases where we use a GRVW=4 with many empty addresses + # when a GRVW=1 will do instead. + validElementsLoadedPerInst = state["LSC%s"%tc] * state["LSP%s"%tc] + grvw //= 2 + while grvw >= minGrvw: + elementsLoadedPerInst = state["NumThreads"]*grvw + if state["DirectToVgpr%s"%tc] and state["ProblemType"]["TLU%s"%tc]: + elementsLoadedPerInst //= state["MatrixInstK"] + if elementsLoadedPerInst < validElementsLoadedPerInst: + break # Went too far, not enough load elements at this VW + if state["LSC%s"%tc] % grvw == 0: + if dbFract: + print(" stepdown success (valid)elementsLoadedPerInst=", validElementsLoadedPerInst, "/", elementsLoadedPerInst, "grvw=", grvw, "lsc=", state["LSC%s"%tc]) + bestVw = grvw + grvw //= 2 + break + + # TODO - could have this generate dwordx3 loads in addition, step down by 1 instead of div2 + # Would need to change asm code gen to generate x3 + grvw //= 2 + # end-- while loop + + if bestVw == -1: + if dbFract: + print ("reject fractional - no acceptable tile dim? GlobalReadVectorWidth", \ + state["GlobalReadVectorWidth"]) + return False # could not find a solution, perhaps only possible for half ? + + state["GlobalLoadVectorWidth%s"%tc] = bestVw + if bestVw != state["GlobalReadVectorWidth"]: + if dbFract: + print(" reducing GlobalLoadVectorWidth%s from %u to %u" \ + % (tc, state["GlobalReadVectorWidth"], bestVw)) + + # How many loads per threads in each dimension. + # threads which are outside the global read tile bounds will be clipped + # in the assembly code generator. + # Multiply the LSC*GRVW + state["NumLoadsCoalesced%s"%tc] = roundupRatio(parDim, state["LSC%s"%tc]) + state["NumLoadsPerpendicular%s"%tc] = roundupRatio(perpDim , state["LSP%s"%tc]) + + nlc = state["NumLoadsCoalesced%s"%tc] + nlp = state["NumLoadsPerpendicular%s"%tc] + + # LoadTile must at least cover the MacroTile: + assert(nlc*state["LSC%s"%tc] >= parDim) + assert(nlp*state["LSP%s"%tc] >= perpDim) + + perpOverhang = perpDim % state["LSP%s"%tc] + state["fractionalPerpOverhang%s"%tc] = perpOverhang + if dbFract: + # how many threads compute Global Read Offsets (GRO) that are not used + print(" PerLoadTile=%ux%u elements Loads/WI=%ux%u LoadTile/WI=%ux%u (MT=%ux%u), %u/%u = %.1f%% WI GRO used %s" \ + % (state["LSC%s"%tc], state["LSP%s"%tc], \ + nlc, nlp, \ + nlc*state["LSC%s"%tc], nlp*state["LSP%s"%tc], \ + parDim, perpDim, \ + parDim*perpDim, \ + nlc*nlp*state["NumThreads"]*state["GlobalLoadVectorWidth%s"%tc], \ + float(parDim*perpDim), \ + float(nlc*nlp*state["NumThreads"]*state["GlobalLoadVectorWidth%s"%tc]) * 100.0) \ + ) + + for p in range(0,nlp): + elementWidth = 4 + if p != nlp-1: + perp = state["LSP%s"%tc] + else: + perp = perpOverhang if perpOverhang else state["LSP%s"%tc] + + validElements = state["LSC%s"%tc] * perp + print(" buffer_load_element_x%u %ux%ux%u bytes, %u/%u valid GRO" %\ + (state["GlobalLoadVectorWidth%s"%tc], \ + state["LSC%s"%tc], perp, \ + elementWidth, \ + validElements//state["GlobalLoadVectorWidth%s"%tc], + state["NumThreads"])) + + return True + + + @staticmethod + def MatrixInstructionToMIParameters(state): + if len(state["MatrixInstruction"]) == 9: + mi = state["MatrixInstruction"] + state["MatrixInstruction"] = [state["MatrixInstruction"][0],state["MatrixInstruction"][1],state["MatrixInstruction"][2],state["MatrixInstruction"][3]] + + waves = mi[7]* mi[8] + miwg0 = mi[4] * mi[0] * mi[7] + state["WorkGroup"][0] = miwg0 + state["WorkGroup"][1] = waves*state["WavefrontSize"] // state["WorkGroup"][0] + state["ThreadTile"][0] = 1 # dummy + state["ThreadTile"][1] = 1 # dummy + + state["MFMA_BF16_1K"] = False + if not (state["ProblemType"]["DataType"].toChar() in validMFMA and \ + state["MatrixInstruction"] in validMFMA[state["ProblemType"]["DataType"].toChar()]): + if state["ProblemType"]["DataType"].isBFloat16() and \ + state["MatrixInstruction"] in validMFMA["B1k"]: + state["MFMA_BF16_1K"] = True + else: + reject(state, "MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], state["ProblemType"]["DataType"])) + + # set EnableMatrixInstruction + state["EnableMatrixInstruction"] = True + + # set MIBlock + MIBlock_BM = miwg0 // mi[0] + MIBlock_BM = min(MIBlock_BM, mi[3]) + MIBlock_BN = mi[3] // MIBlock_BM + + state["MIBlock"] = [32, 32, 2, 1, 1, 1] + state["MIBlock"][0] = mi[0] + state["MIBlock"][1] = mi[1] + state["MIBlock"][2] = mi[2] + state["MIBlock"][3] = mi[3] + state["MIBlock"][4] = MIBlock_BM + state["MIBlock"][5] = MIBlock_BN + + # set MIWaveGroup + state['MIWaveGroup'] = [1, 1] + state['MIWaveGroup'][0] = min((miwg0 // mi[0]) // MIBlock_BM, waves) + state['MIWaveGroup'][1] = waves // state['MIWaveGroup'][0] + + # set MIWaveTile + state['MIWaveTile'] = [1, 1] + state['MIWaveTile'][0] = mi[5] + state['MIWaveTile'][1] = mi[6] + # set MIInputPerThread + state['MIInputPerThread'] = mi[0] * mi[2] * mi[3] // 64 + elif state["MatrixInstruction"] != [] and len(state["MatrixInstruction"]) == 4: + state["EnableMatrixInstruction"] = True + else: + state["EnableMatrixInstruction"] = False + + + ############################################## + # check and calculate Wave Separate Global Read + @staticmethod + def checkAndAssignWaveSeparateGlobalRead(state, tc): + # check can we use WaveSeparateGlobalRead + numOfWaves = state["NumThreads"] // state["WavefrontSize"] + if state["WaveSeparateGlobalRead%s"%tc]: + if state["ProblemType"]["TLU%s"%tc] and (state["DepthU"] > 0) and (state["DepthU"] % numOfWaves != 0): + reject(state, "didn't support WaveSeparateGlobalRead when DepthU is not multiple of wave %u in TLU%s" % (state["DepthU"], tc)) + if not state["ProblemType"]["TLU%s"%tc] and (state["MacroTile%s" % tc] % numOfWaves != 0): + reject(state, "didn't support WaveSeparateGlobalRead when MacroTile is not multiple of wave %u in TLU%s" % (state["MacroTile%s"%tc], tc)) + + + ######################################## + # determine can we use DirectToVgpr + @staticmethod + def isDirectToVgprDoable(state, tc): + tcOther = 'B' if tc == 'A' else 'B' + MIindex = 0 if tc == 'A' else 1 + # With MatrixInstruction only (tentative) + if not state["EnableMatrixInstruction"] : + reject(state, "DirectToVgpr is for MatrixInstruction only") + return False + + # Double/DoubleComplex only (tentative) + if not (state["ProblemType"]["DataType"].isDouble() or state["ProblemType"]["DataType"].isDoubleComplex()): + reject(state, "so far, DirectToVgpr is for double or double complex only") + return False + + # Does not work with TLU = False and PrefetchLocalRead = 0 + if (not state["ProblemType"]["TLU%c"%tc]) and state["PrefetchLocalRead"] == 0: + reject(state, "DirectToVgpr%c does not supports TLU%c = False and PrefetchLocalRead = 0"%(tc, tc)) + return False + + # MIWaveGroup check + # for A, MIWaveGroup should be [4, 1] + # for B, MIWaveGroup should be [1, 4] + # This is to limit the number of Vgpr + if tc == 'A' and not (state['MIWaveGroup'][0] == 4 and state['MIWaveGroup'][1] == 1): + reject(state, "MIWaveGroup should be [4, 1] for DirectToVgprA. Current value is [%s]"%state['MIWaveGroup']) + return False + if tc == 'B' and not (state['MIWaveGroup'][0] == 1 and state['MIWaveGroup'][1] == 4): + reject(state, "MIWaveGroup should be [1, 4] for DirectToVgprB. Current value is [%s]"%state['MIWaveGroup']) + return False + + # Does not work with WaveSeparateGlobalRead + if state["WaveSeparateGlobalRead%c"%tc]: + reject(state, "DirectToVgpr%c does not supports WaveSeparateGlobalRead%c"%(tc, tc)) + return False + + # Does not work with TLU and NumLoadsCoalesced != MIWaveTile / GlobalLoadVectorWidth + # (only for FractionalLoad = False) + if state["FractionalLoad"] == False: + if state["ProblemType"]["TLU%s"%tc] and state["NumLoadsCoalesced%c"%tc] != state['MIWaveTile'][MIindex] / state["GlobalLoadVectorWidth%c"%tc]: + reject(state, "DirectToVgpr%c does not supports NumLoadsCoalesced%c(=%u) != MIWaveTile[%u](=%u) / GlobalLoadVectorWidth%c(=%u)"\ + %(tc, tc, state["NumLoadsCoalesced%c"%tc], MIindex, state['MIWaveTile'][MIindex], tc, state["GlobalLoadVectorWidth%c"%tc])) + return False + # Does not work with TLU and MIWaveTile < GlobalLoadVectorWidth + if state["ProblemType"]["TLU%s"%tc] and state['MIWaveTile'][MIindex] < state["GlobalLoadVectorWidth%c"%tc]: + reject(state, "DirectToVgpr%c does not supports MIWaveTile[%u](=%u) < GlobalLoadVectorWidth%c(=%u)"\ + %(tc, MIindex, state['MIWaveTile'][MIindex], tc, state["GlobalLoadVectorWidth%c"%tc])) + return False + + # Does not work with ExpandPointerSwap = False + if not state["ExpandPointerSwap"]: + reject(state, "DirectToVgpr%c does not supports ExpandPointerSwap = False"%(tc)) + return False + + # Does not work with VectorWidth != GlobalReadVectorWidth (VW = 2 + GRVW = 1 or VW = 1 + GRVW = 2 does not work) + if state["VectorWidth"] != state["GlobalLoadVectorWidth%c"%tc]: + reject(state, "DirectToVgpr%c does not supports VectorWidth(=%u) != GlobalReadVectorWidth%c(%u)"%(tc, state["VectorWidth"], tc, state["GlobalLoadVectorWidth%c"%tc])) + return False + + # Does not work with AssertFree1ElementMultiple % VectorWidth != 0 (edge shift case) for B + if tc == 'B' and state["AssertFree1ElementMultiple"] % state["VectorWidth"] != 0: + reject(state, "DirectToVgpr%c does not supports AssertFree1ElementMultiple %% VectorWidth != 0"%(tc)) + return False + + # Does not work with FractionalLoad and (not TLU) + if state["FractionalLoad"] and (not state["ProblemType"]["TLU%c"%tc]): + reject(state, "DirectToVgpr%c does not supports FractionalLoad + TLU=False"%(tc)) + return False + + # Does not work with TLU=False and PGR=2 and DepthU<=MatrixInstK*VW + if (not state["ProblemType"]["TLU%c"%tc]) and state["PrefetchGlobalRead"] == 2 and state["DepthU"] <= state["MatrixInstK"] * state["VectorWidth"]: + reject(state, "DirectToVgpr%c does not supports TLU=False and PrefetchGlobalRead==2 and DepthU<=MatrixInstK*VectorWidth"%(tc)) + return False + + # Does not work with TLU=False and NumLoadsCoalesced != DepthU//(MatrixInstK*VW) + if (not state["ProblemType"]["TLU%c"%tc]) and state["NumLoadsCoalesced%c"%tc] != state["DepthU"] // (state["MatrixInstK"] * state["VectorWidth"]): + reject(state, "DirectToVgpr%c does not supports TLU=False and NumLoadsCoalesced%c != DepthU//(MatrixInstK*VectorWidth)"%(tc, tc)) + return False + + # Both TLU=False + TransposeLDS case, need GlobalLoadVectorWidth == LocalReadVectorWidth + if (not state["ProblemType"]["TLU%c"%tc]) and (not state["ProblemType"]["TLU%c"%tcOther]) and state["TransposeLDS"] and \ + state["GlobalLoadVectorWidth%c"%tc] != state["LocalReadVectorWidth"]: + reject(state, "DirectToVgpr%c does not supports TLUA=False and TLUB=False and GlobalLoadVectorWidth%c != LocalReadVectorWidth"%(tc, tc)) + return False + + # Does not work with TLU=False and PrefetchLocalRead=1 and VectorWidth>1 + if (not state["ProblemType"]["TLU%c"%tc]) and state["PrefetchLocalRead"] == 1 and state["VectorWidth"] > 1: + reject(state, "DirectToVgpr%c does not supports TLU=False and PrefetchLocalRead=1 and VectorWidth>1)"%(tc)) + return False + + # Does not work with TLU=False and VectorWidth>1 and other side of TLU=True + if (not state["ProblemType"]["TLU%c"%tc]) and state["VectorWidth"] > 1 and state["ProblemType"]["TLU%c"%tcOther]: + reject(state, "DirectToVgpr%c does not supports TLU%c=False VectorWidth>1 and TLU%c=True"%(tc, tc, tcOther)) + return False + + # Does not work with DirectToLDS + # -> this will be checked after DirectToLDS doable check is done + + return True + + ######################################## + # determine can we use DirectToLds + @staticmethod + def isDirectToLdsDoable(state, tc): + numBytes = state["ProblemType"]["DataType"].numBytes() + + # x2/x4 support for directToLds + + # numelements_perlane = 4/numBytes + # TN with transposeLDS feature should work as long as state["AssertSummationElementMultiple"] % (numelements_perlane*2) = 0 + # state["AssertSummationElementMultiple"] % (numelements_perlane*4) = 0 + + #NT + # use only for f32 & DGEMM and TLU = 1 + #TN + # use for all precisions with TransposeLDS=1 + + if state["ProblemType"]["DataType"].isHalf() and state["AssertSummationElementMultiple"] % (2 * state["GlobalLoadVectorWidth%c"%tc]) != 0: + reject(state, "can't use DirectToLds for FP16 with AssertSummationElementMultiple %u" % state["AssertSummationElementMultiple"]) + return False + + if state["ProblemType"]["DataType"].isBFloat16() and state["AssertSummationElementMultiple"] % (2 * state["GlobalLoadVectorWidth%c"%tc]) != 0: + reject(state, "can't use DirectToLds for BF16 with AssertSummationElementMultiple %u" % state["AssertSummationElementMultiple"]) + return False + + if state["NumThreads"] % state["WavefrontSize"] != 0: + reject(state, "can't use DirectToLds for NumThreads % WavefrontSize != 0") + return False + + # GLVW*BPe only for precision(s) < 4 (bpe) + #if (state["ProblemType"]["TLU%c"%tc] == True and numBytes < 4): + if (numBytes < 4): + if state["GlobalLoadVectorWidth%c"%tc] * numBytes != 4: + reject(state, "can't use DirectToLds for bpe < 4 and GlobalLoadVectorWidth * numBytes != 4"%tc) + return False + + if state["ProblemType"]["TLU%c"%tc] == state["UnrollMajorLDS%c" % tc]: + reject(state, "can't use DirectToLds for TLU%c == UnrollMajorLDS%c"%(tc, tc)) + return False + + # avoid picking x2&x4 for precisions < f32/f64 in [ProblemType][TLU] == TRUE + if not state["EnableMatrixInstruction"]: + if state["GlobalLoadVectorWidth%c"%tc] * numBytes * state["WavefrontSize"] > 256: + reject(state, "can't use DirectToLds for not EnableMatrixInstruction and GlobalLoadVectorWidth%c * bpe * WavefrontSize > 256"%tc) + return False + + # TODO revisit fp32 case for failure + #if state["ProblemType"]["TLU%c"%tc] and numBytes < 8 and state["GlobalLoadVectorWidth%c"%tc] * numBytes > 4: + if numBytes < 8 and state["GlobalLoadVectorWidth%c"%tc] * numBytes > 4: + reject(state, "can't use DirectToLds for TLU%c and bpe < 8 and GlobalLoadVectorWidth%c * bpe > 4"%(tc, tc)) + return False + + + if state["WaveSeparateGlobalRead%c" % tc]: + if state["LSC%c"%tc] * state["LSP%c"%tc] * numBytes != state["WavefrontSize"] * state["GlobalLoadVectorWidth%c"%tc] * numBytes: + reject(state, "can't use DirectToLds for LSC%c and LSP%c * bpe!= WavefrontSize * GlobalLoadVectorWidth%c * bpe > 4"%(tc, tc, tc)) + return False + else: + if state["LSC%c"%tc] * state["LSP%c"%tc] * numBytes != state["NumThreads"] * state["GlobalLoadVectorWidth%c"%tc] * numBytes: + reject(state, "can't use DirectToLds for LSC%c and LSP%c * bpe != NumThreads * GlobalLoadVectorWidth%c * bpe > 4"%(tc, tc, tc)) + return False + + if (state["LdsBlockSizePerPad%c"%tc] == 0) \ + and (state["LdsPad%c"%tc] != 0): +# and ((state["LSC%c"%tc] * numBytes) != (state["NumThreads"] * 4)): // TODO: +# and ((state["LSC%c"%tc] * numBytes) % (state["WavefrontSize"] * 4) != 0): + reject(state, "can't use DirectToLds for LdsBlockSizePerPad%c == 0 and LdsPad%c != 0"%(tc, tc)) + return False + + if (state["LdsBlockSizePerPad%c"%tc] != 0) \ + and (state["LdsPad%c"%tc] != 0) \ + and (state["LdsBlockSizePerPad%c"%tc] != state["WavefrontSize"] * state["GlobalLoadVectorWidth%c"%tc] * numBytes): +# and (state["LdsBlockSizePerPad%tc"] % (state["WavefrontSize"] * 4) != 0): // TODO: + reject(state, "can't use DirectToLds for LdsBlockSizePerPad%c != 0 and LdsPad%c != 0 and \ + LdsBlockSizePerPad%c != WavefrontSize * GlobalLoadVectorWidth%c * bpe"%(tc, tc, tc, tc)) + return False + + # so far, DirectToLds does not work well with PGR=2 + # performance is not good and a lot of ds_read for DTL can cause scheduling issue(need fix) + if state["PrefetchGlobalRead"] == 2 and not (state["DirectToVgprA"] or state["DirectToVgprB"]): + reject(state, "can't use DirectToLds for PrefetchGlobalRead == 2 without DirectToVgpr") + return False + + # so far, DirectToLds does not work with LRVW=2 + if state["LocalReadVectorWidth"] == 2: + reject(state, "can't use DirectToLds for LocalReadVectorWidth == 2") + return False + + # Does not work with (NumLoadsCoalesced>1 and UseInstOffsetForGRO) + DGEMM + if state["ProblemType"]["DataType"].isDouble() and \ + (state["NumLoadsCoalesced%c"%tc] > 1 and state["UseInstOffsetForGRO"]): + reject(state, "DirectToLds%c does not supports NumLoadsCoalesced%c > 1 and UseInstOffsetForGRO for dgemm"%(tc, tc)) + return False + + # Does not work with NumLoadsCoalesced>1 + ZGEMM + if state["ProblemType"]["DataType"].isDoubleComplex() and state["NumLoadsCoalesced%c"%tc] > 1: + reject(state, "DirectToLds%c does not supports NumLoadsCoalesced%c > 1 for zgemm"%(tc, tc)) + return False + + # Does not work with PrefetchGlobalRead=2 and PrefetchLocalRead=1 (cannot schedule DTL global read after local read) + if state["PrefetchGlobalRead"] == 2 and state["PrefetchLocalRead"] == 1: + reject(state, "DirectToLds%c does not work with PrefetchGlobalRead=2 and PrefetchLocalRead=1"%(tc)) + return False + + # DirectToLds does not work if MacroTile is not power of 2 + # LDS offset swap/rotate logic works only when MacroTile is power of 2 + mt = state["MacroTile%c"%tc] + if mt & (mt - 1) != 0: + reject(state, "can't use DirectToLds if MacroTile%s is not power of 2"%tc) + return False + + # DirectToLds does not work with TLU=False and bpe > bpr and DepthU//NumLoadsCoalesced < 8 + # bpe > bpr case, Lower and upper 4 bytes elements are stored separately. + # if TLU=False and DepthU//NumLoadsCoalesced is smaller than lower block size (8 elements), + # current offset swap logic does not work + if (not state["ProblemType"]["TLU%c"%tc]) and state["ProblemType"]["DataType"].numRegisters() > 1 and \ + state["DepthU"] // state["NumLoadsCoalesced%c"%tc] < 8: + reject(state, "DirectToLds%c does not work with TLU=False and bpe > bpr and DepthU//NumLoadsCoalesced%c < 8"%(tc, tc)) + return False + + return True + + @staticmethod + def getDivisorName(state, tC): + if state["GlobalReadCoalesceGroup{}".format(tC)]: + if state["GlobalReadCoalesceVector{}".format(tC)]: + divisorName = "LVC{}".format(tC) + else: + # Fractional load use the more accurate lsc, multiply by VW later + divisorName = "LSC{}".format(tC) + else: + if state["GlobalReadCoalesceVector{}".format(tC)]: + divisorName = "LSP{}".format(tC) + else: + divisorName = "LVP{}".format(tC) + return divisorName + + ######################################## + # assign all derived parameters + @staticmethod + def assignDerivedParameters(state): + + Solution.assignProblemIndependentDerivedParameters(state) + + if "AssignedDerivedParameters" in state: + if state["AssignedDerivedParameters"]: + return + state["AssignedDerivedParameters"] = False + + for s in Solution.InternalKeys: + state['_'+s] = state[s] + #del state[s] + + if ("_GlobalAccumulation" not in state) or ("_WorkspaceSizePerElemC" not in state): + state["_GlobalAccumulation"] = None + state["_WorkspaceSizePerElemC"] = 0 + if state["GlobalSplitU"] > 1: + computeName = state["ProblemType"]["ComputeDataType"].toName() + computeBytes = state["ProblemType"]["ComputeDataType"].numBytes() + + if state["GlobalSplitUAlgorithm"] == 'SingleBuffer': + if computeName != state["ProblemType"]["DestDataType"].toName(): + state["_GlobalAccumulation"] = 'SingleBuffer' + elif state["GlobalSplitUAlgorithm"] == 'MultipleBuffer': + state["_GlobalAccumulation"] = 'MultipleBuffer' + + if state["_GlobalAccumulation"] == 'SingleBuffer': + state["_WorkspaceSizePerElemC"] = computeBytes + elif state["_GlobalAccumulation"] == 'MultipleBuffer': + state["_WorkspaceSizePerElemC"] = computeBytes * state["GlobalSplitU"] + + if state["VectorStore"] == -1: + state["_VectorStore"] = 1 # default, may be changed if needed to generate a valid kernel + + ProblemType.assignDerivedParameters(state["ProblemType"]) + if not state["Valid"]: + print2("in assignDerivedParameters, state['Valid'] = False") + return + + # Init LoopIters parameter in case of early exit + # For backwards compatibility with older yaml files + state["LoopIters"] = 0 + if "LoopUnroll" in state: + state["LoopIters"] = state["LoopUnroll"] + + if state["ScheduleIterAlg"] == 2: + state["InnerUnroll"] = state["DepthU"] // state["MatrixInstK"] + state["PrefetchLocalRead"] = 1 + state["ExpandPointerSwap"] = 1 + state["1LDSBuffer"] = 1 + print2("\nSet SIA=2, force PrefetchLocalRead=1, ExpandPointerSwap=1, 1LDSBuffer=1") + + isa = tuple(state["ISA"]) + + if state["WavefrontSize"] == 32 and not globalParameters["ArchCaps"][isa]["HasWave32"]: + reject(state, "WavefrontSize=32 not supported for ISA {}".format(isa)) + + if state["WavefrontSize"] == 32 and state["KernelLanguage"] == "Source": + reject(state, "WavefrontSize=32 not yet supported for source kernels.") + + if state["EnableMatrixInstruction"]: + if not (state["ProblemType"]["DataType"].isSingle() \ + or state["ProblemType"]["DataType"].isDouble() \ + or state["ProblemType"]["DataType"].isBFloat16() \ + or state["ProblemType"]["DataType"].isHalf() \ + or state["ProblemType"]["DataType"].isComplex() \ + or state["ProblemType"]["DataType"].isInt8()): + reject(state, "didn't support Matrix Instruction with type %s" % str(state["ProblemType"]["DataType"])) + if not state["MIBlock"] or len(state["MIBlock"]) != 6: + reject(state, "invalid MIBlock") + if not state["MIWaveGroup"] or len(state["MIWaveGroup"]) != 2: + reject(state, "invalid MIWaveGroup") + if not state["MIWaveTile"] or len(state["MIWaveTile"]) != 2: + reject(state, "invalid MIWaveTile") + if not state["ProblemType"]["HighPrecisionAccumulate"] \ + and state["ProblemType"]["DataType"].numRegisters() < 1 : + reject(state, "Matrix instructions for half, bf16 (or i8) types are natively accumulated" + \ + " in fp32 (or i32) precision. Please add the following config:" + \ + "\n - HighPrecisionAccumulate: True") + else: + if not state["ProblemType"]["HighPrecisionAccumulate"] \ + and state["ProblemType"]["ComputeDataType"].numRegisters() > state["ProblemType"]["DataType"].numRegisters() : + reject(state, "For non-MI Kernel, if sizeof(ComputeDataType) > sizeof(DataType), " + \ + "Please add the following config:" + \ + "\n - HighPrecisionAccumulate: True") + + if state["ThreadTile0"] > 16 or state["ThreadTile1"] > 16: + reject(state, "Invalid value for ThreadTile") + + if state["ScheduleIterAlg"] == 2 or state["ScheduleIterAlg"] == 3: + reject(state, "SIA2 and SIA3 only support MatrixInstruction") + + if state["ProblemType"]["Tensor0"]==0: + state["ThreadTileA"] = state["ThreadTile0"] + state["ThreadTileB"] = state["ThreadTile1"] + state["SubGroupA"] = state["SubGroup0"] + state["SubGroupB"] = state["SubGroup1"] + state["MacroTileA"] = state["MacroTile0"] + state["MacroTileB"] = state["MacroTile1"] + if state["EnableMatrixInstruction"]: + state["MIWaveTileA"] = state["MIWaveTile"][0] + state["MIWaveTileB"] = state["MIWaveTile"][1] + else: + state["ThreadTileB"] = state["ThreadTile0"] + state["ThreadTileA"] = state["ThreadTile1"] + state["SubGroupB"] = state["SubGroup0"] + state["SubGroupA"] = state["SubGroup1"] + state["MacroTileB"] = state["MacroTile0"] + state["MacroTileA"] = state["MacroTile1"] + if state["EnableMatrixInstruction"]: + state["MIWaveTileA"] = state["MIWaveTile"][1] + state["MIWaveTileB"] = state["MIWaveTile"][0] + + Solution.checkAndAssignWaveSeparateGlobalRead(state, 'A') + Solution.checkAndAssignWaveSeparateGlobalRead(state, 'B') + + # Init vars early since there are early-exit return statements below + state["DirectToLdsA"] = False + state["DirectToLdsB"] = False + state["LocalWriteUseSgprA"] = False + state["LocalWriteUseSgprB"] = False + + state["WorkGroupMapping" ] = abs(state["WorkGroupMapping"]) + + problemType = state["ProblemType"] + + for (tc,batchMask) in (('A', 0x1), ('B', 0x2)): + freeDims = [i for i in problemType["IndexAssignments%s"%tc] if i in problemType["IndicesFree"]] + if not freeDims: + reject(state, "tensor%s contains no free indices.") + return False + + # Determine which indices will be packed together as this impacts several different parms (sizes, magic numbers, etc) + # The order in PackedC*Indices also determines the order that dimensions are packed - the first elements in + # the list are the fastest-moving elements. + # The store code optimizes for C0 being the coalesced dimension and C1 the perp dimension. + # C0/C1 indices can come from IndexAssignmentsA or IndexAssignmentsB + # grid size [0,1] + state["PackedC0IdxChars"] = [] + state["PackedC0IndicesX"] = [] + indexChars = globalParameters["IndexChars"] + # Pack all the dimensions (free) of A into grid[0] + + if problemType["Index0"] in problemType["IndexAssignmentsA"]: + tc0 = 'A' + tc1 = 'B' + else: + tc0 = 'B' + tc1 = 'A' + assert(isPackedIndex(state, problemType["Index01A"])) + assert(isPackedIndex(state, problemType["Index01B"])) + + # Pack all the dimensions (batch and free) of A into grid[0] + for idx in problemType["IndexAssignments%s"%tc0]: + if isPackedIndex(state, idx): + assert (idx < problemType["NumIndicesC"]) + state["PackedC0IdxChars"].append("%s" % indexChars[idx]) + state["PackedC0IndicesX"].append(idx) + + state["PackedC1IdxChars"] = [] + state["PackedC1IndicesX"] = [] + for idx in problemType["IndexAssignments%s"%tc1]: + if isPackedIndex(state, idx): + assert (idx < problemType["NumIndicesC"]) + state["PackedC1IdxChars"].append("%s" % indexChars[idx]) + state["PackedC1IndicesX"].append(idx) + + # If dims are packed, then need to ensure a global vector load isn't split by a tensor dim + # (since this could result in non-contiguous addresses) + # Current implementation ensures that the vector load is not partial across the Free* boundary: + # GlobalLoadVectorWidth=1 will always meet this requirement. + # (TODO - could make this more sophisticated if dims use default strides and are thus contiguous) + packedC0 = len(state["PackedC0IdxChars"])>1 + packedC1 = len(state["PackedC1IdxChars"])>1 + + bufferLoad = state["BufferLoad"] and state["KernelLanguage"] == "Assembly" + if not bufferLoad: + state["DirectToLds"] = False + state["_UseSgprForGRO"] = False + if state["PrefetchGlobalRead"] == 2: + reject(state, "BufferLoad=0 does not support PrefetchGlobalRead=2") + return + + if problemType["UseBias"]: + reject(state, "BufferLoad=0 does not support UseBias due to no suppress no load.") + return + + #These modes only work under certain conditions, apply them here: + # - The "NoLoad" loop is only generated if PrefetchGlobalRead>0 + # - And Suppress does not work if GSU>1 for some reason + if state["SuppressNoLoadLoop"] == 1: + if not (bufferLoad and state["PrefetchGlobalRead"] == 1 and (state["GlobalSplitU"]==1)): + state["SuppressNoLoadLoop"] = 0 + + if state["ExpandPointerSwap"] == 1: + # Pointer swap only used if PGR==1 or (PGR>1 and double/double complex) - so set ExpandPointerSwap=0 here + # So far, EPS=1 and PGR>1 works only with double/double complex. + # DirectToVgpr case, bufferLoad=False can work with ExpandPointerSwap=1 + #if not (bufferLoad and state["PrefetchGlobalRead"] == 1): + if not ((bufferLoad or state["DirectToVgprA"] or state["DirectToVgprB"]) and ( state["PrefetchGlobalRead"] == 1 \ + or (state["PrefetchGlobalRead"] > 1 and \ + (state["ProblemType"]["DataType"].isDouble() or state["ProblemType"]["DataType"].isDoubleComplex())))): + state["ExpandPointerSwap"] = 0 + # EPS not supported with SplitLDS yet + if state["DepthULdsDivisor"] > 1: + state["ExpandPointerSwap"] = 0 + + #print("PackedC0IdxChars", state["PackedC0IdxChars"]) + #print("PackedC1IdxChars", state["PackedC1IdxChars"]) + + # Set up stagger shift: + bpeAB = int(4*state["ProblemType"]["DataType"].numRegisters()) + # (1< 1: + # not supported with multiple summations, bug is maybe something with + # how stagger iteration is wrapped when unroll loop exits + state["StaggerU"] = 0 + + # Some restrictions for half: + if state["KernelLanguage"] == "Assembly" \ + and state["ProblemType"]["DataType"].isHalf(): + + # Vector-width must be at least 2 for Half (since unroll loop uses packed operations?) + if (not state["EnableMatrixInstruction"]) and state["VectorWidth"] < 2: + reject(state, "VectorWidth must be >= 2 for half") + if globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"]: + if not state["ProblemType"]["HighPrecisionAccumulate"] and state["AssertFree0ElementMultiple"] % 2 != 0: + # beta-on-edge has AF0EM requirement except for HPA kernels + reject(state, "Archs with HasEccHalf require AF0EM%2==0 except for HPA kernels") + + # Some restrictions for int8: + if state["KernelLanguage"] == "Assembly" and state["ProblemType"]["DataType"].isInt8(): + if (not state["EnableMatrixInstruction"]) and state["VectorWidth"] < 4: + reject(state, "VectorWidth must be >= 4 for Int8") + + # Default GlobalReadVectorWidth + if state["GlobalReadVectorWidth"] == -1: + state["GlobalReadVectorWidth"] = state["VectorWidth"] + + # Default GlobalStoreVectorWidth + if state["StoreVectorWidth"] == -1: + #TODO : re-enable later after running testlists + #state["StoreVectorWidth"] = state["VectorWidth"] + # use wider store for best store optimization + if state["SourceSwap"]: + state["StoreVectorWidth"] = state["VectorWidth"] + elif state["ProblemType"]["DataType"].numRegisters() <= 1: + state["StoreVectorWidth"] = 4 + else: + state["StoreVectorWidth"] = 4//state["ProblemType"]["DataType"].numRegisters() + + if state["EnableMatrixInstruction"]: + if state["SourceSwap"]: + if ((state["VectorWidth"] % state["StoreVectorWidth"]) != 0): + reject(state, "MFMA SourceSwap mode doesn't support vw(%u) with svw(%u)" % (state["VectorWidth"], state["StoreVectorWidth"])) + return + else: + if ((state["MIOutputVectorWidth"] % state["StoreVectorWidth"]) != 0): + reject(state, "MFMA non-SourceSwap mode doesn't support miovw(%u) with svw(%u)" % (state["MIOutputVectorWidth"], state["StoreVectorWidth"])) + return + + # reject - VW too big + if (state["VectorWidth"] * state["ProblemType"]["DataType"].numBytes()) > 16: + reject(state, "VW * DataType.numBytes() > 16") + return + + # reject - GRVW too big + if (state["GlobalReadVectorWidth"] * state["ProblemType"]["DataType"].numBytes()) > 16: + reject(state, "GRVW * DataType.numBytes() > 16") + return + + # LocalSplitU too large? + numElementsPerWorkGroup = state["MacroTile0"]*state["MacroTile1"] + + if numElementsPerWorkGroup < state["NumThreads"]: + reject(state, "NumElementsPerWorkGroup %u < NumThreads %u; reduce LocalSplitU" \ + % (numElementsPerWorkGroup, state["NumThreads"])) + return + + state["NumElementsPerThread"] = numElementsPerWorkGroup // state["NumThreads"] + state["GlobalWriteVectorWidth"] = min(state["VectorWidth"], state["NumElementsPerThread"] ) + if state["NumElementsPerThread"] % state["GlobalWriteVectorWidth"] != 0: + reject(state, "LSU NumElementsPerThread %u not divisible into GWVW %u" \ + % (state["NumElementsPerThread"], state["GlobalWriteVectorWidth"])) + return + state["NumGlobalWriteVectorsPerThread"] = state["NumElementsPerThread"] \ + // state["GlobalWriteVectorWidth"] + + + # LocalSplitU but can't NumThreads%MacroTile doesn't support sideways store + if state["LocalSplitU"] > 1: + if state["NumThreads"] % state["MacroTile0"] != 0: + reject(state, "LocalSplitU but NumThreads=%u not divisible by MT0=%u for sideways store" \ + % (state["NumThreads"], state["MacroTile0"])) + return + if state["MacroTile0"]*state["MacroTile1"] % state["NumThreads"] != 0: + reject(state, "LocalSplitU but MT0*MT1=%u elements doesn't divide into NumThreads=%u" \ + % (state["MacroTile0"]*state["MacroTile1"], state["NumThreads"])) + return + if state["ProblemType"]["DataType"].isInt8(): + reject(state, "int8 doesn't support LocalSplitU") + return + + # GlobalSplitU doesn't work with some other things: + if state["GlobalSplitU"] > 1: + # added GSU support for DGEMM + supported = \ + (state["ProblemType"]["DataType"].isSingle()) or \ + (state["ProblemType"]["DataType"].isDouble() and state["BufferStore"]) or \ + (state["ProblemType"]["DestDataType"].isInt32()) or \ + (state["KernelLanguage"] == "Assembly" and + (state["ProblemType"]["DataType"].isHalf() and not state["ProblemType"]["HighPrecisionAccumulate"]) or + (state["_GlobalAccumulation"]) + ) + if not supported: + reject(state, "GlobalSplitU only compatible with single or asm and (half or mixed) precision") + return + + if state["ProblemType"]["DataType"].isHalf() and state["KernelLanguage"] == "Assembly": + + if (not state["EnableMatrixInstruction"]) and state["VectorWidth"] < 2: + reject(state, "Assembly half requires VectorWidth >= 2 for non-MFMA mode") + + if state["GlobalSplitU"] > 1 and (not state["_GlobalAccumulation"]): + if state["AssertFree0ElementMultiple"] < 2: + reject(state, "Assembly GSU half requires AF0EM>=2 (for atomics on edge tiles)") + + ######################################## + # Initial DepthU + ######################################## + userDepthU = state["DepthU"] + # DepthU == -1 means glvw=1 + if state["DepthU"] == -1: + if state["MacroTile0"] != state["MacroTile1"]: + reject(state, "DepthU=0 requires square MacroTile") + return + + if userDepthU < 0: + depthU = 2 + depthULds = 2 + maxDepthU = globalParameters["MaxDepthU"] + numOfWaves = state["NumThreads"] // state["WavefrontSize"] + if state["ProblemType"]["TLUA"] and state["WaveSeparateGlobalReadA"]: + depthU = max(depthU, numOfWaves) + if state["ProblemType"]["TLUB"] and state["WaveSeparateGlobalReadB"]: + depthU = max(depthU, numOfWaves) + else: + depthU = userDepthU + depthULds = userDepthU//state["DepthULdsDivisor"] + maxDepthU = userDepthU + + state["_DepthULds"] = state["DepthU"]//state["DepthULdsDivisor"] # internal + + ######################################## + # Search DepthU + # Inputs: + # - depthU, userDepthU, state["LocalSplitU"], state["InnerUnroll"], state["MacroTile0/1"], state["GlobalReadVectorWidth"] + # - state["MatrixInstK"], ... + # Outputs: + # - totalVectorsCoalescedA, totalVectorsCoalescedB, totalElementsPerpA, totalElementsPerpB, state["DepthU"] + ####################################### + while True: # exit criteria at end + validDepthU = True + # peek LoopIters + loopIters = (depthULds // state["LocalSplitU"]) // state["InnerUnroll"] + if "MatrixInstK" in state: + loopIters //= state["MatrixInstK"] + if loopIters < 1: + reject(state, "LoopIters need to greater than 0") + return + + # Make sure the prefetch VGPR index plr[x] can be aligned for each loop + # for example, if PLR3 result in 4 VGPR: + # PGR - pre : plr[0], plr[1], plr[2] + # loop - iter0: plr[3], iter1: plr[0], iter2: plr[1], iter3: plr[2] -> restart LOOP (from plr[3]...) -> OK + # + # but if PLR2 result in 3 VGPR: + # PGR - pre : plr[0], plr[1] + # loop - iter0: plr[2], iter1: plr[0], iter2: plr[1], iter3: plr[2] -> restart LOOP (from plr[2]...) -> !! + if (depthULds % ((state["PrefetchLocalRead"]%loopIters)+1)) != 0: + validDepthU = False + + # how many elements to load + if state["ProblemType"]["TLUA"]: + totalElementsCoalescedA = state["MacroTileA"] + totalElementsPerpA = depthU + else: + totalElementsCoalescedA = depthU + totalElementsPerpA = state["MacroTileA"] + + if state["ProblemType"]["TLUB"]: + totalElementsCoalescedB = state["MacroTileB"] + totalElementsPerpB = depthU + else: + totalElementsCoalescedB = depthU + totalElementsPerpB = state["MacroTileB"] + + totalElementsA = totalElementsCoalescedA * totalElementsPerpA + totalElementsB = totalElementsCoalescedB * totalElementsPerpB + + GlobalReadVectorWidth = state["GlobalReadVectorWidth"] + if state["DirectToVgprA"]: + if not state["SourceSwap"]: + GlobalReadVectorWidth = 1 # adjust GlobalReadVectorWidth to 1 in DirectToVgpr case (except for DirectToVgprA + SourceSwap) + tva = totalElementsA // GlobalReadVectorWidth + if not Solution.setGlobalLoadVectorWidth(state, "A", tva, GlobalReadVectorWidth): + validDepthU = False + GlobalReadVectorWidth = state["GlobalReadVectorWidth"] + tvb = totalElementsB // GlobalReadVectorWidth + if not Solution.setGlobalLoadVectorWidth(state, "B", tvb, GlobalReadVectorWidth): + validDepthU = False + + if state["EnableMatrixInstruction"] and state["GlobalLoadVectorWidthA"]: + partialA = state["ProblemType"]["TLUA"] and (state["AssertFree0ElementMultiple"] % state["GlobalLoadVectorWidthA"] != 0) + if partialA: + glvwAlimit = 16 // state["ProblemType"]["DataType"].numBytes() + if state["SourceSwap"]: + matrixInstM = (state["MatrixInstM"] * state["MatrixInstBM"]) if (state["MatrixInstM"] == 4) else state["MatrixInstM"] + glvwAlimit = matrixInstM * state["VectorWidth"] + else: + matrixInstN = (state["MatrixInstN"] * state["MatrixInstBN"]) if (state["MatrixInstN"] == 4) else state["MatrixInstN"] + glvwAlimit = state["MIOutputVectorWidth"] * (state["WavefrontSize"] // matrixInstN) + + # reduce GLVA if GLVA larger than MIOVW + if state["GlobalLoadVectorWidthA"] > glvwAlimit: + tva = totalElementsA // glvwAlimit + if not Solution.setGlobalLoadVectorWidth(state, "A", tva, glvwAlimit): + validDepthU = False + + if state["EnableMatrixInstruction"] and state["GlobalLoadVectorWidthB"]: + partialB = state["ProblemType"]["TLUB"] and (state["AssertFree1ElementMultiple"] % state["GlobalLoadVectorWidthB"] != 0) + if partialB: + glvwBlimit = 16 // state["ProblemType"]["DataType"].numBytes() + if state["SourceSwap"]: + matrixInstM = (state["MatrixInstM"] * state["MatrixInstBM"]) if (state["MatrixInstM"] == 4) else state["MatrixInstM"] + glvwBlimit = state["MIOutputVectorWidth"] * (state["WavefrontSize"] // matrixInstM) + # else: # use origin shiftptr for B + # matrixInstN = (state["MatrixInstN"] * state["MatrixInstBN"]) if (state["MatrixInstN"] == 4) else state["MatrixInstN"] + # glvwBlimit = matrixInstN # not support state["VectorWidth"] for B yet + + # reduce GLVB if GLVB larger than MIOVW + if state["GlobalLoadVectorWidthB"] > glvwBlimit: + tvb = totalElementsB // glvwBlimit + if not Solution.setGlobalLoadVectorWidth(state, "B", tvb, glvwBlimit): + validDepthU = False + + if validDepthU and state["KernelLanguage"] == "Assembly" \ + and (state["ProblemType"]["DataType"].isHalf() \ + or state["ProblemType"]["DataType"].isBFloat16()): + if globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"]: + if state["GlobalLoadVectorWidthA"] == 1 or state["GlobalLoadVectorWidthB"] == 1: + reject(state, "HalfEcc requires GLVWA > 1") + + # TODO- Need this restrict ? + if validDepthU and state["KernelLanguage"] == "Assembly" and state["ProblemType"]["DataType"].isInt8(): + if state["GlobalLoadVectorWidthA"] < 4: + reject(state, "Int8 requires GLVWA >= 4, current is %u"%state["GlobalLoadVectorWidthA"]) + if state["GlobalLoadVectorWidthB"] < 4: + reject(state, "Int8 requires GLVWB >= 4, current is %u"%state["GlobalLoadVectorWidthB"]) + + + # Now convert elements to vectors based on GlobalReadVectorWidth + GlobalLoadVectorWidthA = state["GlobalLoadVectorWidthA"] + GlobalLoadVectorWidthB = state["GlobalLoadVectorWidthB"] + if GlobalLoadVectorWidthA == 0: + GlobalLoadVectorWidthA = GlobalReadVectorWidth + if GlobalLoadVectorWidthB == 0: + GlobalLoadVectorWidthB = GlobalReadVectorWidth + totalVectorsCoalescedA = totalElementsCoalescedA // GlobalLoadVectorWidthA + totalVectorsCoalescedB = totalElementsCoalescedB // GlobalLoadVectorWidthB + totalVectorsA = totalElementsA // GlobalLoadVectorWidthA + totalVectorsB = totalElementsB // GlobalLoadVectorWidthB + + if 0: + print("info:", pvar(state, "NumThreads"), pvar(state, "DepthU"), pvar(state, "DepthULdsDivisor"), + "TT=%ux%u" % (state["ThreadTile0"], state["ThreadTile1"]), + "WG=%ux%u" % (state["WorkGroup"][0], state["WorkGroup"][1]), + "MT=%ux%u" % (state["MacroTile0"], state["MacroTile1"])) + print("info: totalElementsCoalescedA=", totalElementsCoalescedA, + " totalVectorsCoalescedA=", totalVectorsCoalescedA, " totalVectorsA=", totalVectorsA) + print("info: totalElementsCoalescedB=", totalElementsCoalescedB, + " totalVectorsCoalescedB=", totalVectorsCoalescedB, " totalVectorsB=", totalVectorsB) + print("info", pvar(state, "VectorWidth") + , pvar(state, "GlobalLoadVectorWidthA"), pvar(state, "GlobalLoadVectorWidthB")) + + #if state["ProblemType"]["DataType"].isHalf() \ + # and (state["GlobalLoadVectorWidthA"] == 1 \ + # or state["GlobalLoadVectorWidthB"] == 1): + # validDepthU = False + + if userDepthU == -1: # no vectors + if state["GlobalLoadVectorWidthA"] != 1 \ + or state["GlobalLoadVectorWidthB"] != 1: + validDepthU = False + elif userDepthU == -2: + if max( state["GlobalLoadVectorWidthA"], \ + state["GlobalLoadVectorWidthB"]) \ + < state["GlobalReadVectorWidth"]: + validDepthU = False + elif userDepthU <= -3: + if min( state["GlobalLoadVectorWidthA"], \ + state["GlobalLoadVectorWidthB"]) \ + < state["GlobalReadVectorWidth"]: + validDepthU = False + + if validDepthU: + if not state["ProblemType"]["TLUA"]: + if depthU < state["GlobalLoadVectorWidthA"]: + validDepthU = False + + if not state["ProblemType"]["TLUB"]: + if depthU < state["GlobalLoadVectorWidthB"]: + validDepthU = False + + # this depthU is valid, done unless user wants to double (for TN) + if validDepthU: + if userDepthU < -3: # for every int below -3, use next doubled value + userDepthU += 1 + depthU *= 2 + depthULds = 2 + continue + else: # use this found value + state["DepthU"] = depthU + state["_DepthULds"] = depthU//state["DepthULdsDivisor"] + break + + # this depthU not valid + else: + # keep looking + if depthU < maxDepthU: + depthU += 2 + depthULds = depthU//state["DepthULdsDivisor"] + continue + # give up + else: + reject(state, "No valid DepthU found") + return + ######################################## + # end DepthU loop + ######################################## + + assert(state["DepthU"]> 0) + + if not Solution.setGlobalLoadTileDimClassic(state, "A", state["NumLoadsA"], \ + totalVectorsCoalescedA, totalElementsPerpA): + return + if not Solution.setGlobalLoadTileDimClassic(state, "B", state["NumLoadsB"], \ + totalVectorsCoalescedB, totalElementsPerpB): + return + + # allow LocalReadVectorWidth for TLU + MatrixInstruction + # so far, limited to double + (DTVB or (DTVA + no DTL)) only + # some more limitations necessary to make this logic work + # - SourceSwap + # - VectorWidth >= LocalReadVectorWidth + # - AssertFree1ElementMultiple % VectorWidth == 0 (no shift edge for B) + # - the other side of MIWaveTile must be multiple of VectorWidth + state["allowLRVWforTLUandMI"] = state["ProblemType"]["DataType"].isDouble() and \ + (state["DirectToVgprB"] or state["DirectToVgprA"] and not state["DirectToLds"]) and \ + state["EnableMatrixInstruction"] and state["ProblemType"]["TLUA"] and state["ProblemType"]["TLUB"] and \ + state["VectorWidth"] >= state["LocalReadVectorWidth"] and \ + state["AssertFree1ElementMultiple"] % state["VectorWidth"] == 0 and \ + ((state["DirectToVgprA"] and (state["MIWaveTile"][1] % state["VectorWidth"] == 0)) or \ + (state["DirectToVgprB"] and (state["MIWaveTile"][0] % state["VectorWidth"] == 0))) + + # Determine if we can load directly-to-Vgpr + if state["DirectToVgprA"]: + if not Solution.isDirectToVgprDoable(state, 'A'): + return # rejected + if state["DirectToVgprB"]: + if not Solution.isDirectToVgprDoable(state, 'B'): + return # rejected + + # TODO + if (0 and state["LSCA"] % state["GlobalLoadVectorWidthA"] != 0): + reject(state, "lsca % grvw != 0") + return + if (0 and state["LSPA"] % state["GlobalLoadVectorWidthA"] != 0): + reject(state, "lspa % grvw != 0") + return + if (0 and state["LSCB"] % state["GlobalLoadVectorWidthB"] != 0): + reject(state, "lscb % grvw != 0") + return + if (0 and state["LSPB"] % state["GlobalLoadVectorWidthB"] != 0): + reject(state, "lspb % grvw != 0") + return + + state["LVCA"] = roundupRatio(state["LSCA"] , state["GlobalLoadVectorWidthA"]) + state["LVPA"] = roundupRatio(state["LSPA"] , state["GlobalLoadVectorWidthA"]) + state["LVCB"] = roundupRatio(state["LSCB"] , state["GlobalLoadVectorWidthB"]) + state["LVPB"] = roundupRatio(state["LSPB"] , state["GlobalLoadVectorWidthB"]) + + for tc in ('A','B'): + if problemType["TLU%s"%tc]: + pos = problemType["IndexAssignments%s"%tc].index(problemType["Index01%s"%tc]) + else: + pos = problemType["IndexAssignments%s"%tc].index(problemType["IndexUnroll"]) + + # Some of these might become 0? + if 0: + print("info: ", pvar(state, "LVCA"), pvar(state, "LVPA"), \ + pvar(state, "LVCB"), pvar(state, "LVPB")) + + # lds buffer size for A, B + if state["KernelLanguage"] == "Source" and \ + state["LdsPadA"] != state["LdsPadB"]: + reject(state, "Source KernelLanguage only supports LdsPadA == LdsPadB") + return + + # NoTailLoop condition check + # So far, NoTailLoop option is not exposed. + validNoTailLoop = True + invalidComment = "" + if not bufferLoad: + validNoTailLoop = False + invalidComment = "does not support BufferLoad=0" + if not (state["ProblemType"]["TLUA"] or state["ProblemType"]["TLUB"]): + validNoTailLoop = False + invalidComment = "does not support TLUA=False and TLUB=False" + # NoTailLoop parameter initialization. Set True for the following cases + # 1. ASEM is multiple of DepthU. TailLoop code will not be used in this case. + # 2. DirectToVgpr is enabled + # Except for case 1, validNoTailLoop should be True to enable NoTailLoop. + # Otherwise, it will be rejected. + state["NoTailLoop"] = False + if state["AssertSummationElementMultiple"] % state["DepthU"] == 0: + state["NoTailLoop"] = True + elif state["DirectToVgprA"] or state["DirectToVgprB"]: + if not validNoTailLoop: + reject(state, "DirectToVgpr + AssertSummationElementMultiple%%DepthU!=0 %s to enable NoTailLoop"%invalidComment) + return + else: + state["NoTailLoop"] = True + + ######################################## + # LDS + ######################################## + state["UnrollMajorLDSA"] = state["TransposeLDS"] and (not state["ProblemType"]["TLUA"]) + state["UnrollMajorLDSB"] = state["TransposeLDS"] and (not state["ProblemType"]["TLUB"]) + + if state["LdsBlockSizePerPad"] == -1: + if state["MatrixInstruction"] and state["TransposeLDS"]: + state["LdsBlockSizePerPad"] = 128 + if state["_DepthULds"]*state["ProblemType"]["DataType"].numBytes() > state["LdsBlockSizePerPad"]: + state["LdsBlockSizePerPad"] = int(2**(math.ceil(math.log(state["_DepthULds"]*state["ProblemType"]["DataType"].numBytes(), 2)))) + else: + state["LdsBlockSizePerPad"] = 0 + + state["LdsBlockSizePerPadA"] = state["LdsBlockSizePerPad"] if state["UnrollMajorLDSA"] else 0 + state["LdsBlockSizePerPadB"] = state["LdsBlockSizePerPad"] if state["UnrollMajorLDSB"] else 0 + + if state["EnableMatrixInstruction"]: + if state["LdsBlockSizePerPadA"]: + if not state["UnrollMajorLDSA"]: + reject(state, "didn't support LdsBlockSizePerPadA on tile major LDS yet") + if state["LdsBlockSizePerPadA"] < state["_DepthULds"]*state["ProblemType"]["DataType"].numBytes(): + reject(state, "reject: DepthULds %u x bpe > LdsBlockSizePerPadA %u" % (state["_DepthULds"], state["LdsBlockSizePerPad"])) + + if state["LdsBlockSizePerPadB"]: + if not state["UnrollMajorLDSB"]: + reject(state, "didn't support LdsBlockSizePerPadB on tile major LDS yet") + if state["LdsBlockSizePerPadB"] < state["_DepthULds"]*state["ProblemType"]["DataType"].numBytes(): + reject(state, "reject: DepthULds %u x bpe > LdsBlockSizePerPadB %u" % (state["_DepthULds"], state["LdsBlockSizePerPad"])) + else: + if state["UnrollMajorLDSA"] or state["UnrollMajorLDSB"]: + reject(state, "didn't support UnrollMajorLDS in VALU mode yet") + if state["LdsBlockSizePerPadA"] != 0 or state["LdsBlockSizePerPadB"] != 0: + reject(state, "didn't support LdsBlockSizePerPad in VALU mode yet") + + # Determine if we can load directly-to-LDS. + # Transpose requires a trip through registers to perform the transpose so can't use DirectToLdsA + # LDS loads always write 4 bytes apart so can use only 4-byte operations + # TODO - for doubles we need to add something special here? + # The matrix must not require transposing since that is done by reading to VGPR and writing in different order + # The LSC (load size coalesced) must load some multiple of 256 bytes since that is what each DirectToLds load provides + # Note for these matrices LSC is same as MacroTile dim + # MatrixInstruction rules: + # DirectToLDS is supported for TLU=0 (make sure transposeLDS=1) + # LDS (load size coalesced) * LSPA must load some multiple of 256 bytes. + # added support for loadX2/loadx4 . + # x2/x4 (use x4 for better load efficiency) + # x2/x4 support for NT layout for f64/f32 TN layout (f16/bf16/f32/f64) with transposeLDS=1 + # ignore x2/x4 precision < 4 bytes in NT layout + if state["DirectToLds"]: + # DirectToLdsA + DirectToVgprA does not work. Not enable DirectToLdsA if DirectToVgprA is true + if (not state["DirectToVgprA"]) and Solution.isDirectToLdsDoable(state, 'A'): + state["DirectToLdsA"] = True + state["LocalWriteUseSgprA"] = True + #print("DirectToLdsA", state["DirectToLdsA"]) + + # DirectToLdsB + DirectToVgprB does not work. Not enable DirectToLdsB if DirectToVgprB is true + if (not state["DirectToVgprB"]) and Solution.isDirectToLdsDoable(state, 'B'): + state["DirectToLdsB"] = True + state["LocalWriteUseSgprB"] = True + #print("DirectToLdsB", state["DirectToLdsB"]) + + # Update parent variable so kernel display is accurate + state["DirectToLds"] = state["DirectToLdsA"] or state["DirectToLdsB"] + if state["1LDSBuffer"] == -1 and state["DirectToLds"]: + #1LDS buffer must be 0 for DirectToLdsA + state["1LDSBuffer"] = 0 + + # set NoLdsWriteCode if DirectToLds + DirectToVgpr or DirectToLdsA+B is enabled + state["NoLdsWriteCode"] = False + if (state["DirectToVgprA"] and state["DirectToLdsB"]) or (state["DirectToVgprB"] and state["DirectToLdsA"]) or \ + (state["DirectToLdsA"] and state["DirectToLdsB"]): + state["NoLdsWriteCode"] = True + + # Default LocalReadVectorWidth + if state["LocalReadVectorWidth"] == -1: + if state["EnableMatrixInstruction"] and not state["allowLRVWforTLUandMI"]: + state["LocalReadVectorWidth"] = state["MIInputPerThread"] + # enable less than state["MIInputPerThread"] + # for fp64 this means ds_read_b32 + if ((state["DirectToLdsA"] and state["ProblemType"]["TLUA"]) or \ + (state["DirectToLdsB"] and state["ProblemType"]["TLUB"])): + state["LocalReadVectorWidth"] = 1 if (state["ProblemType"]["DataType"].numBytes() >= 4) else state["LocalReadVectorWidth"] + else: + state["LocalReadVectorWidth"] = state["VectorWidth"] + else: + if state["EnableMatrixInstruction"]: + # support LocalReadVectorWidth < miInputPerThread for directToLdsX2/X4 + if state["LocalReadVectorWidth"] < state["MIInputPerThread"] and not (state["DirectToLdsA"] or state["DirectToLdsB"]): + reject(state, "LocalReadVectorWidth < %u" %(state["MIInputPerThread"])) + if state["LocalReadVectorWidth"] > state["MIInputPerThread"] and not state["TransposeLDS"] \ + and not state["allowLRVWforTLUandMI"]: + reject(state, "LocalReadVectorWidth require Transpose LDS") + else: + if state["LocalReadVectorWidth"] != state["VectorWidth"]: + reject(state, "LocalReadVectorWidth must equal VectorWidth for non MI kernels") + + # set pad as readRegs to avoid unaligned read + optPad = state["LocalReadVectorWidth"] + readRegs = state["LocalReadVectorWidth"]*state["ProblemType"]["DataType"].numBytes()//4 + if readRegs > 4: + reject(state, "LocalReadVectorWidth=%u results in attemping to read LDS larger than b128, reject") + if state["EnableMatrixInstruction"]: + # for readRegs = 1 or 4, we need to double pad for MI16x16xNx1 to avoid bank conflict. + if state["MatrixInstB"] == 1 and state["MatrixInstM"] == 16 and \ + (readRegs == 4 or readRegs == 1): + optPad *= 2 + if state["LdsPadA"] == -1: + if state["ProblemType"]["TLUA"]: + state["LdsPadA"] = 0 + else: + if state["EnableMatrixInstruction"] and state["TransposeLDS"]: + state["LdsPadA"] = max(state["GlobalReadVectorWidth"],optPad) + else: + state["LdsPadA"] = state["VectorWidth"] + ## turn-off padding for directToLds + if state["EnableMatrixInstruction"] and state["TransposeLDS"] and state["DirectToLdsA"]: + state["LdsPadA"] = 0 + assert(state["LdsPadA"] >= 0) + if state["LdsPadB"] == -1: + if state["ProblemType"]["TLUB"]: + state["LdsPadB"] = 0 + else: + if state["EnableMatrixInstruction"] and state["TransposeLDS"]: + state["LdsPadB"] = max(state["GlobalReadVectorWidth"],optPad) + else: + state["LdsPadB"] = state["VectorWidth"] + if state["EnableMatrixInstruction"] and state["TransposeLDS"] and state["DirectToLdsB"]: + state["LdsPadB"] = 0 + assert(state["LdsPadB"] >= 0) + + if (state["UnrollMajorLDSA"] or state["UnrollMajorLDSB"]) and (not state["EnableMatrixInstruction"]): + reject(state, "UnrollMajorLDS Supports only in EnableMatrixInstruction=1") + + ldsAlign = int(64 / state["ProblemType"]["DataType"].numRegisters()) + + if state["UnrollMajorLDSA"]: + ldsNumElementsA = (state["_DepthULds"] + state["LdsPadA"]) * state["MacroTileA"] + padInterval = state["LdsBlockSizePerPadA"] // bpeAB + if padInterval != 0: + ldsNumElementsA = int((state["_DepthULds"] * state["MacroTileA"]) / padInterval * (padInterval + state["LdsPadA"])) + ldsNumElementsAlignedA = roundUpToNearestMultiple(ldsNumElementsA, ldsAlign) + else: + ldsNumElementsA = state["_DepthULds"] * (state["MacroTileA"] + state["LdsPadA"]) + ldsNumElementsAlignedA = roundUpToNearestMultiple(ldsNumElementsA, ldsAlign) + if state["DirectToVgprA"]: + # DirectToVgpr does not use LDS. Set to 0. + ldsNumElementsA = 0 + ldsNumElementsAlignedA = 0 + + if state["UnrollMajorLDSB"]: + ldsNumElementsB = (state["_DepthULds"] + state["LdsPadB"]) * state["MacroTileB"] + padInterval = state["LdsBlockSizePerPadB"] // bpeAB + if padInterval != 0: + ldsNumElementsB = int((state["_DepthULds"] * state["MacroTileB"]) / padInterval * (padInterval + state["LdsPadB"])) + ldsNumElementsAlignedB = roundUpToNearestMultiple(ldsNumElementsB, ldsAlign) + else: + ldsNumElementsB = state["_DepthULds"] * (state["MacroTileB"] + state["LdsPadB"]) + ldsNumElementsAlignedB = roundUpToNearestMultiple(ldsNumElementsB, ldsAlign) + if state["DirectToVgprB"]: + # DirectToVgpr does not use LDS. Set to 0. + ldsNumElementsB = 0 + ldsNumElementsAlignedB = 0 + + # todo, can the alignment be a power of 2? + state["LdsOffsetA"] = 0 + if state["PrefetchGlobalRead"]: + state["LdsNumElementsAlignedA"] = ldsNumElementsAlignedA + state["LdsNumElementsAlignedB"] = ldsNumElementsAlignedB + state["LdsOffsetB"] = state["LdsOffsetA"] + state["LdsNumElementsAlignedA"] + + offsetBlk = state["LdsOffsetB"] + ldsNumElementsAlignedB + offsetBlk = int(2**(math.ceil(math.log(offsetBlk, 2)))) + + state["LdsOffsetA_Blk"] = offsetBlk + state["LdsOffsetB_Blk"] = state["LdsOffsetA_Blk"] + state["LdsNumElementsAlignedA"] + ldsNumElementsAB = state["LdsOffsetB_Blk"]+ ldsNumElementsB + else: + state["LdsOffsetB"] = ldsNumElementsAlignedA + ldsNumElementsAB = ldsNumElementsAlignedA + ldsNumElementsB + + # lds buffer size for reduction + ldsNumElementsReduction = state["LocalSplitU"]*state["MacroTile0"]*state["MacroTile1"] if state["LocalSplitU"] > 1 else 0 + + # lds max occupancy + ldsSizeOccupancy = globalParameters["DeviceLDS"] // state["MaxOccupancy"] + ldsNumElementsOccupancy = ldsSizeOccupancy // state["ProblemType"]["DestDataType"].numBytes() + + #print("ldsNumElementsA", ldsNumElementsA) + #print("ldsNumElementsB", ldsNumElementsB) + #print("ldsNumElementsAlignedA", ldsNumElementsAlignedA) + #print("ldsNumElementsAlignedB", ldsNumElementsAlignedB) + #print("ldsNumElementsAB", ldsNumElementsAB) + + if state["EnableMatrixInstruction"]: + if state["DirectToLds"] and state["1LDSBuffer"]: + reject(state, "1LDSBuffer must be 0 for directToLds") + + if state["1LDSBuffer"] == -1: + if ldsNumElementsAB * state["ProblemType"]["DataType"].numBytes() > globalParameters["MaxLDS"]: + state["1LDSBuffer"] = 1 + else: + state["1LDSBuffer"] = 0 + + if state["1LDSBuffer"]: + if not state["PrefetchGlobalRead"]: + reject(state, "PGR=0 already use 1 LDS buffer only") + # Should be able to support as long as NO scheduleLocalWrite + if (not state["ScheduleIterAlg"] == 2) and (not state["ScheduleIterAlg"] == 3) and (state["ScheduleLocalWrite"]): + reject(state, "1LDSBuffer only support SIA2 or SIA3, or SIA1 without SLW") + state["LdsOffsetB"] = ldsNumElementsAlignedA + ldsNumElementsAB = ldsNumElementsAlignedA + ldsNumElementsB + + # lds size is the greater of the two + ldsNumElements = max(ldsNumElementsAB, ldsNumElementsReduction, ldsNumElementsOccupancy) + + if state["StoreRemapVectorWidth"] == -1: + # use de_read_b64 as default in storeRemap to avoid bank conflict + defaultRemap = 8 // state["ProblemType"]["DestDataType"].numBytes() + defaultRemap = max(defaultRemap, state["MacroTile0"]//state["WavefrontSize"]) + ldsRemapPad = max(defaultRemap, state["MIOutputVectorWidth"]) + ldsNumElementsRemapC = (state["MacroTile0"]+ldsRemapPad)* state["MatrixInstN"] * state["MIWaveGroup"][1] + if state["_GlobalAccumulation"]: + computeBytes = state["ProblemType"]["ComputeDataType"].numBytes() + ldsNumElementsRemapC *= (computeBytes / state["ProblemType"]["DestDataType"].numBytes()) + ldsSize = ldsNumElementsRemapC * state["ProblemType"]["DestDataType"].numBytes() + if not math.log(state["MacroTile0"],2).is_integer() or \ + ldsSize > globalParameters["MaxLDS"] or \ + state["SourceSwap"] or \ + (state["GlobalSplitU"] > 1) and (state["_GlobalAccumulation"] != 'MultipleBuffer') or \ + state["MatrixInstBN"] > 1 and state["MatrixInstN"] == 4: + state["StoreRemapVectorWidth"] = 0 + else: + state["StoreRemapVectorWidth"] = defaultRemap + + # GuaranteeNoPartial + if state["ProblemType"]["TLUA"]: + state["GuaranteeNoPartialA"] = state["AssertFree0ElementMultiple"]%state["GlobalLoadVectorWidthA"]==0 + else: + state["GuaranteeNoPartialA"] = True + + if state["ProblemType"]["TLUB"]: + state["GuaranteeNoPartialB"] = state["AssertFree1ElementMultiple"]%state["GlobalLoadVectorWidthB"]==0 + else: + state["GuaranteeNoPartialB"] = True + + # SourceSwap + if state["SourceSwap"]: + if not state["EnableMatrixInstruction"]: + reject(state, "SourceSwap only applies to MatrixInstruction kernels") + return + if state["StoreRemapVectorWidth"]: + reject(state, "SourceSwap not compatible with StoreRemap") + return + + # check if need to use lds init Acc vgprs + state["LdsInitCVgprs"] = False + if globalParameters["ArchCaps"][isa]["HasAccCD"] and \ + state["EnableMatrixInstruction"] and state["StorePriorityOpt"] and \ + state["ProblemType"]["DataType"].isDouble(): + state["LdsInitCVgprs"] = True + + if state["MIArchVgpr"]: + if not globalParameters["ArchCaps"][isa]["HasAccCD"] or \ + not state["EnableMatrixInstruction"]: + reject(state, "MIArchVgpr requires gcn support ACC_CD bit for MatrixInstruction") + return + + if not (state["ProblemType"]["ComputeDataType"].isDouble() or \ + state["ProblemType"]["ComputeDataType"].isSingle() or \ + (state["ProblemType"]["ComputeDataType"].isHalf() and state["ProblemType"]["HighPrecisionAccumulate"]) or \ + state["ProblemType"]["ComputeDataType"].isInt32() or \ + state["ProblemType"]["ComputeDataType"].isDoubleComplex()): + reject(state, "MIArchVgpr now only support fp64, fp32, fp16, int8 MatrixInstruction.") + return + + if state["ProblemType"]["Fp16AltImpl"]: + if not (state["ProblemType"]["DataType"].isHalf() and \ + state["ProblemType"]["HighPrecisionAccumulate"] and \ + state["EnableMatrixInstruction"]): + reject(state, "Fp16AltImpl requires FP16 HPA MFMA") + return + + #check not support cases and calculate lds resources + if state["StoreRemapVectorWidth"]: + if not state["EnableMatrixInstruction"]: + reject(state, "storeRemap only support MatrixInstruction kernel") + return + if (state["GlobalSplitU"] > 1) and (state["_GlobalAccumulation"] != 'MultipleBuffer'): + reject(state, "storeRemap doesn't support GlobalSplitU yet, except GSU algorithm 2") + return + if packedC0 or packedC1: + reject(state, "storeRemap doesn't support packedC0 and packedC1 yet") + return + if state["MatrixInstBN"] > 1 and state["MatrixInstN"] == 4: + reject(state, "storeRemap doesn't support MI4x4 multi blocks in N direction yet") + return + if not math.log(state["MacroTile0"],2).is_integer(): + reject(state, "storeRemap only supports power-of-2 MT0") + # TODO - this return should be here, but this is a hotfix, + # Somehow we have a "Validation Failed" kernel in rocBLAS now (SRVW=4 and MT0=96) and this will stop the whole building process + # Actions: 1. Hotfix, comment out this "return" temporarily for that invalidated kernel + # 2. Remove / replace that invalidated kernel + # 3. Put back this return + # 4. How to design a better way to prevent from invalid kernel in rocBLAS? + # return + + storeInstMinWidth = 1 # minimum dwordx1 + storeInstMaxWidth = 4 # maximum dwordx4 + srMinVw = max(storeInstMinWidth, int(storeInstMinWidth/state["ProblemType"]["DestDataType"].numRegisters())) + numReg = state["ProblemType"]["DestDataType"].numRegisters() + if state["_GlobalAccumulation"]: + numReg = state["ProblemType"]["ComputeDataType"].numRegisters() + + srMaxVw = int(storeInstMaxWidth/numReg) + if srMinVw > state["StoreRemapVectorWidth"] or srMaxVw < state["StoreRemapVectorWidth"]: + reject(state, "StoreRemapVectorWidth %u is not allowed for this data type" % state["StoreRemapVectorWidth"]) + return + + if state["StoreRemapVectorWidth"] * state["WavefrontSize"] < state["MacroTile0"]: + reject(state, "storeRemap: Per wave single global write instruction doesn't enough to write one M column." + \ + " Please use larger StoreRemapVectorWidth.") + return + if (state["MacroTile0"]*state["MatrixInstN"])//state["MIWaveGroup"][0] < state["StoreRemapVectorWidth"]*state["WavefrontSize"]: + reject(state, "storeRemap: number elements of lds less than per wave per local read elements." + \ + " Please use smaller StoreRemapVectorWidth.") + return + ldsRemapPad = max(state["StoreRemapVectorWidth"],state["MIOutputVectorWidth"]) + ldsNumElementsRemapC = (state["MacroTile0"]+ldsRemapPad)* state["MatrixInstN"] * state["MIWaveGroup"][1] + + if state["_GlobalAccumulation"]: + computeBytes = state["ProblemType"]["ComputeDataType"].numBytes() + multiplier = computeBytes // state["ProblemType"]["DataType"].numBytes() + elif state["ProblemType"]["DestDataType"].numBytes() > state["ProblemType"]["DataType"].numBytes(): + # Determine ratio of output to input element size. + # SRVW remaps output so we need to scale up resources. + multiplier = state["ProblemType"]["DestDataType"].numBytes() // state["ProblemType"]["DataType"].numBytes() + else: + multiplier = 1 + + ldsNumElementsRemapC *= multiplier + + #print("ldsNumElementsRemapC=%u" % ldsNumElementsRemapC) + + # if LDS is bound by RemapC (SRVW), then 1LDSBuffer actually doesn't help in SIA3 + # since LDS usage couldn't be reduced + if state["1LDSBuffer"] and (state["ScheduleIterAlg"] == 3) and (ldsNumElements < ldsNumElementsRemapC): + # TODO- Remove this DataType test condition, + # Currently we do this test is just because we don't want to affect existing logic in rocBLAS + if state["ProblemType"]["DataType"].isInt8(): + reject(state, "LDS usage is bound be StoreRemap, thus 1LDSBuffer wouldn't have any help. Skip.") + return + + ldsNumElements = max(ldsNumElements, ldsNumElementsRemapC) + + state["LdsNumElements"] = ldsNumElements + ldsSize = ldsNumElements * state["ProblemType"]["DataType"].numBytes() + if ldsSize > globalParameters["MaxLDS"]: + reject(state, "Kernel Uses %u > %u bytes of LDS" % ( ldsSize, globalParameters["MaxLDS"])) + return + + # LoopUnroll = DepthU / LocalSplitU + if "LocalSplitU" in state and "_DepthULds" in state: + state["LoopUnroll"] = state["_DepthULds"] // state["LocalSplitU"] + if state["LoopUnroll"] * state["LocalSplitU"] != state["_DepthULds"]: + state["Valid"] = False + if state["KernelLanguage"] != "Assembly" and state["InnerUnroll"] != 1: + reject(state, "InnerUnroll only supported on assembly") + state["LoopUnroll"] //= state["InnerUnroll"] + + if 0: + print("info: ", pvar(state, "LoopUnroll"), " LDS Stats:", pvar(state, "LdsOffsetA"), pvar(state, "LdsOffsetB")) + print("info: ", pvar(state["ProblemType"], "TLUA"), \ + pvar(state, "NumLoadsCoalescedA"), pvar(state, "NumLoadsPerpendicularA"), \ + pvar(state, "LSCA"), pvar(state, "LSPA")) + print("info:", pvar(state["ProblemType"], "TLUB"), \ + pvar(state, "NumLoadsCoalescedB"), pvar(state, "NumLoadsPerpendicularB"), \ + pvar(state, "LSCB"), pvar(state, "LSPB")) + + state["LoopIters"] = state["LoopUnroll"] + if "MatrixInstK" in state: + state["LoopIters"] //= state["MatrixInstK"] + + if state["LoopIters"] < 1: + reject(state, "LoopIters need to greater than 0") + return + + # PLR > 2xLoopIters is redundant setting + # TODO- Why need to x2 ? Why not (if state["PrefetchLocalRead"] >= state["LoopIters"]:) + if state["PrefetchLocalRead"] >= 2*state["LoopIters"]: + reject(state, "Reject since PrefetchLocalRead %u >= 2x LoopIters %u" % (state["PrefetchLocalRead"],state["LoopIters"])) + + # reject low performance + if state["PrefetchLocalRead"]%state["LoopIters"] > 1: + reject(state, "PrefetchLocalRead: %u, LoopIters: %u performance is low" % (state["PrefetchLocalRead"],state["LoopIters"])) + + # prefetch wider read iteration > LoopIters, no enough iterations for prefetching + if state["EnableMatrixInstruction"] and state["PrefetchLocalRead"] > 0: + # Multiple = WLR-size / input-size = how many iters could be covered by one WLR ? + wlrMultiple = state["LocalReadVectorWidth"]//state["MIInputPerThread"] + if wlrMultiple == 0: + reject(state, "LocalReadVectorWidth %u is less than MIInput" % (state["LocalReadVectorWidth"])) + return + # for example, if the original ds_read is b32... + # 1. if LoopIters = 5 (b32 x 5 times), WLR-Multiple = 2 (b64), then we can fit the WLR + # 2. if LoopIters = 2 (b32 x 2 times), WLR-Multiple = 4 (b128), this is not allowed + # 3. if LoopIters = 2 (b32 x 2 times), WLR-Multiple = 2 (b64), this is allowed + if state["LoopIters"] % wlrMultiple != 0: + reject(state, "LocalReadVectorWidth %u cannot be distributed evenly, LoopIters %u should be divisible by WLR-Multiple %u" \ + % (state["LocalReadVectorWidth"], state["LoopIters"], wlrMultiple)) + + PLR = (state["PrefetchLocalRead"] % state["LoopIters"]) + if PLR != 0 and state["LoopIters"] - (PLR * wlrMultiple) <= 0 : + reject(state, "with PrefetchLocalRead %u LoopIters %u LocalReadVectorWidth %u, not enough LoopIters to prefetch %ux%u iterations, " \ + % (state["PrefetchLocalRead"],state["LoopIters"],state["LocalReadVectorWidth"], PLR , wlrMultiple) ) + + # # reject conditions with lower performance + # if state["ScheduleIterAlg"] == 2 and \ + # (state["ExpandPointerSwap"] != 1 or state["LoopIters"] != 1 or state["ScheduleGlobalRead"] != 1): + # reject(state, "ScheduleIterAlg 2 only work with EPS1_SGR1, LoopIter=1") + + if state["TransposeLDS"] == 1: + if not state["EnableMatrixInstruction"]: + reject(state, "TransposeLds Supports only in MatrixInstruction=1") + if state["ProblemType"]["TLUA"] and state["ProblemType"]["TLUB"]: + # TODO: Now in rocBLAS, lot of logic yamls are Type=NT and TLDS=1? Why aren't they rejected and how to get rid of them? + reject(state, "TransposeLds requires TLUA=0 or TLUB=0") + if state["EnableMatrixInstruction"]: + # enable widerLocalRead + if state["LocalReadVectorWidth"] > state["MIInputPerThread"]: + # wider localRead support 2 types + # 1. prefetch all lds to register + # 2. using larger InnerUnroll + if not (state["PrefetchLocalRead"] >= state["LoopIters"] and state["InnerUnroll"] == 1) and \ + not state["InnerUnroll"] >= state["LocalReadVectorWidth"] // state["MIInputPerThread"]: + reject(state, "wider localRead only support (PrefetchLocalRead %u >= LoopIters %u) or (InnerUnroll %u > LocalReadxN)" % (state["PrefetchLocalRead"],state["LoopIters"],state["InnerUnroll"])) + + if state["DepthULdsDivisor"] > 1: + if state["PrefetchGlobalRead"] == 2: + reject(state, "DepthULdsDivisor > 1 does not support PrefetchGlobalRead=2") + if state["ScheduleIterAlg"] != 3: + reject(state, "DepthULdsDivisor > 1 does not support SchedulIterAlg other than 3") + if state["DirectToLds"] == True: + reject(state, "DepthULdsDivisor > 1 does not support DirectToLds") + if state["ProblemType"]["TLUA"] or state["ProblemType"]["TLUA"] or not state["TransposeLDS"]: + reject(state, "DepthULdsDivisor > 1: Only works with TN problem layout and TransposeLDS") + if state["PrefetchGlobalRead"]==1 and state["PrefetchLocalRead"]==0: + reject(state, "PGR1 + PLR0 in SplitLDS requires double G2L buffer which is yet to be implemented") + if state["ProblemType"]["DataType"].numRegisters()*state["GlobalReadVectorWidth"] < state["DepthULdsDivisor"]: + reject(state, "SplitLDS requires wider GlobalReadVectorWidth; needs RegisterPerElem (%f) * GRVW (%u) >= DepthULdsDivisor (%u)"% + (state["ProblemType"]["DataType"].numRegisters(),state["GlobalReadVectorWidth"],state["DepthULdsDivisor"])) + + if state["GlobalReadPerMfma"] > 1 and state["PrefetchGlobalRead"] == 2: + reject(state, "GlobalReadPerMfma need to be 1 if PGR2") + + if state["UseInstOffsetForGRO"] == -1: + state["UseInstOffsetForGRO"] = 1 if state["DirectToLds"] else 0 + + # guard against out of bounds reads + # None: don't guard against ou + # ShiftPtr: shift read pointers to be in bounds, then unshift registers (source & assembly), + # ShiftPtr does not support very small problem dims < global load vector width since the shift + # would move outside the array bounds. + # If GLVW==1 or Assert*ElementMultiple for the coalesced dim is > GRVW, then shifting is not + # necessary and the shift/unshift code will not be generated + state["EdgeType"] = "ShiftPtr" # Use ShiftPtr by default + + # Precise bounds check uses the "num_records" field in the buffer to + # precisely detect when we are inbounds or not. Only a one-dimensional + # check is used since this is faster and also for computation we only + # need to ensure that none of the loads fault. threads which are + # computing bogus sections of the C tile will later be ignored. + # precise checking only works when all elements of the load are in-bounds + # since if the vload crosses boundary we ignore all components not just the + # ones that are OOB. See comments for groOffsetInMacroTile in KernelWriterAssembly.py + # + # So check for the cases where the unroll loop can + # generate partial loads here and reject PBC solutions: + # For non-TLU the free dim is in perp dim - should always be TRUE? TODO + + #-- + # ShiftPtr can't use UseSgprForGRO since it needs to modify the VGPR pointers + if bufferLoad and state["_UseSgprForGRO"] and state["EdgeType"]=="ShiftPtr": + if not state["GuaranteeNoPartialA"] or not state["GuaranteeNoPartialB"]: + state["_UseSgprForGRO"] = False + #reject(state, "PBC with wide load has insufficient overlap guarantees- try GRVW=1 or adding appropriate Assert*ElementMultiple") + + + + + if state["EnableMatrixInstruction"]: + cont1 = not state["GuaranteeNoPartialB"] + cont2 = ((state["MatrixInstN"] % state["GlobalLoadVectorWidthB"]) != 0) + if cont1 and cont2: + reject(state, "MatrixInstN %u %% GlobalLoadVectorWidthB %u must be 0" % \ + (state["MatrixInstN"], state["GlobalLoadVectorWidthB"])) + else: + if not bufferLoad or not state["GuaranteeNoPartialA"]: + # Restrict GRVW/VW combos so shift-ptr logic will work + if state["GlobalLoadVectorWidthA"] > 1 \ + and state["GlobalLoadVectorWidthA"] != state["VectorWidth"]: + reject(state, "GlobalLoadVectorWidthA %u must be == VectorWidth %u or == 1" % \ + (state["GlobalLoadVectorWidthA"], state["VectorWidth"])) + + if not bufferLoad or not state["GuaranteeNoPartialB"]: + # Restrict GRVW/VW combos so shift-ptr logic will work + if state["GlobalLoadVectorWidthB"] > 1 \ + and state["GlobalLoadVectorWidthB"] != state["VectorWidth"]: + reject(state, "GlobalLoadVectorWidthB %u must be == VectorWidth %u or == 1" % \ + (state["GlobalLoadVectorWidthB"], state["VectorWidth"])) + + # Use SGPR to store an offset from GlobalReadOffsetA+0. + # (as opposed to using dedicated VGPR for each GRO + # Requires preciseBounds check since we rely on the buffer bounds check, not + # individual vector registers doing bounds compares. + + if state["_UseSgprForGRO"] == -1: + # Don't use SGPR if it looks like we might not have enough - better to leave PBC enabled even if we have to use VGPR + # 40 is based on current SGPR usage, this may need to be tuned in the future: + numLoadsA = state["NumLoadsCoalescedA"]*state["NumLoadsPerpendicularA"] + numLoadsB = state["NumLoadsCoalescedB"]*state["NumLoadsPerpendicularB"] + if numLoadsA + numLoadsB > 35 or state["DirectToVgprA"] or state["DirectToVgprB"]: # force _UseSgprForGRO = 0 if DirectToVgpr is enabled + #print "info: Disabling UseSgprForGRO since predicting too many SGPR will be used" + state["_UseSgprForGRO"] = 0 + else: + state["_UseSgprForGRO"] = 1 + + + if packedC0 and not state["GuaranteeNoPartialA"]: + reject(state, "packedC0 requires GuaranteeNoPartialA") + if packedC1 and not state["GuaranteeNoPartialB"]: + reject(state, "packedC1 requires GuaranteeNoPartialB") + + if packedC0 or packedC1: + state["_UseSgprForGRO"] = 0 + + if state["EdgeType"] != "ShiftPtr": + reject(state, "Packed dims requires EdgeType==ShiftPtr") + if state["KernelLanguage"] == "Assembly": + if not bufferLoad: + reject(state, "Packed dims for Assembly requires BufferLoad") + + if packedC0: # VectorWidth must not span tensor dim + if state["KernelLanguage"] == "Source": + if state["AssertFree0ElementMultiple"]=VectorWidth (for loads and stores)") + else: + if state["AssertFree0ElementMultiple"]=VectorWidth or not VectorStore (for stores)") + + state["AssignedDerivedParameters"] = True + + # UnrollLoopEfficiencyEnable does not work with f16/bf16/int8x4 + if globalParameters["UnrollLoopEfficiencyEnable"] and (state["ProblemType"]["DataType"].isHalf() or \ + state["ProblemType"]["DataType"].isBFloat16() or state["ProblemType"]["DataType"].isInt8x4()): + reject(state, "UnrollLoopEfficiencyEnable does not support f16/bf16/int8x4") + + # UnrollLoopEfficiencyEnable supports only ThreadTile0,1=[6,4] or [4,6] or [4,4] or [6.6] or [8,4] or [4,8] + if globalParameters["UnrollLoopEfficiencyEnable"] and \ + not ((state["ThreadTile0"] == 6 and state["ThreadTile1"] == 4) or \ + (state["ThreadTile0"] == 4 and state["ThreadTile1"] == 6) or \ + (state["ThreadTile0"] == 4 and state["ThreadTile1"] == 4) or \ + (state["ThreadTile0"] == 6 and state["ThreadTile1"] == 6) or \ + (state["ThreadTile0"] == 8 and state["ThreadTile1"] == 4) or \ + (state["ThreadTile0"] == 4 and state["ThreadTile1"] == 8)): + reject(state, "UnrollLoopEfficiencyEnable does not support ThreadTile0,1 = [%u,%u]"%(state["ThreadTile0"], state["ThreadTile1"])) + + # Activation + # Function call is set to false if GSU != 1 or Activation is not fused or ActivationType is not All. + if not ((state["GlobalSplitU"] == 1) and state["ActivationFused"] and state["ProblemType"]["ActivationType"] == 'all') \ + and state["ActivationFuncCall"]: + state["ActivationFuncCall"] = False + + ######################################## + # create a dictionary with booleans on whether to include parameter in name + @staticmethod + def getMinNaming(objs): + nonCKObjs = [obj for obj in objs if not isCustomKernelConfig(obj)] + + # early return + if len(nonCKObjs) == 0: + return {} + + # determine keys + requiredParameters = {} + if isinstance(nonCKObjs[0], Solution): + keys = list(nonCKObjs[0]._state.keys()) + else: + keys = list(nonCKObjs[0].keys()) + # only 1, rather than name being nothing, it'll be everything + if len(nonCKObjs) == 1: + for key in keys: + if key in list(validParameters.keys()): + requiredParameters[key] = False + else: + for key in keys: + required = False + if key in list(validParameters.keys()): + for i in range(1, len(nonCKObjs)): + if nonCKObjs[0][key] != nonCKObjs[i][key]: + required = True + break + if required: + requiredParameters[key] = True + else: + requiredParameters[key] = False + + if "MatrixInstM" in nonCKObjs[0]._state: + # Use MIWaveGroup and MIWaveTile instead of WG and MT + requiredParameters["MIWaveTile"] = True + requiredParameters["ThreadTile"] = False + + requiredParameters["ProblemType"] = False # always prepended + requiredParameters["MacroTile0"] = False # always prepended + requiredParameters["MacroTile1"] = False # always prepended + requiredParameters["DepthU"] = False # always prepended + requiredParameters["MatrixInstruction"] = False # always prepended + requiredParameters["MatrixInstM"] = False # always prepended + requiredParameters["MatrixInstN"] = False # always prepended + requiredParameters["MatrixInstK"] = False # always prepended + requiredParameters["MatrixInstB"] = False # always prepended + requiredParameters["MatrixInstBM"] = False # always prepended + requiredParameters["MatrixInstBN"] = False # always prepended + requiredParameters["CustomKernelName"] = False # Will not affect naming + requiredParameters["Fp16AltImpl"] = False # Will show up as a different type + + requiredParameters["Kernel"] = True # distinguish kernels from solutions + # for single-source compilation + return requiredParameters + + ######################################## + @ staticmethod + def getNameFull(state): + requiredParameters = {} + for key in state: + if key in list(validParameters.keys()): + requiredParameters[key] = True + if "MatrixInstM" in state: + # Use MIWaveGroup and MIWaveTile instead of WG and MT + requiredParameters["MIWaveTile"] = True + requiredParameters["ThreadTile"] = False + return Solution.getNameMin(state, requiredParameters) + + ######################################## + # Get Name Min + @ staticmethod + def getNameMin(state, requiredParameters): + if isCustomKernelConfig(state): + return state["CustomKernelName"] + + name = "" + first = True + # put problem first + if "ProblemType" in state: + name += str(state["ProblemType"]) + "_" + if "MacroTile0" in state \ + and "MacroTile1" in state \ + and "DepthU" in state: + name += "%s%ux%ux%u_" \ + % ( Solution.getParameterNameAbbreviation("MacroTile"), \ + state["MacroTile0"], state["MacroTile1"], state["DepthU"] ) + if "MatrixInstM" in state: + name += "%s%ux%ux%ux%u_" \ + % ( Solution.getParameterNameAbbreviation("MatrixInstruction"), \ + state["MatrixInstM"], state["MatrixInstN"], state["MatrixInstK"], state["MatrixInstB"]) + name += "SN_" # LdcEqualsLdd Removed + for key in sorted(state.keys()): + if key in requiredParameters and key[0] != '_': + if requiredParameters[key] and key != "CustomKernelName": + if not first: + name += "_" + else: + first = False + name += "%s%s" % ( Solution.getParameterNameAbbreviation(key), \ + Solution.getParameterValueAbbreviation(key, state[key]) ) + return name + + ######################################## + # create a dictionary of lists of parameter values + @staticmethod + def getSerialNaming(objs): + data = {} + for objIdx in range(0, len(objs)): + obj = objs[objIdx] + for paramName in sorted(obj.keys()): + if paramName in list(validParameters.keys()): + paramValue = obj[paramName] + if paramName in data: + if paramValue not in data[paramName]: + data[paramName].append(paramValue) + else: + data[paramName] = [ paramValue ] + maxObjs = 1 + for paramName in data: + if not isinstance(data[paramName][0],dict): + data[paramName] = sorted(data[paramName]) + maxObjs *= len(data[paramName]) + numDigits = len(str(maxObjs)) + return [ data, numDigits ] + + ######################################## + # Get Name Serial + @ staticmethod + def getNameSerial(state, serialNaming): + data = serialNaming[0] + numDigits = serialNaming[1] + + serial = 0 + multiplier = 1 + for paramName in sorted(state.keys()): + if paramName in list(validParameters.keys()): + paramValue = state[paramName] + paramData = data[paramName] + paramNameMultiplier = len(paramData) + if paramValue in paramData: + paramValueIdx = paramData.index(paramValue) + serial += paramValueIdx * multiplier + multiplier *= paramNameMultiplier + name = "%s%0*u" % ("S" if isinstance(state, Solution) else "K", \ + numDigits, serial) + return name + + + ######################################## + @ staticmethod + def getParametersIndented(state, indent): + s = "" + s += "%sProblemType: %s\n" % (indent, str(state["ProblemType"])) + for key in sorted(state): + s += "%s%s: %s\n" % (indent, str(key), str(state[key])) + return s + + ######################################## + @ staticmethod + def getParameterNameAbbreviation( name ): + return ''.join([c for c in name if not c.islower()]) + + ######################################## + @ staticmethod + def getParameterValueAbbreviation( key, value ): + if key == 'ISA': + return str(value[0]) + str(value[1]) + ('%x' % value[2]) + elif isinstance(value, str): + return ''.join([c for c in value if c.isupper()]) + elif isinstance(value, bool): + return "1" if value else "0" + elif isinstance(value, int): + if value >= 0: + return "%u" % value + else: # -1 -> n1 + return "n%01u" % abs(value) + elif isinstance(value, ProblemType): + return str(value) + elif isinstance(value, tuple): + abbrev = "" + for i in range(0, len(value)): + abbrev += str(value[i]) + return abbrev + elif isinstance(value, list): + abbrev = "" + for i in range(0, len(value)): + abbrev += Solution.getParameterValueAbbreviation(key, value[i]) + if i < len(value)-1: + abbrev += "_" + return abbrev + elif isinstance(value, dict): + s = "_".join(["%d%d"%(pos,k) for pos,k in value.items()]) + return s + elif isinstance(value, float): + val1 = int(value) + val2 = int(round(value*100)) - int(value)*100 + if val2 > 0: + s = "%dp%s" % (val1,str(val2).zfill(2)) + else: + s = "%d" % (val1) + return s + else: + printExit('Parameter {key}={value} is new object type ({t})'.format(key=key, value=value, t=type(value))) + return str(value) + + + ########################## + # make class look like dict + def keys(self): + return list(self._state.keys()) + + def __len__(self): + return len(self._state) + + def __iter__(self): + return iter(self._state) + + def __getitem__(self, key): + return self._state[key] + + def __setitem__(self, key, value): + self._name = None + self._state[key] = value + + def __str__(self): + if self._name is None: + self._name = Solution.getNameFull(self._state) + return self._name + + def __repr__(self): + return self.__str__() + + def getAttributes(self): + return deepcopy(self._state) + + def __hash__(self): + return hash(str(self)) + #return hash(self.getAttributes()) + + def __eq__(self, other): + #return isinstance(other, Solution) and self.getAttributes() == other.getAttributes() + return isinstance(other, Solution) and str(self) == str(other) + + def __ne__(self, other): + result = self.__eq__(other) + if result is NotImplemented: + return result + return not result + + @property + def enabledSplitLDS(self): + return self["DepthULdsDivisor"] > 1 + + @property + def enabledSetPrioSplitLDS(self): + # The interaction between SplitLDS's priority policy and StorePriorityOpt's is yet to be + # investigated. For now, disable SplitLDS's priority policy when StorePriorityOpt is present + # TODO: determine suitable priority policy when both are present + return self.enabledSplitLDS and not self["StorePriorityOpt"] diff --git a/tensilelite/Tensile/Source/CMakeLists.txt b/tensilelite/Tensile/Source/CMakeLists.txt new file mode 100644 index 0000000000..e817d62201 --- /dev/null +++ b/tensilelite/Tensile/Source/CMakeLists.txt @@ -0,0 +1,85 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +cmake_minimum_required(VERSION 3.13) + +# Override all paths arguments as they do not work properly +file(TO_CMAKE_PATH "$ENV{ROCM_PATH}" ROCM_PATH_ENV_VALUE) +list(APPEND CMAKE_PREFIX_PATH ${ROCM_PATH_ENV_VALUE} /opt/rocm) + +project(Tensile) + +set(TENSILE_USE_HIP ON CACHE BOOL "Use the Hip runtime.") +if(Tensile_LIBRARY_FORMAT MATCHES "msgpack") + set(TENSILE_USE_MSGPACK ON CACHE BOOL "Use message pack for parsing config files.") +else() + set(TENSILE_USE_LLVM ON CACHE BOOL "Use LLVM for parsing config files.") +endif() +set(TENSILE_USE_OPENMP ON CACHE BOOL "Use OpenMP to improve performance.") +set(TENSILE_STATIC_ONLY ON CACHE BOOL "Disable exposing Tensile symbols in a shared library.") + +if(NOT DEFINED CXX_VERSION_STRING) + if(CMAKE_CXX_COMPILER MATCHES ".*/hipcc$") + # Determine if CXX Compiler is hip-clang or nvcc + execute_process(COMMAND ${CMAKE_CXX_COMPILER} "--version" OUTPUT_VARIABLE CXX_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE) + string(REGEX MATCH "[A-Za-z]* ?clang version" TMP_CXX_VERSION ${CXX_OUTPUT}) + string(REGEX MATCH "[A-Za-z]+" CXX_VERSION_STRING ${TMP_CXX_VERSION}) + endif() +endif() + +if(CMAKE_CXX_COMPILER STREQUAL "hipcc") + set(TENSILE_GPU_ARCHS gfx803 gfx900 gfx906:xnack- gfx908:xnack- gfx90a:xnack- gfx1010 gfx1011 gfx1012 gfx1030 gfx1100 gfx1101 gfx1102 CACHE STRING "GPU architectures") +else() + set(TENSILE_GPU_ARCHS gfx803 gfx900 gfx906 gfx908 gfx90a gfx1010 gfx1011 gfx1012 gfx1030 gfx1100 gfx1101 gfx1102 CACHE STRING "GPU architectures") +endif() + +include(CMakeDependentOption) +CMAKE_DEPENDENT_OPTION(TENSILE_BUILD_CLIENT "Build the benchmarking client" ON + "TENSILE_USE_HIP" OFF) + +if(TENSILE_USE_HIP) + find_package(HIP REQUIRED CONFIG PATHS ${ROCM_PATH_ENV_VALUE} /opt/rocm) +endif() + +if(TENSILE_USE_OPENMP) + # Workaround for https://gitlab.kitware.com/cmake/cmake/-/issues/21787 + # ensures we link to HIP's libomp and get an rpath to it. + add_library(custom_openmp_cxx INTERFACE) + + if(TENSILE_USE_HIP) + target_compile_options(custom_openmp_cxx INTERFACE "-fopenmp") + target_link_options(custom_openmp_cxx INTERFACE "-fopenmp") + else () + find_package(OpenMP REQUIRED) + target_link_libraries(custom_openmp_cxx INTERFACE OpenMP::OpenMP_CXX) + endif () +endif() + +add_subdirectory(lib) + +if(TENSILE_BUILD_CLIENT) + add_subdirectory(client) +endif() + +export(TARGETS TensileHost NAMESPACE Tensile:: FILE TensileExports.cmake) + diff --git a/tensilelite/Tensile/Source/EnableWarnings.cmake b/tensilelite/Tensile/Source/EnableWarnings.cmake new file mode 100644 index 0000000000..c4a98f799b --- /dev/null +++ b/tensilelite/Tensile/Source/EnableWarnings.cmake @@ -0,0 +1,236 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +# - Enable warning all for gcc/clang or use /W4 for visual studio + +## Strict warning level +if (MSVC) + # Use the highest warning level for visual studio. + set(CMAKE_CXX_WARNING_LEVEL 4) + if (CMAKE_CXX_FLAGS MATCHES "/W[0-4]") + string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + else () + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") + endif () + + set(CMAKE_C_WARNING_LEVEL 4) + if (CMAKE_C_FLAGS MATCHES "/W[0-4]") + string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + else () + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4") + endif () + + # 4201 - nameless struct/union: used for complex.y, complex.y + # 4100 - unused formal parameter: overriding functions don't have to use all parameters + # 4715 - not all controll paths return a value: switch with returns where all enums handled + # 4127 - conditional is constant: cpu algorithm uses if template for complex conjugate + set (WARNINGS_TO_IGNORE "/wd4201 /wd4100 /wd4715 /wd4127") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WARNINGS_TO_IGNORE}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${WARNINGS_TO_IGNORE}") + +else() + foreach(COMPILER C CXX) + set(CMAKE_COMPILER_WARNINGS) + # use -Wall for gcc and clang + list(APPEND CMAKE_COMPILER_WARNINGS + -Wall + -Wextra + -Wcomment + -Wendif-labels + -Wformat + -Winit-self + -Wreturn-type + -Wsequence-point + -Wshadow + -Wswitch + -Wtrigraphs + -Wundef + -Wuninitialized + -Wunreachable-code + -Wunused + + -Wno-sign-compare + ) + if (CMAKE_${COMPILER}_COMPILER_ID MATCHES "Clang") + list(APPEND CMAKE_COMPILER_WARNINGS + -Wabi + -Wabstract-final-class + -Waddress + -Waddress-of-array-temporary + -Waddress-of-temporary + -Waggregate-return + -Wambiguous-macro + -Wambiguous-member-template + -Wanonymous-pack-parens + -Warray-bounds + -Warray-bounds-pointer-arithmetic + -Wassign-enum + -Watomic-properties + -Wattributes + -Wavailability + -Wbackslash-newline-escape + -Wbad-array-new-length + -Wbad-function-cast + -Wbind-to-temporary-copy + -Wbuiltin-macro-redefined + -Wbuiltin-requires-header + -Wcast-align + -Wcast-qual + -Wchar-align + -Wcomments + -Wcompare-distinct-pointer-types + -Wconditional-type-mismatch + -Wconditional-uninitialized + -Wconfig-macros + -Wconstant-logical-operand + -Wconstexpr-not-const + -Wconversion-null + -Wcovered-switch-default + -Wctor-dtor-privacy + -Wdangling-field + -Wdangling-initializer-list + -Wdelete-incomplete + -Wdeprecated + -Wdivision-by-zero + -Wduplicate-decl-specifier + -Wduplicate-enum + -Wduplicate-method-arg + -Wduplicate-method-match + -Wdynamic-class-memaccess + -Wempty-body + -Wenum-compare + -Wexplicit-ownership-type + -Wextern-initializer + -Wfloat-equal + -Wgnu-array-member-paren-init + -Wheader-guard + -Wheader-hygiene + -Widiomatic-parentheses + -Wignored-attributes + -Wimplicit-conversion-floating-point-to-bool + -Wimplicit-exception-spec-mismatch + -Wimplicit-fallthrough + -Wincompatible-library-redeclaration + -Wincompatible-pointer-types + # -Winherited-variadic-ctor + -Winline + -Wint-conversions + -Wint-to-pointer-cast + -Winteger-overflow + -Winvalid-constexpr + -Winvalid-noreturn + -Winvalid-offsetof + -Winvalid-pch + -Winvalid-pp-token + -Winvalid-source-encoding + -Winvalid-token-paste + -Wloop-analysis + -Wmain + -Wmain-return-type + -Wmalformed-warning-check + -Wmethod-signatures + -Wmismatched-parameter-types + -Wmismatched-return-types + -Wmissing-declarations + -Wmissing-format-attribute + -Wmissing-include-dirs + -Wmissing-sysroot + -Wmissing-variable-declarations + -Wnarrowing + -Wnested-externs + -Wnewline-eof + -Wnon-pod-varargs + -Wnon-virtual-dtor + -Wnull-arithmetic + -Wnull-character + -Wnull-dereference + -Wodr + -Wold-style-definition + -Wout-of-line-declaration + -Wover-aligned + -Woverflow + -Woverriding-method-mismatch + -Wpacked + -Wpointer-sign + -Wpointer-to-int-cast + -Wpointer-type-mismatch + -Wpredefined-identifier-outside-function + -Wredundant-decls + -Wreinterpret-base-class + -Wreserved-user-defined-literal + -Wreturn-stack-address + -Wsection + -Wserialized-diagnostics + -Wshift-count-negative + -Wshift-count-overflow + -Wshift-overflow + -Wshift-sign-overflow + # -Wsign-compare + -Wsign-promo + -Wsizeof-pointer-memaccess + -Wstack-protector + -Wstatic-float-init + -Wstring-compare + -Wstrlcpy-strlcat-size + -Wstrncat-size + -Wswitch-default + -Wswitch-enum + -Wsynth + -Wtautological-compare + -Wtentative-definition-incomplete-type + -Wthread-safety + -Wtype-limits + -Wtype-safety + -Wtypedef-redefinition + -Wtypename-missing + -Wundefined-inline + -Wundefined-reinterpret-cast + -Wunicode + -Wunicode-whitespace + -Wunused-exception-parameter + -Wunused-macros + -Wunused-member-function + -Wunused-parameter + -Wunused-volatile-lvalue + -Wused-but-marked-unused + -Wuser-defined-literals + -Wvarargs + -Wvector-conversions + -Wvexing-parse + -Wvisibility + -Wvla + -Wweak-template-vtables + # -Wweak-vtables + -Wwrite-strings + ) + else() + list(APPEND CMAKE_COMPILER_WARNINGS + -Wno-missing-field-initializers + ) + endif() + + foreach(FLAG ${CMAKE_COMPILER_WARNINGS}) + if(NOT CMAKE_${COMPILER}_FLAGS MATCHES ${FLAG}) + set(CMAKE_${COMPILER}_FLAGS "${CMAKE_${COMPILER}_FLAGS} ${FLAG}") + endif() + endforeach() + endforeach() +endif () diff --git a/tensilelite/Tensile/Source/FindHIP.cmake b/tensilelite/Tensile/Source/FindHIP.cmake new file mode 100644 index 0000000000..a1752f7607 --- /dev/null +++ b/tensilelite/Tensile/Source/FindHIP.cmake @@ -0,0 +1,111 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +include(CMakeFindDependencyMacro OPTIONAL RESULT_VARIABLE _CMakeFindDependencyMacro_FOUND) +if (NOT _CMakeFindDependencyMacro_FOUND) + macro(find_dependency dep) + if (NOT ${dep}_FOUND) + set(cmake_fd_version) + if (${ARGC} GREATER 1) + set(cmake_fd_version ${ARGV1}) + endif() + set(cmake_fd_exact_arg) + if(${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION_EXACT) + set(cmake_fd_exact_arg EXACT) + endif() + set(cmake_fd_quiet_arg) + if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY) + set(cmake_fd_quiet_arg QUIET) + endif() + set(cmake_fd_required_arg) + if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED) + set(cmake_fd_required_arg REQUIRED) + endif() + find_package(${dep} ${cmake_fd_version} + ${cmake_fd_exact_arg} + ${cmake_fd_quiet_arg} + ${cmake_fd_required_arg} + ) + if (NOT ${dep}_FOUND) + set(${CMAKE_FIND_PACKAGE_NAME}_NOT_FOUND_MESSAGE "${CMAKE_FIND_PACKAGE_NAME} could not be found because dependency ${dep} could not be found.") + set(${CMAKE_FIND_PACKAGE_NAME}_FOUND False) + return() + endif() + set(cmake_fd_version) + set(cmake_fd_required_arg) + set(cmake_fd_quiet_arg) + set(cmake_fd_exact_arg) + endif() + endmacro() +endif() + +find_package(CUDA QUIET) + +if( CUDA_INCLUDE_DIRS AND CUDA_VERSION AND CUDA_NVCC_EXECUTABLE) + message(STATUS "CUDA_VERSION = ${CUDA_VERSION}") + message(STATUS "CUDA_INCLUDE_DIRS = ${CUDA_INCLUDE_DIRS}") + message(STATUS "CUDA_NVCC_EXECUTABLE = ${CUDA_NVCC_EXECUTABLE}") + + set( HIP_PLATFORM "nvcc" ) + + #export the environment variable, so that HIPCC can find it. + set(ENV{HIP_PLATFORM} nvcc) + +else() + if (NOT DEFINED HIP_PATH) + find_path( HIP_INCLUDE_DIR + NAMES + hip/hip_runtime.h + PATHS + ENV HIP_PATH + /opt/rocm + PATH_SUFFIXES + /include/hip + /include + ) + + set (HIP_INCLUDE_DIRS ${HIP_INCLUDE_DIR}) + set (HIP_PATH ${HIP_INCLUDE_DIR}) + if (NOT DEFINED ENV{HIP_PATH}) + set( ENV{HIP_PATH} ${HIP_PATH}) + endif( ) + endif() + + message(STATUS "ENV HIP_PATH = $ENV{HIP_PATH}") + + find_program(HIPCC + NAMES hipcc + PATHS + ENV HIP_PATH + /opt/rocm + PATH_SUFFIXES + /bin + ) + + message(STATUS "HIPCC = ${HIPCC}") + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args( + HIP + FOUND_VAR HIP_FOUND + REQUIRED_VARS HIP_PATH HIP_INCLUDE_DIRS HIPCC) + +endif() diff --git a/tensilelite/Tensile/Source/FindOpenCL.cmake b/tensilelite/Tensile/Source/FindOpenCL.cmake new file mode 100644 index 0000000000..7ffb2be0fb --- /dev/null +++ b/tensilelite/Tensile/Source/FindOpenCL.cmake @@ -0,0 +1,184 @@ +################################################################################ +# Copyright 2016-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +# Author: Kent Knox +# Locate an OpenCL implementation. +# Currently supports AMD APP SDK (http://developer.amd.com/sdks/AMDAPPSDK/Pages/default.aspx/) +# +# Defines the following variables: +# +# OPENCL_FOUND - Found the OPENCL framework +# OPENCL_INCLUDE_DIRS - Include directories +# +# Also defines the library variables below as normal +# variables. These contain debug/optimized keywords when +# a debugging library is found. +# +# OPENCL_LIBRARIES - libopencl +# +# Accepts the following variables as input: +# +# OPENCL_ROOT - (as a CMake or environment variable) +# The root directory of the OpenCL implementation found +# +# FIND_LIBRARY_USE_LIB64_PATHS - Global property that controls whether findOpenCL should search for +# 64bit or 32bit libs +#----------------------- +# Example Usage: +# +# find_package(OPENCL REQUIRED) +# include_directories(${OPENCL_INCLUDE_DIRS}) +# +# add_executable(foo foo.cc) +# target_link_libraries(foo ${OPENCL_LIBRARIES}) +# +#----------------------- +include( CheckSymbolExists ) +include( CMakePushCheckState ) + +if( DEFINED OPENCL_ROOT OR DEFINED ENV{OPENCL_ROOT}) + message( STATUS "Defined OPENCL_ROOT: ${OPENCL_ROOT}, ENV{OPENCL_ROOT}: $ENV{OPENCL_ROOT}" ) +endif( ) + +find_path(OPENCL_INCLUDE_DIRS + NAMES OpenCL/cl.h CL/cl.h + HINTS + ${OPENCL_ROOT}/include + $ENV{OPENCL_ROOT}/include + $ENV{AMDAPPSDKROOT}/include + $ENV{CUDA_PATH}/include + PATHS + /usr/include + /usr/local/include + /usr/local/cuda/include + /opt/rocm/opencl/include + DOC "OpenCL header file path" +) +mark_as_advanced( OPENCL_INCLUDE_DIRS ) +message( STATUS "OPENCL_INCLUDE_DIRS: ${OPENCL_INCLUDE_DIRS}" ) + +set( OpenCL_VERSION "0.0" ) + +cmake_push_check_state( RESET ) +set( CMAKE_REQUIRED_INCLUDES "${OPENCL_INCLUDE_DIRS}" ) + +# Bug in check_symbol_exists prevents us from specifying a list of files, so we loop +# Only 1 of these files will exist on a system, so the other file will not clobber the output variable +if( APPLE ) + set( CL_HEADER_FILE "OpenCL/cl.h" ) +else( ) + set( CL_HEADER_FILE "CL/cl.h" ) +endif( ) + +check_symbol_exists( CL_VERSION_2_0 ${CL_HEADER_FILE} HAVE_CL_2_0 ) +check_symbol_exists( CL_VERSION_1_2 ${CL_HEADER_FILE} HAVE_CL_1_2 ) +check_symbol_exists( CL_VERSION_1_1 ${CL_HEADER_FILE} HAVE_CL_1_1 ) +# message( STATUS "HAVE_CL_2_0: ${HAVE_CL_2_0}" ) +# message( STATUS "HAVE_CL_1_2: ${HAVE_CL_1_2}" ) +# message( STATUS "HAVE_CL_1_1: ${HAVE_CL_1_1}" ) + +# set OpenCL_VERSION to the highest detected version +if( HAVE_CL_2_0 ) + set( OpenCL_VERSION "2.0" ) +elseif( HAVE_CL_1_2 ) + set( OpenCL_VERSION "1.2" ) +elseif( HAVE_CL_1_1 ) + set( OpenCL_VERSION "1.1" ) +endif( ) + +cmake_pop_check_state( ) + +# Search for 64bit/32bit libs +if( CMAKE_SIZEOF_VOID_P EQUAL 8 ) + set(LIB64 ON) + message( STATUS "FindOpenCL searching for 64-bit libraries" ) +else( ) + set(LIB64 OFF) + message( STATUS "FindOpenCL searching for 32-bit libraries" ) +endif( ) + +if( LIB64 ) + find_library( OPENCL_LIBRARIES + NAMES OpenCL + HINTS + ${OPENCL_ROOT}/lib + $ENV{OPENCL_ROOT}/lib + $ENV{AMDAPPSDKROOT}/lib + $ENV{CUDA_PATH}/lib + DOC "OpenCL dynamic library path" + PATH_SUFFIXES x86_64 x64 + PATHS + /usr/lib + /usr/local/cuda/lib + /opt/rocm/opencl/lib + ) +else( ) + find_library( OPENCL_LIBRARIES + NAMES OpenCL + HINTS + ${OPENCL_ROOT}/lib + $ENV{OPENCL_ROOT}/lib + $ENV{AMDAPPSDKROOT}/lib + $ENV{CUDA_PATH}/lib + DOC "OpenCL dynamic library path" + PATH_SUFFIXES x86 Win32 + PATHS + /usr/lib + /usr/local/cuda/lib + /opt/rocm/opencl/lib + ) +endif( ) +mark_as_advanced( OPENCL_LIBRARIES ) + +# If we asked for OpenCL 1.2, and we found a version installed greater than that, pass the 'use deprecated' flag +if( (OpenCL_FIND_VERSION VERSION_LESS "2.0") AND (OpenCL_VERSION VERSION_GREATER OpenCL_FIND_VERSION) ) + add_definitions( -DCL_USE_DEPRECATED_OPENCL_2_0_APIS ) + add_definitions( -DCL_USE_DEPRECATED_OPENCL_1_2_APIS ) + + # If we asked for OpenCL 1.1, and we found a version installed greater than that, pass the 'use deprecated' flag + if( (OpenCL_FIND_VERSION VERSION_LESS "1.2") AND (OpenCL_VERSION VERSION_GREATER OpenCL_FIND_VERSION) ) + add_definitions( -DCL_USE_DEPRECATED_OPENCL_1_1_APIS ) + endif( ) +endif( ) + +include( FindPackageHandleStandardArgs ) +FIND_PACKAGE_HANDLE_STANDARD_ARGS( OPENCL + REQUIRED_VARS OPENCL_LIBRARIES OPENCL_INCLUDE_DIRS + VERSION_VAR OpenCL_VERSION + ) + +if( NOT OPENCL_FOUND ) + message( STATUS "FindOpenCL looked for libraries named: OpenCL" ) +else( ) + message(STATUS "FindOpenCL ${OPENCL_LIBRARIES}, ${OPENCL_INCLUDE_DIRS}") + + # UNKNOWN implies on windows I can use the .lib file for IMPORTED_LOCATION + # we don't care about the location of opencl.dll, it's installed in system directories + if (WIN32) + add_library( opencl UNKNOWN IMPORTED ) + endif() + + set_target_properties( opencl PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${OPENCL_INCLUDE_DIRS}" + IMPORTED_LOCATION "${OPENCL_LIBRARIES}" + ) + +endif() diff --git a/tensilelite/Tensile/Source/KernelHeader.h b/tensilelite/Tensile/Source/KernelHeader.h new file mode 100644 index 0000000000..4de8f2556e --- /dev/null +++ b/tensilelite/Tensile/Source/KernelHeader.h @@ -0,0 +1,79 @@ +/******************************************************************************* + * Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + *******************************************************************************/ + +#ifndef KERNEL_HEADER +#define KERNEL_HEADER + +#if Tensile_RUNTIME_LANGUAGE_OCL +#include +#else +#include "TensileTypes.h" +#include + +__device__ inline int GenDot4(int a, int b, int c) +{ +#if(__hcc_workweek__ >= 19092) || __HIP_CLANG_ONLY__ + union + { + int32_t i; + char4 z; + } va, vb; +#else + typedef struct + { + int c0 : 8, c1 : 8, c2 : 8, c3 : 8; + } C4I8; + union + { + int32_t i; + C4I8 z; + } va, vb; +#endif + va.i = a; + vb.i = b; + +#if(__hcc_workweek__ >= 19092) || __HIP_CLANG_ONLY__ + return amd_mixed_dot(va.z, vb.z, c, true); +} +#else + return c + (vb.z.c3 * va.z.c3 + vb.z.c2 * va.z.c2 + vb.z.c1 * va.z.c1 + vb.z.c0 * va.z.c0); +} +#endif + +#endif // HIP + +typedef _Float16 tensile_half2 __attribute__((ext_vector_type(2))); +typedef _Float16 tensile_half; + +extern "C" __device__ tensile_half2 llvm_fma_v2f16(tensile_half2, + tensile_half2, + tensile_half2) __asm("llvm.fma.v2f16"); + +__device__ inline tensile_half2 + tensile_fmadd_half2(tensile_half2 multiplier, tensile_half2 multiplicand, tensile_half2 addend) +{ + tensile_half2 result; + result = llvm_fma_v2f16(multiplier, multiplicand, addend); + return result; +}; + +#endif diff --git a/tensilelite/Tensile/Source/TensileCreateLibrary.cmake b/tensilelite/Tensile/Source/TensileCreateLibrary.cmake new file mode 100644 index 0000000000..0e07501374 --- /dev/null +++ b/tensilelite/Tensile/Source/TensileCreateLibrary.cmake @@ -0,0 +1,129 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +include(CMakeParseArguments) + +################################################################################ +# Create A Tensile Library from LibraryLogic.yaml files +################################################################################ +function(TensileCreateLibraryCmake + Tensile_LOGIC_PATH + Tensile_RUNTIME_LANGUAGE + Tensile_COMPILER + Tensile_CODE_OBJECT_VERSION + Tensile_ARCHITECTURE + Tensile_LIBRARY_FORMAT + Tensile_MERGE_FILES + Tensile_SHORT_FILE_NAMES + Tensile_LIBRARY_PRINT_DEBUG + Tensile_CPU_THREADS + Tensile_SEPARATE_ARCHITECTURES) + +# make Tensile_PACKAGE_LIBRARY and optional parameter +# to avoid breaking applications which us this + if (ARGN) + list (GET ARGN 0 Tensile_PACKAGE_LIBRARY) + # list (GET ARGN 1 Tensile_INCLUDE_LEGACY_CODE) + else() + set(Tensile_PACKAGE_LIBRARY OFF) + # set(Tensile_INCLUDE_LEGACY_CODE ON) + endif() + + # set(options PACKAGE_LIBRARY Tensile_INCLUDE_LEGACY_CODE) + message(STATUS "Tensile_RUNTIME_LANGUAGE from TensileCreateLibraryCmake : ${Tensile_RUNTIME_LANGUAGE}") + message(STATUS "Tensile_CODE_OBJECT_VERSION from TensileCreateLibraryCmake : ${Tensile_CODE_OBJECT_VERSION}") + message(STATUS "Tensile_COMPILER from TensileCreateLibraryCmake : ${Tensile_COMPILER}") + message(STATUS "Tensile_ARCHITECTURE from TensileCreateLibraryCmake : ${Tensile_ARCHITECTURE}") + message(STATUS "Tensile_LIBRARY_FORMAT from TensileCreateLibraryCmake : ${Tensile_LIBRARY_FORMAT}") + message(STATUS "Tensile_CPU_THREADS from TensileCreateLibraryCmake : ${Tensile_CPU_THREADS}") + + #execute_process(COMMAND chmod 755 ${Tensile_ROOT}/bin/TensileCreateLibrary) + #execute_process(COMMAND chmod 755 ${Tensile_ROOT}/bin/Tensile) + file(COPY ${Tensile_ROOT}/bin/TensileCreateLibrary DESTINATION ${Tensile_ROOT}/bin FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + file(COPY ${Tensile_ROOT}/bin/Tensile DESTINATION ${Tensile_ROOT}/bin FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + + set(Tensile_CREATE_COMMAND "${Tensile_ROOT}/bin/TensileCreateLibrary") + + set(Tensile_SOURCE_PATH "${PROJECT_BINARY_DIR}/Tensile") + message(STATUS "Tensile_SOURCE_PATH=${Tensile_SOURCE_PATH}") + + # TensileLibraryWriter optional arguments + if(${Tensile_MERGE_FILES}) + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--merge-files") + else() + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--no-merge-files") + endif() + + if(${Tensile_PACKAGE_LIBRARY}) + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--package-library") + endif() + + if(${Tensile_SEPARATE_ARCHITECTURES}) + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--separate-architectures") + endif() + + if(${Tensile_SHORT_FILE_NAMES}) + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--short-file-names") + else() + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--no-short-file-names") + endif() + + if(${Tensile_LIBRARY_PRINT_DEBUG}) + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--library-print-debug") + else() + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--no-library-print-debug") + endif() + + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--architecture=${Tensile_ARCHITECTURE}") + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--code-object-version=${Tensile_CODE_OBJECT_VERSION}") + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--cxx-compiler=${Tensile_COMPILER}") + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--library-format=${Tensile_LIBRARY_FORMAT}") + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--jobs=${Tensile_CPU_THREADS}") + + # TensileLibraryWriter positional arguments + set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} + ${Tensile_LOGIC_PATH} + ${Tensile_SOURCE_PATH} + ${Tensile_RUNTIME_LANGUAGE} + ) + + #string( REPLACE ";" " " Tensile_CREATE_COMMAND "${Tensile_CREATE_COMMAND}") + message(STATUS "Tensile_CREATE_COMMAND: ${Tensile_CREATE_COMMAND}") + + # execute python command + if($ENV{TENSILE_SKIP_LIBRARY}) + message(STATUS "Skipping build of ${Tensile_OUTPUT_PATH}") + else() + if (WIN32) + set(CommandLine ${VIRTUALENV_BIN_DIR}/${VIRTUALENV_PYTHON_EXENAME} ${Tensile_CREATE_COMMAND}) + else() + set(CommandLine ${Tensile_CREATE_COMMAND}) + endif() + execute_process( + COMMAND ${CommandLine} + RESULT_VARIABLE Tensile_CREATE_RESULT + ) + if(Tensile_CREATE_RESULT) + message(FATAL_ERROR "Error generating kernels") + endif() + endif() + +endfunction() diff --git a/tensilelite/Tensile/Source/TensileTypes.h b/tensilelite/Tensile/Source/TensileTypes.h new file mode 100644 index 0000000000..ab579d8fa2 --- /dev/null +++ b/tensilelite/Tensile/Source/TensileTypes.h @@ -0,0 +1,734 @@ +/******************************************************************************* + * Copyright 2016-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + *******************************************************************************/ + +#ifndef TENSILETYPES_H +#define TENSILETYPES_H +#include "tensile_bfloat16.h" +#include +#include +#include +#include +#include + +#include + +// OpenCL +#if Tensile_RUNTIME_LANGUAGE_OCL +#include "CL/cl.h" +#define TensileStatus cl_int +#define tensileStatusSuccess CL_SUCCESS +#define tensileStatusFailure -1 +#define tensileStatusAssertFailure -2 +#define TensileComplexFloat cl_float2 +#define TensileComplexDouble cl_double2 +#define TensileHalf cl_half + +// HIP +#else +#include +#define TensileStatus hipError_t +#define tensileStatusSuccess hipSuccess +// FIXME - steal hip error encodings since rocBLAS expects hip error codes to be +// returned... +#define tensileStatusFailure hipErrorUnknown +#define tensileStatusAssertFailure hipErrorRuntimeOther +#define TensileHalf _Float16 +inline std::ostream& operator<<(std::ostream& os, const _Float16& dt) +{ + os << (float)(dt); + return os; +} + +template +struct tensile_complex +{ + t x; + t y; + + __host__ __device__ tensile_complex() = default; + + constexpr __host__ __device__ tensile_complex(t real, t imag = 0) + : x{real} + , y{imag} + { + } + + template {}>::type* = nullptr> + tensile_complex& operator=(const u a) + { + x = a; + y = 0; + return (*this); + } +}; + +template +inline std::ostream& operator<<(std::ostream& os, const tensile_complex& data) +{ + if(data.y >= 0) + { + os << data.x << "+" << data.y << "i"; + } + else + { + os << data.x << data.y << "i"; + } + + return os; +} + +template +constexpr __host__ __device__ tensile_complex operator+(tensile_complex const& data) +{ + return data; +} + +template +constexpr __host__ __device__ tensile_complex operator-(tensile_complex const& data) +{ + return tensile_complex{-data.x, -data.y}; +} + +template +constexpr __host__ __device__ tensile_complex operator+(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{a.x + b.x, a.y + b.y}; +} + +template +constexpr __host__ __device__ tensile_complex operator-(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{a.x - b.x, a.y - b.y}; +} + +template +constexpr __host__ __device__ tensile_complex operator*(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{a.x * b.x - a.y * b.y, a.x * b.y + a.y * b.x}; +} + +template +constexpr __host__ __device__ tensile_complex operator/(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{(a.x * b.x + a.y * b.y) / (b.x * b.x + b.y * b.y), + (a.y * b.x - a.x * b.y) / (b.x * b.x + b.y * b.y)}; +} + +template +constexpr __host__ __device__ tensile_complex& operator+=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a + b); +} + +template +constexpr __host__ __device__ tensile_complex& operator-=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a - b); +} + +template +constexpr __host__ __device__ tensile_complex& operator*=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a * b); +} + +template +constexpr __host__ __device__ tensile_complex& operator/=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a / b); +} +template +constexpr __host__ __device__ bool operator==(tensile_complex const& a, + tensile_complex const& b) +{ + return (a.x == b.x) && (a.y == b.y); +} + +using tensile_float_complex = tensile_complex; +using tensile_double_complex = tensile_complex; + +#define TensileComplexFloat tensile_float_complex +#define TensileComplexDouble tensile_double_complex +#endif // HIP + +#define TensileInt8x4 uint32_t +#define TensileInt32 int32_t + +/******************************************************************************* + * tensileSetup + ******************************************************************************/ +TensileStatus tensileSetup(); + +/******************************************************************************* + * tensileTeardown + ******************************************************************************/ +TensileStatus tensileTeardown(); + +/******************************************************************************* + * tensileCheckStatus + ******************************************************************************/ +#define tensileStatusCheck(RET) \ + { \ + TensileStatus tensileCheckStatusTmp = RET; \ + if(tensileCheckStatusTmp != tensileStatusSuccess) \ + { \ + fprintf(stderr, \ + "ERROR: TensileStatusFailure %i on line %u of %s\n", \ + tensileCheckStatusTmp, \ + __LINE__, \ + __FILE__); \ + abort(); \ + } \ + } + +// class ProblemDims +// - stores all dimensions of the problems (sizes and strides) +// TensileCreateLibrary.cpp will create a typedef for each specific problem, ie +// ProblemDims_Cijk_Ailk_Bljk_SB. +template +class ProblemDims +{ +public: + using SizeType = unsigned int; + + // Constructor accepts variable number of sizes: + template + explicit ProblemDims(Ts... args) + { + init(args...); + }; + + const SizeType strideD(int idx) const + { + return _dims[idx]; + } + + const SizeType strideC(int idx) const + { + return _dims[LastStrideD - FirstStride + idx]; + } + + const SizeType strideA(int idx) const + { + return _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + idx]; + } + + const SizeType strideB(int idx) const + { + return _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + LastStrideA + - FirstStride + idx]; + } + + const SizeType sizes(int idx = 0) const + { + return _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + LastStrideA + - FirstStride + LastStrideB - FirstStride + idx]; + } + + static int numSizes() + { + return NumSizes; + }; + + std::ostream& print(std::ostream& os) const + { + for(int i = 0; i < NumSizes; i++) + { + if(i != 0) + { + os << ", "; + }; + os << _dims[i]; + }; + return os; + }; + +private: + template + void init(T v) + { + _dims[NumSizes - I] = v; + } + + template + void init(T v, Ts... args) + { + _dims[NumSizes - I] = v; + init(args...); + } + +private: + // order: Dstride, Cstride, Astride, Bstrides, sizes + SizeType _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + LastStrideA - FirstStride + + LastStrideB - FirstStride + NumSizes]; +}; + +// Base template for ProblemKey +// - stores the sizes and strides +// - supports hash generation and comparison for lookup +// TensileCreateLibrary.cpp will create a typedef for each specific problem, ie +// ProblemKey_Cijk_Ailk_Bljk_SB. +// Some templates below use a parm called ProblemKeyType which can be any of +// these generated types. +template +class ProblemKey +{ +public: + using SizeType = unsigned int; + + // Constructor accepts variable number of sizes: + template + explicit ProblemKey(Ts... args) + : _db(0) + { + init(args...); + } + + template + explicit ProblemKey(const ProblemDims& pdims) + : _db(0) + , _equalStrides(true) + { + for(int i = 0; i < NumSizes; i++) + { + _sizes[i] = pdims.sizes(i); + } + + _equalStrides + = ((pdims.strideD(0) == pdims.strideC(0)) && (pdims.strideD(1) == pdims.strideC(1))); + } + + bool operator<(const ProblemKey& p) const + { + if(p._equalStrides != this->_equalStrides) + return true; + for(int i = 0; i < NumSizes; i++) + { + if(p._sizes[i] < this->_sizes[i]) + return true; + else if(p._sizes[i] > this->_sizes[i]) + return false; + // if equal, continue to next index + } + return false; // get here if all indices are equal + }; + + bool operator==(const ProblemKey& p) const + { + if(p._equalStrides != this->_equalStrides) + return false; + for(int i = 0; i < NumSizes; i++) + { + if(p._sizes[i] != this->_sizes[i]) + return false; + } + return true; + }; + + size_t hash() const + { + size_t h = 0; + for(int i = 0; i < NumSizes; i++) + { + h ^= _sizes[i] + 0x9b9773e99e3779b9 + (h << 6) + (h >> 2); + } + return h; + } + + const SizeType sizes(int i) const + { + return _sizes[i]; + }; + + static int numSizes() + { + return NumSizes; + }; + + std::ostream& print(std::ostream& os) const + { + for(int i = 0; i < NumSizes; i++) + { + if(i != 0) + { + os << ", "; + }; + os << _sizes[i]; + }; + return os; + }; + +private: + template + void init(const SizeType& v) + { + _sizes[NumSizes - I - 1] = v; + } + + template + void init(const SizeType& v, Ts... args) + { + _sizes[NumSizes - I - 1] = v; + init(args...); + } + +private: + // Data members: + SizeType _sizes[NumSizes]; + bool _equalStrides; + uint32_t _db; +}; + +//------------- +// Distance Functions between two problem sizes - used to find nearest neighbor +// or closest solution Assumes p1 and p2 have same number of sizes +//------------- + +template +struct RatioDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + double distance = 1.0; + for(int i = 0; i < p1.numSizes(); i++) + { + distance += ::fabs(::log(double(p1.sizes(i)) / double(p2.sizes(i)))); + } + return distance; + } +}; + +template +struct ManhattanDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + double distance = 0; + for(int i = 0; i < p1.numSizes(); i++) + { + distance += ::fabs(double(p1.sizes(i)) - double(p2.sizes(i))); + } + return distance; + } +}; + +template +struct EuclideanDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + double distance = 0; + for(int i = 0; i < p1.numSizes(); i++) + { + if(p1.sizes(i) > p2.sizes(i)) + distance += pow(p1.sizes(i) - p2.sizes(i), 2); + else + distance += pow(p2.sizes(i) - p1.sizes(i), 2); + } + // distance = sqrt(distance); + return distance; + } +}; + +template +struct RandomDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + return double(rand()); + } +}; + +// ProblemType +// Stores the different dimensions (free, index, batch) and other +// problem-specific info A single ProblemType may have many ProblemDims +// Combination of a ProblemType and ProblemDim defines a Problem +class ProblemType +{ +public: + ProblemType(const std::vector& indicesFree, + const std::vector& indicesSummation, + const std::vector& indicesBatch, + const std::vector& indexAssignmentsA, + const std::vector& indexAssignmentsB) + : _indicesFree(indicesFree) + , _indicesSummation(indicesSummation) + , _indicesBatch(indicesBatch) + , _indexAssignmentsA(indexAssignmentsA) + , _indexAssignmentsB(indexAssignmentsB) + { + } + + unsigned lastSummationIdx() const + { + return _indicesSummation.back(); + }; + unsigned free0Idx() const + { + return _indicesFree[0]; + }; + unsigned free1Idx() const + { + return _indicesFree[1]; + }; + bool isBatchIdx(unsigned idx) const + { + return std::find(_indicesBatch.begin(), _indicesBatch.end(), idx) != _indicesBatch.end(); + }; + unsigned numIndicesC() const + { + return _indicesFree.size() + _indicesBatch.size(); + }; + + const std::vector indexAssignmentsA() const + { + return _indexAssignmentsA; + } + const std::vector indexAssignmentsB() const + { + return _indexAssignmentsB; + } + +private: + const std::vector _indicesFree; + const std::vector _indicesSummation; + const std::vector _indicesBatch; + const std::vector _indexAssignmentsA; + const std::vector _indexAssignmentsB; +}; + +// These are properties of the problemDims and problemType which drive the +// 'assertions' +// - Solutions may be optimized so they only work for a subset of +// ProblemProperties - +// - At runtime, the ProblemProperties are computed for the given ProblemType +// and +// ProblemDims, and then checked against the Solution requirements. +// Must be checked by the runtime before launching the solution +struct ProblemProperties +{ + + // Constructor used in solution tables- + // See writeSolutionAndExactTable in TensileCreateLibrary - this constructor + // must be in-sync with the table written there. + ProblemProperties(unsigned summationElementMultiple, + unsigned free0ElementMultiple, + unsigned free1ElementMultiple, + int approxSize, + bool equalStrides, + bool allBatchAStridesAreZero, + bool allBatchBStridesAreZero) + : _summationElementMultiple(summationElementMultiple) + , _free0ElementMultiple(free0ElementMultiple) + , _free1ElementMultiple(free1ElementMultiple) + , _approxSize(approxSize) + , _equalStrides(equalStrides) + , _allBatchAStridesAreZero(allBatchAStridesAreZero) + , _allBatchBStridesAreZero(allBatchBStridesAreZero) + , _db(0) + { + const char* db = std::getenv("TENSILE_DB"); + if(db) + { + _db = strtol(db, nullptr, 0); + } + } + + // Constructor used to compute assertions for a specified problem size + template + ProblemProperties(const ProblemDimsType& pdims, const ProblemType* props) + : _db(0) + { + _summationElementMultiple = 1; // problem summation element multiple + auto sumSize = pdims.sizes(props->lastSummationIdx()); + if((sumSize & 0x7) == 0) + _summationElementMultiple = 8; + else if((sumSize & 0x3) == 0) + _summationElementMultiple = 4; + else if((sumSize & 0x1) == 0) + _summationElementMultiple = 2; + + auto free0Size = pdims.sizes(props->free0Idx()); + _free0ElementMultiple = 1; // problem free0 element multiple + if((free0Size & 0x7) == 0) + _free0ElementMultiple = 8; + else if((free0Size & 0x3) == 0) + _free0ElementMultiple = 4; + else if((free0Size & 0x1) == 0) + _free0ElementMultiple = 2; + + auto free1Size = pdims.sizes(props->free1Idx()); + _free1ElementMultiple = 1; // problem free1 element multiple + if((free1Size & 0x7) == 0) + _free1ElementMultiple = 8; + else if((free1Size & 0x3) == 0) + _free1ElementMultiple = 4; + else if((free1Size & 0x1) == 0) + _free1ElementMultiple = 2; + + bool allBelow1 = true; + bool anyBelow1 = false; + bool anyBelow4 = false; + for(int si = 0; si != pdims.numSizes(); si++) + { + if(!props->isBatchIdx(si)) + { + auto size = pdims.sizes(si); + if(size > 1) + allBelow1 = false; + if(size == 1) + anyBelow1 = true; + else if(size < 4) + anyBelow4 = true; + } + } + if(allBelow1) + _approxSize = 1; // really small + else if(anyBelow1 || anyBelow4) + _approxSize = 2; // one dim not big enough + else + _approxSize = 99; // big enough + + _equalStrides = true; + // Would need to fix for UseInitialStrides, if UseInitialStrides==1 then + // don't subtract 1 + for(int i = 0; i < props->numIndicesC() - 1; i++) + { + if(pdims.strideD(i) != pdims.strideC(i)) + { + _equalStrides = false; + break; + } + } + + _allBatchAStridesAreZero = 1; + int strideIdx = 0; + for(auto& idx : props->indexAssignmentsA()) + { + bool isb = props->isBatchIdx(idx); + // Would need to fix for UseInitialStrides + auto stride = strideIdx == 0 ? 1 : pdims.strideA(strideIdx - 1); + if(isb && stride != 0) + { + _allBatchAStridesAreZero = 0; + } + strideIdx++; + } + + _allBatchBStridesAreZero = 1; + strideIdx = 0; + for(auto& idx : props->indexAssignmentsB()) + { + bool isb = props->isBatchIdx(idx); + // Would need to fix for UseInitialStrides + auto stride = strideIdx == 0 ? 1 : pdims.strideB(strideIdx - 1); + if(isb && stride != 0) + { + _allBatchBStridesAreZero = 0; + } + strideIdx++; + } + }; + + // Returns True if this AsssertionProperties meet the requirements for the + // specified soluition (this object represents the 'Problem') + bool validForSolution(const ProblemProperties& solutionRequirements) const + { + bool rv + = (this->_summationElementMultiple >= solutionRequirements._summationElementMultiple) + && (this->_free0ElementMultiple >= solutionRequirements._free0ElementMultiple) + && (this->_free1ElementMultiple >= solutionRequirements._free1ElementMultiple) + && ((this->_approxSize) >= solutionRequirements._approxSize) + && ((this->_equalStrides) == solutionRequirements._equalStrides) + && ((this->_allBatchAStridesAreZero) >= solutionRequirements._allBatchAStridesAreZero) + && ((this->_allBatchBStridesAreZero) + >= solutionRequirements._allBatchBStridesAreZero); + +#if 0 + if(this->_db & 0x10) + { + if(!rv) + { +#define check_print(name, oper) \ + if(!(this->name oper solutionRequirements.name)) \ + { \ + std::cout << #name << "failed: !(" << this->name << #oper << solutionRequirements.name \ + << ")" << std::endl; \ + } + + std::cout << "Assertion failed:" << std::endl; + check_print(_summationElementMultiple, >=); + check_print(_free0ElementMultiple, >=); + check_print(_free1ElementMultiple, >=); + check_print(_approxSize, >=); + check_print(_equalStrides, ==); + check_print(_allBatchAStridesAreZero, >=); + check_print(_allBatchBStridesAreZero, >=); +#undef check_print + } + else + { + std::cout << "Satisfied asserts!" << std::endl; + } + } +#endif + + return rv; + } + + unsigned _summationElementMultiple; + unsigned _free0ElementMultiple; + unsigned _free1ElementMultiple; + int _approxSize; + bool _equalStrides; + bool _allBatchAStridesAreZero; // for problems: true if all batchA strides are + // 0. Solutions: true if all batchA strides + // must be 0 to run the kernel. Prob,Soln: x,0: + // ok: 0,1: FAILED_ASSERT, 1,1: OK . P>=S + bool _allBatchBStridesAreZero; // true if all batchB strides are 0 + uint32_t _db; +}; + +#endif diff --git a/tensilelite/Tensile/Source/client/CMakeLists.txt b/tensilelite/Tensile/Source/client/CMakeLists.txt new file mode 100644 index 0000000000..a3c555bbaa --- /dev/null +++ b/tensilelite/Tensile/Source/client/CMakeLists.txt @@ -0,0 +1,58 @@ +set(client_sources + source/BenchmarkTimer.cpp + source/CSVStackFile.cpp + source/ClientProblemFactory.cpp + source/ConvolutionProblem.cpp + source/DataInitialization.cpp + source/HardwareMonitor.cpp + source/HardwareMonitorListener.cpp + source/LibraryUpdateReporter.cpp + source/MetaRunListener.cpp + source/PerformanceReporter.cpp + source/ProgressListener.cpp + source/Reference.cpp + source/ReferenceValidator.cpp + source/ResultFileReporter.cpp + source/ResultReporter.cpp + source/SolutionIterator.cpp + source/TimingEvents.cpp + ) + +find_package(Boost COMPONENTS program_options filesystem REQUIRED) + +add_library(TensileClient STATIC ${client_sources}) + +set_target_properties(TensileClient + PROPERTIES + CXX_STANDARD 14 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF) + +find_package(ROCmSMI QUIET) +if(NOT ROCmSMI_FOUND) + set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" "${Tensile_DIR}" "${Tensile_DIR}/../Source/cmake" "${CMAKE_HOME_DIRECTORY}/cmake") + find_package(ROCmSMI REQUIRED) +endif() + +target_include_directories(TensileClient PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")# "${rocm_smi_root}/include") + +target_link_libraries(TensileClient PRIVATE TensileHost ${Boost_LIBRARIES} rocm_smi) +if(TENSILE_USE_OPENMP) + target_link_libraries(TensileClient PRIVATE custom_openmp_cxx) +endif() + +add_executable(tensile_client main.cpp) +set_target_properties(tensile_client + PROPERTIES + CXX_STANDARD 14 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF) + +target_link_libraries(tensile_client PRIVATE TensileHost TensileClient ${Boost_LIBRARIES}) +if(TENSILE_USE_OPENMP) + target_link_libraries(tensile_client PRIVATE custom_openmp_cxx) +endif() + +foreach(arch IN LISTS TENSILE_GPU_ARCHS) + target_link_libraries(tensile_client PRIVATE "--amdgpu-target=${arch}") +endforeach(arch) diff --git a/tensilelite/Tensile/Source/client/include/BenchmarkTimer.hpp b/tensilelite/Tensile/Source/client/include/BenchmarkTimer.hpp new file mode 100644 index 0000000000..6d9b8c9b1f --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/BenchmarkTimer.hpp @@ -0,0 +1,122 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "RunListener.hpp" + +#include +#include + +#include + +#include +#include + +namespace Tensile +{ + namespace Client + { + namespace po = boost::program_options; + + class BenchmarkTimer : public RunListener + { + public: + using clock = std::chrono::steady_clock; + + BenchmarkTimer(po::variables_map const& args, Hardware const& hardware); + + virtual bool needMoreBenchmarkRuns() const override; + virtual void preBenchmarkRun() override; + virtual void postBenchmarkRun() override; + + virtual void preProblem(ContractionProblem const& problem) override; + virtual void postProblem() override; + + virtual void preSolution(ContractionSolution const& solution) override; + virtual void postSolution() override; + + virtual bool needMoreRunsInSolution() const override; + + virtual size_t numWarmupRuns() override; + virtual void setNumWarmupRuns(size_t count) override; + virtual void preWarmup() override; + virtual void postWarmup() override; + virtual void validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + + virtual size_t numSyncs() override; + virtual void setNumSyncs(size_t count) override; + virtual void preSyncs() override; + virtual void postSyncs() override; + + virtual size_t numEnqueuesPerSync() override; + virtual void setNumEnqueuesPerSync(size_t count) override; + virtual void preEnqueues() override; + virtual void postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + virtual void validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + + virtual void finalizeReport() override; + virtual int error() const override; + + private: + const int m_numWarmups; + const int m_numBenchmarks; + const int m_numEnqueuesPerSync; + const int m_numSyncsPerBenchmark; + const int m_numEnqueuesPerSolution; + const size_t m_minFlopsPerSync = 0; + + const bool m_useGPUTimer; + const bool m_syncAfterWarmups = true; + const int m_sleepPercent; + + int m_numBenchmarksRun = 0; + + Hardware const& m_hardware; + ContractionProblem m_problem; + ContractionSolution m_solution; + + int m_numEnqueuesInSolution = 0; + int m_numSyncsInBenchmark = 0; + int m_curNumEnqueuesPerSync = 0; + + clock::time_point m_startTime; + clock::time_point m_endTime; + + using double_millis = std::chrono::duration; + using double_micros = std::chrono::duration; + using double_nanos = std::chrono::duration; + + double_millis m_timeInSolution; + double_millis m_totalGPUTime; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/CSVStackFile.hpp b/tensilelite/Tensile/Source/client/include/CSVStackFile.hpp new file mode 100644 index 0000000000..780df23ac1 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/CSVStackFile.hpp @@ -0,0 +1,87 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include + +#include + +namespace Tensile +{ + namespace Client + { + class CSVStackFile + { + public: + CSVStackFile(std::string const& filename, std::string const& separator = ", "); + CSVStackFile(std::ostream& stream, std::string const& separator = ", "); + CSVStackFile(std::shared_ptr stream, std::string const& separator = ", "); + + ~CSVStackFile(); + + void setHeaderForKey(std::string const& key, std::string const& header); + + void setValueForKey(std::string const& key, std::string const& value); + void setValueForKey(std::string const& key, double const& value); + + template + typename std::enable_if::value, void>::type + setValueForKey(std::string const& key, T const& value) + { + setValueForKey(key, boost::lexical_cast(value)); + } + + void push(); + void pop(); + + void writeCurrentRow(); + void readCurrentRow(std::unordered_map& outMap); + void clearCurrentRow(); + + private: + std::string escape(std::string const& value); + std::string escapeQuote(std::string const& value); + + void writeRow(std::unordered_map const& row); + + std::shared_ptr m_stream; + + std::string m_separator; + + bool m_firstRow = true; + std::vector m_keyOrder; + std::unordered_map m_headers; + + std::unordered_map m_currentRow; + + std::vector> m_stack; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/ClientProblemFactory.hpp b/tensilelite/Tensile/Source/client/include/ClientProblemFactory.hpp new file mode 100644 index 0000000000..9939d91892 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/ClientProblemFactory.hpp @@ -0,0 +1,117 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include + +namespace Tensile +{ + namespace Client + { + + namespace po = boost::program_options; + + class ClientProblemFactory + { + public: + ClientProblemFactory(po::variables_map const& args); + ~ClientProblemFactory(); + + ClientProblemFactory(ContractionProblem const& problem) + : m_problems({problem}) + { + } + + template + ClientProblemFactory(Iterator begin, Iterator end) + : m_problems(begin, end) + { + } + + std::vector const& problems() const; + + std::vector createProblems(); + + private: + std::vector m_problems; + + ContractionProblem::FreeIndices m_freeIndices; + ContractionProblem::BatchIndices m_batchIndices; + ContractionProblem::BoundIndices m_boundIndices; + + DataType m_aType; + DataType m_bType; + DataType m_cType; + DataType m_dType; + DataType m_alphaType; + DataType m_betaType; + bool m_stridedBatched; + bool m_highPrecisionAccumulate; + bool m_deterministicMode; + bool m_cEqualsD; + bool m_useBias; + ArithmeticUnit m_arithmeticUnit; + KernelLanguage m_kernelLanguage; + PerformanceMetric m_performanceMetric; + bool m_fp16AltImpl; + ActivationType m_activationType; + bool m_activationHPA; + std::vector m_activationEnumArg; + + std::vector> m_problemSizes; + std::vector> m_aStrides; + std::vector> m_bStrides; + std::vector> m_cStrides; + std::vector> m_dStrides; + std::vector> m_aZeroPads; + std::vector> m_bZeroPads; + std::vector> m_convProblemSizes; + + TensorOps m_aOps; + TensorOps m_bOps; + TensorOps m_cOps; + TensorOps m_dOps; + + size_t m_aOffset; + size_t m_bOffset; + size_t m_cOffset; + size_t m_dOffset; + + double m_beta; + double m_alpha; + }; + + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/ConvolutionProblem.hpp b/tensilelite/Tensile/Source/client/include/ConvolutionProblem.hpp new file mode 100644 index 0000000000..40b9659727 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/ConvolutionProblem.hpp @@ -0,0 +1,255 @@ +/** + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include +#include + +namespace Tensile +{ + + class TENSILE_API ConvolutionProblem + { + public: + static const size_t MaxNumSpatialDims = 3; + static const size_t InvalidPos; + enum class TensorFormat + { + NCHW, + NHWC, + CNHW, // Activation Formats + KCYX, + CKYX // Weight Formats + }; + + class TENSILE_API BaseFormat + { + public: + std::string formatIdentifier() const + { + return m_formatIdentifier; + }; + TensorFormat format() const + { + return m_format; + }; + + protected: + std::string m_formatIdentifier; + TensorFormat m_format; + }; + + class TENSILE_API ActivationFormat : public BaseFormat + { + public: + ActivationFormat(); + void FromIdentifier(std::string identifier, + size_t formatNumSpatialDims, + size_t numSpatialDims, + std::vector* filters); + std::string description() const; + + size_t batchPosition() const + { + return m_batchPosition; + }; + size_t channelPosition() const + { + return m_channelPosition; + }; + + const std::vector filterPositions() const + { + return m_filterPositions; + }; + + // 0,1,2 order is X,Y,Z + // size is number of spatial dims, no extra InvalidPos values + const std::vector spatialPositions() const + { + return m_spatialPositions; + }; + + private: + size_t m_batchPosition; + size_t m_channelPosition; + + //! 0,1,2 order is X,Y,Z + //! size is number of spatial dims, no extra InvalidPos values + std::vector m_filterPositions; + std::vector m_spatialPositions; + }; + + class TENSILE_API WeightFormat : public BaseFormat + { + public: + WeightFormat(); + void FromIdentifier(std::string identifier, + bool transposeCK, + size_t numSpatialDims, + std::vector* filters); + + std::string description() const; + size_t coutPosition() const + { + return m_coutPosition; + }; + size_t cinPosition() const + { + return m_cinPosition; + }; + const std::vector filterPositions() const + { + return m_filterPositions; + }; + + private: + size_t m_cinPosition; + size_t m_coutPosition; + //! 0,1,2 order is X,Y,Z + //! always MaxNumSpatialDims elements. Positions may be InvalidPos + std::vector m_filterPositions; + }; + + //! Use for formatB and output which can take either format + struct ComboFormat + { + const ActivationFormat& activation() const + { + return m_activation; + }; + const WeightFormat& weights() const + { + return m_weights; + }; + ActivationFormat& activationW() + { + return m_activation; + }; + WeightFormat& weightsW() + { + return m_weights; + }; + + private: + ActivationFormat m_activation; + WeightFormat m_weights; + }; + + struct TENSILE_API LoopCounts + { + LoopCounts() + : scount(MaxNumSpatialDims, 1){}; + void setupForData(ConvolutionProblem const& convProblem, + ContractionProblem const& problem); + void setupFormat(ConvolutionProblem const& convProblem); + const ActivationFormat& formatA() const + { + return m_formatA; + }; + const ComboFormat& formatB() const + { + return m_formatB; + }; + const ComboFormat& formatD() const + { + return m_formatD; + }; + std::string description() const; + size_t batchCount; + size_t cinCount; + size_t coutCount; + std::vector scount; + std::vector spatialCount; //whd (original convolution input tensor) + std::vector filterCount; //xyz + std::vector strideCount; //vu# + std::vector dilationCount; //jl^ + std::vector padStartCount; //qp$ + std::vector padEndCount; //q_p_$_ + + ActivationFormat m_formatA; + ComboFormat m_formatB; + ComboFormat m_formatD; // output tensor + }; + + ConvolutionProblem() {} + + void FromIdentifier(std::string identifier); + void setupFormat(std::vector& filters); + TensorDescriptor setupDataActivation(LoopCounts const& counts, + ContractionProblem const& problem) const; + TensorDescriptor setupDataOutput(LoopCounts const& counts, + ContractionProblem const& problem) const; + TensorDescriptor setupForwardWeights(LoopCounts const& counts, + ContractionProblem const& problem) const; + void validate(const ContractionProblem& problem, + const ConvolutionProblem::LoopCounts& counts) const; + + //! Number of spatial dims after packing. + size_t numSpatialDims() const + { + return m_numSpatialDims; + } + + std::string description() const; + std::string const& operationIdentifier() const + { + return m_operationIdentifier; + } + std::string const& AIdentifier() const + { + return m_AIdentifier; + } + std::string const& BIdentifier() const + { + return m_BIdentifier; + } + std::string const& DIdentifier() const + { + return m_DIdentifier; + } + size_t numFormatSpatialDims() const + { + return m_numFormatSpatialDims; + } + + private: + //! ConvolutionForward, ConvolutionBackwardData, ConvolutionBackwardWeights + std::string m_operationIdentifier; + std::string m_AIdentifier; + std::string m_BIdentifier; + std::string m_DIdentifier; + + size_t m_groups = 1; + + size_t m_numSpatialDims = 0; + size_t m_numFormatSpatialDims = 0; + }; + + TENSILE_API std::ostream& operator<<(std::ostream& stream, + ConvolutionProblem const& convolution); +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/DataInitialization.hpp b/tensilelite/Tensile/Source/client/include/DataInitialization.hpp new file mode 100644 index 0000000000..17a96ca92b --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/DataInitialization.hpp @@ -0,0 +1,1751 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include + +#include "ClientProblemFactory.hpp" + +#include +#include + +#include "RunListener.hpp" + +namespace po = boost::program_options; + +namespace Tensile +{ + namespace Client + { + // Problem-indept. from 0~7, and 16 (fixed values for every problem) + // And problem-dept. from 8~15 (values depend on problem) + // RandomNegPosLimited: integer -128~128. fp -1.0~1.0 + enum class InitMode + { + Zero = 0, // 0 + One, // 1 + Two, // 2 + Random, // 3 + NaN, // 4 + Inf, // 5 + BadInput, // 6 + BadOutput, // 7 + SerialIdx, // 8 + SerialDim0, // 9 + SerialDim1, // 10 + Identity, // 11 + TrigSin, // 12 + TrigCos, // 13 + TrigAbsSin, // 14 + TrigAbsCos, // 15 + RandomNarrow, // 16 + NegOne, // 17 + Max, // 18 + DenormMin, // 19 + DenormMax, // 20 + RandomNegPosLimited, // 21 + Count + }; + + static bool IsProblemDependent(InitMode const& mode) + { + return mode == InitMode::SerialIdx || mode == InitMode::SerialDim0 + || mode == InitMode::SerialDim1 || mode == InitMode::Identity + || mode == InitMode::TrigSin || mode == InitMode::TrigCos + || mode == InitMode::TrigAbsSin || mode == InitMode::TrigAbsCos; + } + + std::string ToString(InitMode mode); + + std::ostream& operator<<(std::ostream& stream, InitMode const& mode); + std::istream& operator>>(std::istream& stream, InitMode& mode); + + const int pageSize = 2 * 1024 * 1024; + + enum class BoundsCheckMode + { + Disable = 0, + NaN, + GuardPageFront, + GuardPageBack, + GuardPageAll, + MaxMode + }; + + std::ostream& operator<<(std::ostream& stream, BoundsCheckMode const& mode); + std::istream& operator>>(std::istream& stream, BoundsCheckMode& mode); + + template + class TypedDataInitialization; + + class DataInitialization : public RunListener + { + public: + static double GetRepresentativeBetaValue(po::variables_map const& args); + + /** + * Factory function. + */ + static std::shared_ptr + Get(po::variables_map const& args, + ClientProblemFactory const& problemFactory, + size_t maxWorkspaceSize = 0); + + template + static std::shared_ptr> + GetTyped(po::variables_map const& args, + ClientProblemFactory const& problemFactory, + size_t maxWorkspaceSize = 0); + + DataInitialization(po::variables_map const& args, + ClientProblemFactory const& problemFactory, + size_t maxWorkspaceSize = 0); + ~DataInitialization(); + + /** + * Returns a ContractionInputs object with pointers to CPU memory, + * suitable for using to calculate reference results. + */ + virtual std::shared_ptr + prepareCPUInputs(ContractionProblem const& problem) = 0; + + /** + * Returns a ContractionInputs object with pointers to GPU memory, + * suitable for using to run the kernel. + */ + virtual std::shared_ptr + prepareGPUInputs(ContractionProblem const& problem) = 0; + + virtual std::shared_ptr cpuConvInputs() const = 0; + + template + static inline T convertDoubleTo(double value); + + template + static T getValue(InitMode mode) + { + switch(mode) + { + case InitMode::Zero: + return getValue(); + case InitMode::One: + return getValue(); + case InitMode::Two: + return getValue(); + case InitMode::Random: + return getValue(); + case InitMode::RandomNarrow: + return getValue(); + case InitMode::NaN: + return getValue(); + case InitMode::Inf: + return getValue(); + case InitMode::BadInput: + return getValue(); + case InitMode::BadOutput: + return getValue(); + case InitMode::NegOne: + return getValue(); + case InitMode::Max: + return getValue(); + case InitMode::DenormMin: + return getValue(); + case InitMode::DenormMax: + return getValue(); + case InitMode::RandomNegPosLimited: + return getValue(); + case InitMode::SerialIdx: + case InitMode::SerialDim0: + case InitMode::SerialDim1: + case InitMode::Identity: + case InitMode::TrigSin: + case InitMode::TrigCos: + case InitMode::TrigAbsSin: + case InitMode::TrigAbsCos: + case InitMode::Count: + throw std::runtime_error("Invalid InitMode."); + } + } + + template + static inline T getValue(); + + template + static inline T ConvertTo(size_t i); + + template + static inline T getTrigValue(int idx, bool useCos, bool useAbs); + + template + static bool isBadInput(T value); + + template + static bool isBadOutput(T value); + + // Fills max buffer size + template + void initArray(InitMode mode, T* array, size_t elements) + { + switch(mode) + { + case InitMode::Zero: + initArray(array, elements); + break; + case InitMode::One: + initArray(array, elements); + break; + case InitMode::Two: + initArray(array, elements); + break; + case InitMode::Random: + initArray(array, elements); + break; + case InitMode::RandomNarrow: + initArray(array, elements); + break; + case InitMode::NaN: + initArray(array, elements); + break; + case InitMode::Inf: + initArray(array, elements); + break; + case InitMode::BadInput: + initArray(array, elements); + break; + case InitMode::BadOutput: + initArray(array, elements); + break; + case InitMode::NegOne: + initArray(array, elements); + break; + case InitMode::Max: + initArray(array, elements); + break; + case InitMode::DenormMin: + initArray(array, elements); + break; + case InitMode::DenormMax: + initArray(array, elements); + case InitMode::RandomNegPosLimited: + initArray(array, elements); + break; + case InitMode::SerialIdx: + initArrayConvert(array, elements); + break; + case InitMode::SerialDim0: + case InitMode::SerialDim1: + case InitMode::Identity: + case InitMode::TrigSin: + case InitMode::TrigCos: + case InitMode::TrigAbsSin: + case InitMode::TrigAbsCos: + case InitMode::Count: + throw std::runtime_error("Invalid InitMode."); + } + } + + // For problem dependent data initialization + template + void initArray(InitMode mode, T* array, TensorDescriptor const& tensor) + { + switch(mode) + { + case InitMode::Zero: + initArray(array, tensor); + break; + case InitMode::One: + initArray(array, tensor); + break; + case InitMode::Two: + initArray(array, tensor); + break; + case InitMode::Random: + initArray(array, tensor); + break; + case InitMode::RandomNarrow: + initArray(array, tensor); + break; + case InitMode::NaN: + initArray(array, tensor); + break; + case InitMode::Inf: + initArray(array, tensor); + break; + case InitMode::BadInput: + initArray(array, tensor); + break; + case InitMode::BadOutput: + initArray(array, tensor); + break; + case InitMode::NegOne: + initArray(array, tensor); + break; + case InitMode::Max: + initArray(array, tensor); + break; + case InitMode::DenormMin: + initArray(array, tensor); + break; + case InitMode::DenormMax: + initArray(array, tensor); + break; + case InitMode::SerialIdx: + initArraySerialIdx(array, tensor); + break; + case InitMode::SerialDim0: + initArraySerialDim(array, 0, tensor); + break; + case InitMode::SerialDim1: + initArraySerialDim(array, 1, tensor); + break; + case InitMode::Identity: + initArrayIdentity(array, tensor); + break; + case InitMode::TrigSin: + initArrayTrig(array, tensor); + break; + case InitMode::TrigCos: + initArrayTrig(array, tensor); + break; + case InitMode::TrigAbsSin: + initArrayTrig(array, tensor); + break; + case InitMode::TrigAbsCos: + initArrayTrig(array, tensor); + break; + case InitMode::RandomNegPosLimited: + initArray(array, tensor); + break; + case InitMode::Count: + throw std::runtime_error("Invalid InitMode."); + } + } + + template + void initArray(T* array, size_t elements) + { + for(size_t i = 0; i < elements; i++) + { + array[i] = getValue(); + } + } + + template + void initArrayConvert(T* array, size_t elements) + { + for(size_t i = 0; i < elements; i++) + { + array[i] = ConvertTo(i); + } + } + + template + void initArray(T* array, TensorDescriptor const& tensor) + { + size_t elements = tensor.totalAllocatedElements(); + initArray(array, elements); + } + + template + void initArraySerialIdx(T* array, TensorDescriptor const& tensor) + { + auto const& sizes = tensor.sizes(); + auto count = CoordCount(sizes.begin(), sizes.end()); + std::vector coord(tensor.dimensions(), 0); + for(size_t idx = 0; idx < count; idx++) + { + CoordNumbered(idx, coord.begin(), coord.end(), sizes.begin(), sizes.end()); + array[tensor.index(coord)] = static_cast(idx); + } + } + + template + void initArraySerialDim(T* array, int dim, TensorDescriptor const& tensor) + { + auto const& sizes = tensor.sizes(); + auto count = CoordCount(sizes.begin(), sizes.end()); + std::vector coord(tensor.dimensions(), 0); + for(size_t idx = 0; idx < count; idx++) + { + CoordNumbered(idx, coord.begin(), coord.end(), sizes.begin(), sizes.end()); + array[tensor.index(coord)] = static_cast(coord[dim]); + } + } + + template + void initArrayIdentity(T* array, TensorDescriptor const& tensor) + { + auto const& sizes = tensor.sizes(); + auto count = CoordCount(sizes.begin(), sizes.end()); + std::vector coord(tensor.dimensions(), 0); + for(size_t idx = 0; idx < count; idx++) + { + CoordNumbered(idx, coord.begin(), coord.end(), sizes.begin(), sizes.end()); + array[tensor.index(coord)] = static_cast(coord[0] == coord[1] ? 1 : 0); + } + } + + template + void initArrayTrig(T* array, TensorDescriptor const& tensor) + { + auto const& sizes = tensor.sizes(); + auto count = CoordCount(sizes.begin(), sizes.end()); + std::vector coord(tensor.dimensions(), 0); + for(size_t idx = 0; idx < count; idx++) + { + CoordNumbered(idx, coord.begin(), coord.end(), sizes.begin(), sizes.end()); + array[tensor.index(coord)] = getTrigValue(idx, useCos, useAbs); + } + } + + size_t workspaceSize() const + { + return m_workspaceSize; + } + + BoundsCheckMode getCurBoundsCheck() + { + return m_curBoundsCheck; + } + + virtual bool needMoreBenchmarkRuns() const override + { + return false; + }; + virtual void preBenchmarkRun() override{}; + virtual void postBenchmarkRun() override{}; + virtual void preProblem(ContractionProblem const& problem) override{}; + virtual void postProblem() override{}; + virtual void preSolution(ContractionSolution const& solution) override{}; + virtual void postSolution() override{}; + virtual bool needMoreRunsInSolution() const override + { + return m_numRunsInSolution < m_numRunsPerSolution; + }; + + virtual size_t numWarmupRuns() override + { + return 0; + }; + virtual void setNumWarmupRuns(size_t count) override{}; + virtual void preWarmup() override{}; + virtual void postWarmup() override{}; + virtual void validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + m_numRunsInSolution++; + }; + + virtual size_t numSyncs() override + { + return 0; + }; + virtual void setNumSyncs(size_t count) override{}; + virtual void preSyncs() override{}; + virtual void postSyncs() override{}; + + virtual size_t numEnqueuesPerSync() override + { + return 0; + }; + virtual void setNumEnqueuesPerSync(size_t count) override{}; + virtual void preEnqueues() override{}; + virtual void postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) override{}; + virtual void validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override{}; + + virtual void finalizeReport() override{}; + + virtual int error() const override + { + return 0; + }; + + protected: + InitMode m_aInit, m_bInit, m_cInit, m_dInit; + InitMode m_alphaInit, m_betaInit; + InitMode m_biasInit; + + size_t m_aBufferOffset; + size_t m_bBufferOffset; + size_t m_cBufferOffset; + size_t m_dBufferOffset; + + size_t m_aMaxElements; + size_t m_bMaxElements; + size_t m_cMaxElements; + size_t m_dMaxElements; + size_t m_maxBatch; + + size_t m_workspaceSize; + + bool m_stridedBatched; + + bool m_cEqualsD; + bool m_convolutionVsContraction; + + bool m_useBias; + + ActivationType m_activationType; + bool m_activationHPA; + // Reserve for multiple argument inputs + std::vector> m_activationAdditionalArgs; + + int m_elementsToValidate = 0; + + /// If true, we will allocate an extra copy of the inputs on the GPU. + /// This will improve performance as we don't have to copy from the CPU + /// with each kernel launch, but it will use extra memory. + bool m_keepPristineCopyOnGPU = true; + + /// If set "::NaN", we will initialize all out-of-bounds inputs to NaN, and + /// all out-of-bounds outputs to a known value. This allows us to + /// verify that out-of-bounds values are not used or written to. + /// If set "::GuardPageFront/::GuardPageBack", we will allocate matrix memory + /// with page aligned, and put matrix start/end address to memory start/end address. + /// Out-of-bounds access would trigger memory segmentation faults. + /// m_boundsCheck keep the setting from args. + /// m_curBoundsCheck keep the current running boundsCheck mode. + /// If set "::GuardPageAll", DataInit would need 2 runs per solution. + /// First run would apply "::GuardPageFront" and second run would apply "::GuardPageBack". + BoundsCheckMode m_boundsCheck = BoundsCheckMode::Disable; + BoundsCheckMode m_curBoundsCheck = BoundsCheckMode::Disable; + int m_numRunsPerSolution = 0; + int m_numRunsInSolution = 0; + + /// If true, the data is dependent on the problem size (e.g. serial) + /// and must be reinitialized for each problem. Pristine copy on GPU + /// cannot be used with problem dependent data. + bool m_problemDependentData = false; + }; + + template <> + inline float DataInitialization::getValue() + { + return 0.0f; + } + template <> + inline float DataInitialization::getValue() + { + return 1.0f; + } + template <> + inline float DataInitialization::getValue() + { + return 2.0f; + } + template <> + inline float DataInitialization::getValue() + { + return -1.0f; + } + template <> + inline float DataInitialization::getValue() + { + return std::numeric_limits::max(); + } + template <> + inline float DataInitialization::getValue() + { + return std::numeric_limits::denorm_min(); + } + template <> + inline float DataInitialization::getValue() + { + union + { + uint32_t bits; + float value; + } x; + + x.bits = 0x007FFFFF; + return x.value; + } + template <> + inline float DataInitialization::getValue() + { + return std::numeric_limits::quiet_NaN(); + } + template <> + inline float DataInitialization::getValue() + { + return std::numeric_limits::infinity(); + } + + template <> + inline float DataInitialization::getValue() + { + return static_cast((rand() % 201) - 100); + } + + template <> + inline float DataInitialization::getValue() + { + return getValue(); + } + template <> + inline float DataInitialization::getValue() + { + return getValue(); + } + + template <> + inline double DataInitialization::getValue() + { + return 0.0; + } + template <> + inline double DataInitialization::getValue() + { + return 1.0; + } + template <> + inline double DataInitialization::getValue() + { + return 2.0; + } + template <> + inline double DataInitialization::getValue() + { + return -1.0; + } + template <> + inline double DataInitialization::getValue() + { + return std::numeric_limits::max(); + } + template <> + inline double DataInitialization::getValue() + { + return std::numeric_limits::denorm_min(); + } + template <> + inline double DataInitialization::getValue() + { + union + { + uint64_t bits; + double value; + } x; + + x.bits = 0x000FFFFFFFFFFFFF; + return x.value; + } + template <> + inline double DataInitialization::getValue() + { + return std::numeric_limits::quiet_NaN(); + } + template <> + inline double DataInitialization::getValue() + { + return std::numeric_limits::infinity(); + } + + template <> + inline double DataInitialization::getValue() + { + return static_cast((rand() % 2001) - 1000); + } + + template <> + inline double DataInitialization::getValue() + { + return getValue(); + } + template <> + inline double DataInitialization::getValue() + { + return getValue(); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::Zero>() + { + return std::complex(0.0f, 0.0f); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::One>() + { + return std::complex(1.0f, 0.0f); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::Two>() + { + return std::complex(2.0f, 0.0f); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::NegOne>() + { + return std::complex(-1.0f, 0.0f); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::Max>() + { + return std::complex(std::numeric_limits::max(), + std::numeric_limits::max()); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::DenormMin>() + { + return std::complex(std::numeric_limits::denorm_min(), + std::numeric_limits::denorm_min()); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::DenormMax>() + { + union + { + uint32_t bits; + float value; + } x; + + x.bits = 0x007FFFFF; + return std::complex(x.value, x.value); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::NaN>() + { + return std::complex(std::numeric_limits::quiet_NaN(), + std::numeric_limits::quiet_NaN()); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::Inf>() + { + return std::complex(std::numeric_limits::infinity(), + std::numeric_limits::infinity()); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::Random>() + { + return std::complex(getValue(), + getValue()); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::BadInput>() + { + return getValue, InitMode::NaN>(); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::BadOutput>() + { + return getValue, InitMode::Inf>(); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::Zero>() + { + return std::complex(0.0, 0.0); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::One>() + { + return std::complex(1.0, 0.0); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::Two>() + { + return std::complex(2.0, 0.0); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::NegOne>() + { + return std::complex(-1.0, 0.0); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::Max>() + { + return std::complex(std::numeric_limits::max(), + std::numeric_limits::max()); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::DenormMin>() + { + return std::complex(std::numeric_limits::denorm_min(), + std::numeric_limits::denorm_min()); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::DenormMax>() + { + union + { + uint64_t bits; + double value; + } x; + + x.bits = 0x000FFFFFFFFFFFFF; + return std::complex(x.value, x.value); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::NaN>() + { + return std::complex(std::numeric_limits::quiet_NaN(), + std::numeric_limits::quiet_NaN()); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::Inf>() + { + return std::complex(std::numeric_limits::infinity(), + std::numeric_limits::infinity()); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::Random>() + { + return std::complex(getValue(), + getValue()); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::BadInput>() + { + return getValue, InitMode::NaN>(); + } + template <> + inline std::complex + DataInitialization::getValue, InitMode::BadOutput>() + { + return getValue, InitMode::Inf>(); + } + + template <> + inline int32_t DataInitialization::getValue() + { + return 0; + } + template <> + inline int32_t DataInitialization::getValue() + { + return 1; + } + template <> + inline int32_t DataInitialization::getValue() + { + return 2; + } + template <> + inline int32_t DataInitialization::getValue() + { + return -1; + } + template <> + inline int32_t DataInitialization::getValue() + { + return std::numeric_limits::max(); + } + template <> + inline int32_t DataInitialization::getValue() + { + throw std::runtime_error("DenormMin not available for int32_t."); + } + template <> + inline int32_t DataInitialization::getValue() + { + throw std::runtime_error("DenormMax not available for int32_t."); + } + template <> + inline int32_t DataInitialization::getValue() + { + throw std::runtime_error("NaN not available for int32_t."); + } + template <> + inline int32_t DataInitialization::getValue() + { + throw std::runtime_error("Inf not available for int32_t."); + } + + template <> + inline int32_t DataInitialization::getValue() + { + return rand() % 7 - 3; + } + + template <> + inline int32_t DataInitialization::getValue() + { + return std::numeric_limits::max(); + } + template <> + inline int32_t DataInitialization::getValue() + { + return std::numeric_limits::min(); + } + + template <> + inline Int8x4 DataInitialization::getValue() + { + return Int8x4{0, 0, 0, 0}; + } + template <> + inline Int8x4 DataInitialization::getValue() + { + return Int8x4{1, 1, 1, 1}; + } + template <> + inline Int8x4 DataInitialization::getValue() + { + return Int8x4{2, 2, 2, 2}; + } + template <> + inline Int8x4 DataInitialization::getValue() + { + return Int8x4{-1, -1, -1, -1}; + } + template <> + inline Int8x4 DataInitialization::getValue() + { + return Int8x4{127, 127, 127, 127}; + } + template <> + inline Int8x4 DataInitialization::getValue() + { + throw std::runtime_error("DenormMin not available for Int8x4."); + } + template <> + inline Int8x4 DataInitialization::getValue() + { + throw std::runtime_error("DenormMax not available for Int8x4."); + } + template <> + inline Int8x4 DataInitialization::getValue() + { + throw std::runtime_error("NaN not available for Int8x4."); + } + template <> + inline Int8x4 DataInitialization::getValue() + { + throw std::runtime_error("Inf not available for Int8x4."); + } + + template <> + inline Int8x4 DataInitialization::getValue() + { + return Int8x4{static_cast((rand() % 7) - 3), + static_cast((rand() % 7) - 3), + static_cast((rand() % 7) - 3), + static_cast((rand() % 7) - 3)}; + } + + template <> + inline Int8x4 DataInitialization::getValue() + { + auto val = std::numeric_limits::max(); + return Int8x4{val, val, val, val}; + } + + template <> + inline Int8x4 DataInitialization::getValue() + { + auto val = std::numeric_limits::min(); + return Int8x4{val, val, val, val}; + } + + template <> + inline Half DataInitialization::getValue() + { + return static_cast(0); + } + template <> + inline Half DataInitialization::getValue() + { + return static_cast(1); + } + template <> + inline Half DataInitialization::getValue() + { + return static_cast(2); + } + template <> + inline Half DataInitialization::getValue() + { + return static_cast(-1); + } + template <> + inline Half DataInitialization::getValue() + { + return static_cast(65504.0); + } + template <> + inline Half DataInitialization::getValue() + { + union + { + uint16_t bits; + Half value; + } x; + + x.bits = 0x0001; + return x.value; + } + template <> + inline Half DataInitialization::getValue() + { + union + { + uint16_t bits; + Half value; + } x; + + x.bits = 0x03FF; + return x.value; + } + template <> + inline Half DataInitialization::getValue() + { + union + { + uint16_t bits; + Half value; + } x; + + x.bits = 0xFFFF; + return x.value; + } + template <> + inline Half DataInitialization::getValue() + { + union + { + uint16_t bits; + Half value; + } x; + + x.bits = 0x7C00; + return x.value; + } + + template <> + inline Half DataInitialization::getValue() + { + return static_cast((rand() % 7) - 3); + } + + template <> + inline Half DataInitialization::getValue() + { + return getValue(); + } + template <> + inline Half DataInitialization::getValue() + { + return getValue(); + } + + template <> + inline BFloat16 DataInitialization::getValue() + { + return static_cast(0); + } + template <> + inline BFloat16 DataInitialization::getValue() + { + return static_cast(1); + } + template <> + inline BFloat16 DataInitialization::getValue() + { + return static_cast(2); + } + template <> + inline BFloat16 DataInitialization::getValue() + { + return static_cast(-1); + } + template <> + inline BFloat16 DataInitialization::getValue() + { + return static_cast(DataInitialization::getValue()); + } + template <> + inline BFloat16 DataInitialization::getValue() + { + BFloat16 bf16; + bf16.data = 0x0001; + return bf16; + } + template <> + inline BFloat16 DataInitialization::getValue() + { + BFloat16 bf16; + bf16.data = 0x007F; + return bf16; + } + template <> + inline BFloat16 DataInitialization::getValue() + { + return static_cast(std::numeric_limits::quiet_NaN()); + } + template <> + inline BFloat16 DataInitialization::getValue() + { + return static_cast(std::numeric_limits::infinity()); + } + + template <> + inline BFloat16 DataInitialization::getValue() + { + return static_cast((rand() % 7) - 3); + } + + template <> + inline BFloat16 DataInitialization::getValue() + { + return getValue(); + } + template <> + inline BFloat16 DataInitialization::getValue() + { + return getValue(); + } + + template <> + inline int8_t DataInitialization::getValue() + { + return 0; + } + template <> + inline int8_t DataInitialization::getValue() + { + return 1; + } + template <> + inline int8_t DataInitialization::getValue() + { + return 2; + } + template <> + inline int8_t DataInitialization::getValue() + { + return -1; + } + template <> + inline int8_t DataInitialization::getValue() + { + return 127; + } + template <> + inline int8_t DataInitialization::getValue() + { + throw std::runtime_error("DenormMin not available for int8_t."); + } + template <> + inline int8_t DataInitialization::getValue() + { + throw std::runtime_error("DenormMax not available for int8_t."); + } + template <> + inline int8_t DataInitialization::getValue() + { + throw std::runtime_error("NaN not available for int8_t."); + } + template <> + inline int8_t DataInitialization::getValue() + { + throw std::runtime_error("Inf not available for int8_t."); + } + + template <> + inline int8_t DataInitialization::getValue() + { + return static_cast((rand() % 7) - 3); + } + + template <> + inline int8_t DataInitialization::getValue() + { + return std::numeric_limits::max(); + } + template <> + inline int8_t DataInitialization::getValue() + { + return std::numeric_limits::min(); + } + + template <> + inline bool DataInitialization::isBadInput(float value) + { + return std::isnan(value); + } + + template <> + inline bool DataInitialization::isBadInput(double value) + { + return std::isnan(value); + } + + template <> + inline bool DataInitialization::isBadInput>(std::complex value) + { + return std::isnan(value.real()) && std::isnan(value.imag()); + } + + template <> + inline bool DataInitialization::isBadInput>(std::complex value) + { + return std::isnan(value.real()) && std::isnan(value.imag()); + } + + template <> + inline bool DataInitialization::isBadInput(int32_t value) + { + return value == DataInitialization::getValue(); + } + + template <> + inline bool DataInitialization::isBadInput(Int8x4 value) + { + return value == DataInitialization::getValue(); + } + + template <> + inline bool DataInitialization::isBadInput(Half value) + { + return std::isnan(static_cast(value)); + } + + template <> + inline bool DataInitialization::isBadInput(BFloat16 value) + { + return std::isnan(value); + } + + template <> + inline bool DataInitialization::isBadInput(int8_t value) + { + return value == DataInitialization::getValue(); + } + + template <> + inline bool DataInitialization::isBadOutput(float value) + { + return std::isinf(value); + } + + template <> + inline bool DataInitialization::isBadOutput(double value) + { + return std::isinf(value); + } + + template <> + inline bool DataInitialization::isBadOutput>(std::complex value) + { + return std::isinf(value.real()) && std::isinf(value.imag()); + } + + template <> + inline bool + DataInitialization::isBadOutput>(std::complex value) + { + return std::isinf(value.real()) && std::isinf(value.imag()); + } + + template <> + inline bool DataInitialization::isBadOutput(int32_t value) + { + return value == DataInitialization::getValue(); + } + + template <> + inline bool DataInitialization::isBadOutput(Int8x4 value) + { + return value == DataInitialization::getValue(); + } + + template <> + inline bool DataInitialization::isBadOutput(Half value) + { + return std::isinf(static_cast(value)); + } + + template <> + inline bool DataInitialization::isBadOutput(BFloat16 value) + { + return std::isinf(value); + } + + template <> + inline bool DataInitialization::isBadOutput(int8_t value) + { + return value == DataInitialization::getValue(); + } + + template <> + inline float DataInitialization::getTrigValue(int idx, bool useCos, bool useAbs) + { + float val = useCos ? cos(idx) : sin(idx); + if(useAbs) + val = abs(val); + return val; + } + + template <> + inline double DataInitialization::getTrigValue(int idx, bool useCos, bool useAbs) + { + double val = useCos ? cos(idx) : sin(idx); + if(useAbs) + val = abs(val); + return val; + } + + template <> + inline Half DataInitialization::getTrigValue(int idx, bool useCos, bool useAbs) + { + return static_cast(getTrigValue(idx, useCos, useAbs)); + } + + template <> + inline BFloat16 + DataInitialization::getTrigValue(int idx, bool useCos, bool useAbs) + { + return static_cast(getTrigValue(idx, useCos, useAbs)); + } + + template <> + inline int32_t DataInitialization::getTrigValue(int idx, bool useCos, bool useAbs) + { + throw std::runtime_error("Trig not available for int32_t."); + } + + template <> + inline Int8x4 DataInitialization::getTrigValue(int idx, bool useCos, bool useAbs) + { + throw std::runtime_error("Trig not available for Int8x4."); + } + + template <> + inline int8_t DataInitialization::getTrigValue(int idx, bool useCos, bool useAbs) + { + throw std::runtime_error("Trig not available for Int8."); + } + + template <> + inline std::complex + DataInitialization::getTrigValue>(int idx, bool useCos, bool useAbs) + { + return std::complex(getTrigValue(idx, useCos, useAbs), + getTrigValue(idx, useCos, useAbs)); + } + + template <> + inline std::complex DataInitialization::getTrigValue>( + int idx, bool useCos, bool useAbs) + { + return std::complex(getTrigValue(idx, useCos, useAbs), + getTrigValue(idx, useCos, useAbs)); + } + + template + struct FP_PARAM; + + template <> + struct FP_PARAM + { + using UINT_T = uint64_t; + static constexpr int NUMSIG = 52; + static constexpr int NUMEXP = 11; + }; + + template <> + struct FP_PARAM + { + using UINT_T = uint32_t; + static constexpr int NUMSIG = 23; + static constexpr int NUMEXP = 8; + }; + + template <> + struct FP_PARAM + { + using UINT_T = uint16_t; + static constexpr int NUMSIG = 7; + static constexpr int NUMEXP = 8; + }; + + template <> + struct FP_PARAM + { + using UINT_T = uint16_t; + static constexpr int NUMSIG = 10; + static constexpr int NUMEXP = 5; + }; + + template + struct rocm_random_common : FP_PARAM + { + using typename FP_PARAM::UINT_T; + using FP_PARAM::NUMSIG; + using FP_PARAM::NUMEXP; + using random_fp_int_dist = std::uniform_int_distribution; + + static_assert(sizeof(UINT_T) == sizeof(T), "Type sizes do not match"); + static constexpr UINT_T expmask = (((UINT_T)1 << NUMEXP) - 1) << NUMSIG; + static constexpr UINT_T expbias = ((UINT_T)1 << (NUMEXP - 1)) - 1; + inline static T signsig_exp(UINT_T signsig, UINT_T exp) + { + union + { + UINT_T u; + T fp; + }; + u = signsig & ~expmask | ((exp + expbias) << NUMSIG) & expmask; + return fp; + } + }; + + template <> + inline BFloat16 + rocm_random_common::signsig_exp(FP_PARAM::UINT_T signsig, + FP_PARAM::UINT_T exp) + { + FP_PARAM::UINT_T u; + u = signsig & ~expmask | ((exp + expbias) << NUMSIG) & expmask; + return static_cast(u); + } + + template + struct rocm_random : rocm_random_common + { + using typename rocm_random_common::random_fp_int_dist; + __attribute__((flatten)) T operator()() + { + static std::mt19937 rng; + int exp = std::uniform_int_distribution{}(rng); + exp = exp % (HIGH_EXP - LOW_EXP + 1) + LOW_EXP; + return this->signsig_exp(random_fp_int_dist{}(rng), exp); + } + }; + + template + struct rocm_random_narrow_range; + + template <> + struct rocm_random_narrow_range : rocm_random + { + }; + + template <> + struct rocm_random_narrow_range : rocm_random + { + }; + + template <> + struct rocm_random_narrow_range : rocm_random + { + }; + + template <> + struct rocm_random_narrow_range : rocm_random + { + }; + + template <> + inline float DataInitialization::getValue() + { + return rocm_random_narrow_range{}(); + } + + template <> + inline double DataInitialization::getValue() + { + return rocm_random_narrow_range{}(); + } + + template <> + inline BFloat16 DataInitialization::getValue() + { + return rocm_random_narrow_range{}(); + } + + template <> + inline Half DataInitialization::getValue() + { + return rocm_random_narrow_range{}(); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::RandomNarrow>() + { + return std::complex(rocm_random_narrow_range{}(), + rocm_random_narrow_range{}()); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::RandomNarrow>() + { + return std::complex(rocm_random_narrow_range{}(), + rocm_random_narrow_range{}()); + } + + template <> + inline int32_t DataInitialization::getValue() + { + return getValue(); + } + + template <> + inline Int8x4 DataInitialization::getValue() + { + return getValue(); + } + + template <> + inline int8_t DataInitialization::getValue() + { + return getValue(); + } + + template + inline T getValueWithUpperLowerBoundFP(double upper = 1.0, double lower = -1.0) + { + return static_cast(lower + + static_cast(rand()) + / static_cast(RAND_MAX / (upper - lower))); + } + + template + inline T getValueWithUpperLowerBoundInteger(int upper = 128, int lower = -128) + { + return static_cast(lower + rand() % (upper - lower + 1)); + } + + template <> + inline float DataInitialization::getValue() + { + return getValueWithUpperLowerBoundFP(); + } + + template <> + inline double DataInitialization::getValue() + { + return getValueWithUpperLowerBoundFP(); + } + + template <> + inline BFloat16 DataInitialization::getValue() + { + return getValueWithUpperLowerBoundFP(); + } + + template <> + inline Half DataInitialization::getValue() + { + return getValueWithUpperLowerBoundFP(); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::RandomNegPosLimited>() + { + return std::complex(getValueWithUpperLowerBoundFP(), + getValueWithUpperLowerBoundFP()); + } + + template <> + inline std::complex + DataInitialization::getValue, InitMode::RandomNegPosLimited>() + { + return std::complex(getValueWithUpperLowerBoundFP(), + getValueWithUpperLowerBoundFP()); + } + + template <> + inline int32_t DataInitialization::getValue() + { + return getValueWithUpperLowerBoundInteger(); + } + + template <> + inline Int8x4 DataInitialization::getValue() + { + return Int8x4{getValueWithUpperLowerBoundInteger(), + getValueWithUpperLowerBoundInteger(), + getValueWithUpperLowerBoundInteger(), + getValueWithUpperLowerBoundInteger()}; + } + + template <> + inline int8_t DataInitialization::getValue() + { + return getValueWithUpperLowerBoundInteger(); + } + + template <> + inline float DataInitialization::ConvertTo(size_t i) + { + return static_cast(i); + } + + template <> + inline double DataInitialization::ConvertTo(size_t i) + { + return static_cast(i); + } + + template <> + inline BFloat16 DataInitialization::ConvertTo(size_t i) + { + return static_cast(i); + } + + template <> + inline Half DataInitialization::ConvertTo(size_t i) + { + return static_cast(i); + } + + template <> + inline std::complex + DataInitialization::ConvertTo>(size_t i) + { + return std::complex(static_cast(i), + static_cast(i)); + } + + template <> + inline std::complex + DataInitialization::ConvertTo>(size_t i) + { + return std::complex(static_cast(i), + static_cast(i)); + } + + template <> + inline int32_t DataInitialization::ConvertTo(size_t i) + { + return static_cast(i); + } + + template <> + inline Int8x4 DataInitialization::ConvertTo(size_t i) + { + return Int8x4{static_cast(i), + static_cast(i), + static_cast(i), + static_cast(i)}; + } + + template <> + inline int8_t DataInitialization::ConvertTo(size_t i) + { + return static_cast(i); + } + + template <> + inline float DataInitialization::convertDoubleTo(double value) + { + return static_cast(value); + } + + template <> + inline double DataInitialization::convertDoubleTo(double value) + { + return value; + } + + template <> + inline BFloat16 DataInitialization::convertDoubleTo(double value) + { + return static_cast(value); + } + + template <> + inline Half DataInitialization::convertDoubleTo(double value) + { + return static_cast(value); + } + + template <> + inline std::complex + DataInitialization::convertDoubleTo>(double value) + { + throw std::runtime_error("convertDoubleTo not available for std::complex."); + } + + template <> + inline std::complex + DataInitialization::convertDoubleTo>(double value) + { + throw std::runtime_error("convertDoubleTo not available for std::complex."); + } + + template <> + inline int32_t DataInitialization::convertDoubleTo(double value) + { + return static_cast(value); + } + + template <> + inline Int8x4 DataInitialization::convertDoubleTo(double value) + { + throw std::runtime_error("convertDoubleTo not available for Int8x4."); + } + + template <> + inline int8_t DataInitialization::convertDoubleTo(double value) + { + return static_cast(value); + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/DataInitializationTyped.hpp b/tensilelite/Tensile/Source/client/include/DataInitializationTyped.hpp new file mode 100644 index 0000000000..8cff5cdff9 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/DataInitializationTyped.hpp @@ -0,0 +1,981 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "DataInitialization.hpp" + +#include + +#include +#include +#include + +#include // memcpy + +namespace Tensile +{ + namespace Client + { + template + struct ManagedContractionInputs : public TypedContractionInputs + { + using Base = TypedContractionInputs; + using AType = A; + using BType = B; + using CType = C; + using DType = D; + using AlphaType = Alpha; + using BetaType = Beta; + + ManagedContractionInputs(std::shared_ptr _a, + std::shared_ptr _b, + std::shared_ptr _c, + std::shared_ptr _d, + std::shared_ptr _batchA, + std::shared_ptr _batchB, + std::shared_ptr _batchC, + std::shared_ptr _batchD, + size_t _aElements, + size_t _bElements, + size_t _cElements, + size_t _dElements, + Alpha _alpha, + Beta _beta, + bool _gpu, + std::shared_ptr _bias = nullptr, + std::shared_ptr _ws = nullptr, + size_t _workspaceSize = 0) + + : Base(_a.get(), + _b.get(), + _c.get(), + _d.get(), + _batchA.get(), + _batchB.get(), + _batchC.get(), + _batchD.get(), + _alpha, + _beta, + _bias.get(), + _ws.get()) + , managedA(_a) + , managedB(_b) + , managedC(_c) + , managedD(_d) + , managedBatchA(_batchA) + , managedBatchB(_batchB) + , managedBatchC(_batchC) + , managedBatchD(_batchD) + , aElements(_aElements) + , bElements(_bElements) + , cElements(_cElements) + , dElements(_dElements) + , gpu(_gpu) + , managedBias(_bias) + , managedWS(_ws) + , workspaceSize(_workspaceSize) + { + } + + ~ManagedContractionInputs() = default; + + std::shared_ptr managedA; + std::shared_ptr managedB; + std::shared_ptr managedC; + std::shared_ptr managedD; + std::shared_ptr managedBatchA; + std::shared_ptr managedBatchB; + std::shared_ptr managedBatchC; + std::shared_ptr managedBatchD; + std::shared_ptr managedWS; + std::shared_ptr managedBias; + + size_t aElements; + size_t bElements; + size_t cElements; + size_t dElements; + size_t workspaceSize; + + bool gpu; + }; + + template + class TypedDataInitialization : public DataInitialization + { + public: + using AType = typename TypedInputs::AType; + using BType = typename TypedInputs::BType; + using CType = typename TypedInputs::CType; + using DType = typename TypedInputs::DType; + using AlphaType = typename TypedInputs::AlphaType; + using BetaType = typename TypedInputs::BetaType; + using ManagedInputs + = ManagedContractionInputs; + + TypedDataInitialization(po::variables_map const& args, + ClientProblemFactory const& problemFactory, + size_t maxWorkspaceSize = 0) + : DataInitialization(args, problemFactory, maxWorkspaceSize) + { + } + + virtual std::shared_ptr + prepareCPUInputs(ContractionProblem const& problem) + { + return prepareCPUInputsTyped(problem); + } + + virtual std::shared_ptr + prepareGPUInputs(ContractionProblem const& problem) + { + if(m_numRunsInSolution > 0 && m_curBoundsCheck == BoundsCheckMode::GuardPageFront + && m_boundsCheck == BoundsCheckMode::GuardPageAll) + m_curBoundsCheck = BoundsCheckMode::GuardPageBack; + + return prepareGPUInputsTyped(problem); + } + + std::shared_ptr prepareCPUInputsTyped(ContractionProblem const& problem) + { + if(!m_cpuInputsPristine) + m_cpuInputsPristine = createNewCPUInputs(problem); + + if(m_cpuInputs && m_curBoundsCheck == BoundsCheckMode::Disable + && !m_problemDependentData) + { + copyD(m_cpuInputs, m_cpuInputsPristine); + } + else + { + if(!m_cpuInputs) + m_cpuInputs = allocNewCPUInputs(); + + if(m_problemDependentData) + initializeCPUInputs(*m_cpuInputsPristine, problem); + + if(m_curBoundsCheck == BoundsCheckMode::NaN && !m_cpuBadInputs) + { + m_cpuBadInputs = createNewCPUBadInputs(); + } + + copyInputs(m_cpuInputs, m_cpuInputsPristine, m_cpuBadInputs, problem); + } + + if(m_convolutionVsContraction) + { + bool allocated = false; + if(!m_cpuConvInputs) + { + allocated = true; + m_cpuConvInputs = allocNewCPUInputs(); + } + + if(allocated || m_curBoundsCheck == BoundsCheckMode::NaN) + copyInputs(m_cpuConvInputs, m_cpuInputsPristine, m_cpuBadInputs, problem); + } + + return m_cpuInputs; + } + + virtual std::shared_ptr cpuConvInputs() const + { + return m_cpuConvInputs; + }; + + template + void initGPUBatchedInput(T base, + T* array, + TensorDescriptor const& tensor, + const std::vector& batchIdx) + { + std::vector batchSizes; + std::vector batchStrides; + for(auto& idx : batchIdx) + { + batchSizes.push_back(tensor.sizes().at(idx)); + batchStrides.push_back(tensor.strides().at(idx)); + } + std::vector coord(batchSizes.size(), 0); + + auto count = CoordCount(batchSizes.begin(), batchSizes.end()); + + T* cpuArray = (T*)std::malloc(count * sizeof(T)); + for(size_t idx = 0; idx < count; idx++) + { + CoordNumbered( + idx, coord.begin(), coord.end(), batchSizes.begin(), batchSizes.end()); + cpuArray[idx] = base; + for(size_t i = 0; i < batchSizes.size(); i++) + { + cpuArray[idx] += coord[i] * batchStrides[i]; + } + } + + HIP_CHECK_EXC(hipMemcpy(array, cpuArray, count * sizeof(T), hipMemcpyHostToDevice)); + + std::free(cpuArray); + } + + void initializeGPUBatchedInputs(ManagedInputs& inputs, + ContractionProblem const& problem) + { + if(!inputs.gpu) + throw std::runtime_error("Initializing GPU batched inputs"); + + auto batchIdxs = problem.batchIndices(); + std::vector batchIdxA(batchIdxs.size(), 0); + std::vector batchIdxB(batchIdxs.size(), 0); + std::vector batchIdxC(batchIdxs.size(), 0); + std::vector batchIdxD(batchIdxs.size(), 0); + + ptrdiff_t aPadding = 0; + ptrdiff_t bPadding = 0; + ptrdiff_t cPadding = 0; + ptrdiff_t dPadding = 0; + + for(size_t i = 0; i < batchIdxs.size(); i++) + { + batchIdxA[i] = batchIdxs[i].a; + batchIdxB[i] = batchIdxs[i].b; + batchIdxC[i] = batchIdxs[i].c; + batchIdxD[i] = batchIdxs[i].d; + } + + if(m_curBoundsCheck == BoundsCheckMode::NaN) + { + aPadding = (inputs.aElements - problem.a().totalAllocatedElements()) / 2; + bPadding = (inputs.bElements - problem.b().totalAllocatedElements()) / 2; + cPadding = (inputs.cElements - problem.c().totalAllocatedElements()) / 2; + dPadding = (inputs.dElements - problem.d().totalAllocatedElements()) / 2; + } + else if(m_curBoundsCheck == BoundsCheckMode::GuardPageBack) + { + aPadding = inputs.aElements - problem.a().totalAllocatedElements(); + bPadding = inputs.bElements - problem.b().totalAllocatedElements(); + cPadding = inputs.cElements - problem.c().totalAllocatedElements(); + dPadding = inputs.dElements - problem.d().totalAllocatedElements(); + } + + initGPUBatchedInput(inputs.managedA.get() + aPadding, + inputs.managedBatchA.get(), + problem.a(), + batchIdxA); + initGPUBatchedInput(inputs.managedB.get() + bPadding, + inputs.managedBatchB.get(), + problem.b(), + batchIdxB); + initGPUBatchedInput(inputs.managedC.get() + cPadding, + inputs.managedBatchC.get(), + problem.c(), + batchIdxC); + initGPUBatchedInput(inputs.managedD.get() + dPadding, + inputs.managedBatchD.get(), + problem.d(), + batchIdxD); + } + + std::shared_ptr prepareGPUInputsTyped(ContractionProblem const& problem) + { + std::shared_ptr pristine; + std::shared_ptr bad; + + if(m_keepPristineCopyOnGPU && !m_problemDependentData) + { + if(!m_gpuInputsPristine) + m_gpuInputsPristine = createNewGPUInputs(problem); + + pristine = m_gpuInputsPristine; + + if(m_curBoundsCheck == BoundsCheckMode::NaN) + { + if(!m_gpuBadInputs) + m_gpuBadInputs = createNewGPUBadInputs(); + + bad = m_gpuBadInputs; + } + } + else + { + if(!m_cpuInputsPristine) + m_cpuInputsPristine = createNewCPUInputs(problem); + + pristine = m_cpuInputsPristine; + + if(m_curBoundsCheck == BoundsCheckMode::NaN) + { + if(!m_cpuBadInputs) + m_cpuBadInputs = createNewCPUBadInputs(); + + bad = m_cpuBadInputs; + } + } + + if(m_gpuInputs && m_curBoundsCheck == BoundsCheckMode::Disable + && !m_problemDependentData) + { + if(m_elementsToValidate) + copyD(m_gpuInputs, pristine); + } + else + { + if(!m_gpuInputs) + m_gpuInputs = allocNewGPUInputs(pristine); + + if(m_problemDependentData) + initializeCPUInputs(*m_cpuInputsPristine, problem); + + copyInputs(m_gpuInputs, pristine, bad, problem); + } + + initializeGPUBatchedInputs(*m_gpuInputs, problem); + + return m_gpuInputs; + } + + std::shared_ptr createNewCPUInputs(ContractionProblem const& problem) + { + auto rv = allocNewCPUInputs(); + initializeCPUInputs(*rv, problem); + + return rv; + } + + std::shared_ptr createNewGPUInputs(ContractionProblem const& problem) + { + auto rv = allocNewGPUInputs(); + if(!m_cpuInputsPristine) + m_cpuInputsPristine = createNewCPUInputs(problem); + copyInputBuffers(rv, m_cpuInputsPristine); + + return rv; + } + + std::shared_ptr createNewCPUBadInputs() + { + auto rv = allocNewCPUInputs(); + initializeCPUBadInputs(*rv); + + return rv; + } + + std::shared_ptr createNewGPUBadInputs() + { + auto rv = allocNewGPUInputs(); + if(!m_cpuBadInputs) + m_cpuBadInputs = createNewCPUBadInputs(); + copyInputBuffers(rv, m_cpuBadInputs); + + return rv; + } + + std::shared_ptr allocNewCPUInputs(std::shared_ptr pristine + = nullptr) + { + std::shared_ptr a; + std::shared_ptr b; + std::shared_ptr c; + std::shared_ptr d; + std::shared_ptr bias; + + if(pristine) + { + a = pristine->managedA; + b = pristine->managedB; + } + else + { + a = std::shared_ptr( + (AType*)std::malloc(TypeInfo::ElementSize * m_aMaxElements), + std::free); + if(a == nullptr) + throw std::runtime_error("out of host memory allocating a"); + b = std::shared_ptr( + (BType*)std::malloc(TypeInfo::ElementSize * m_bMaxElements), + std::free); + if(a == nullptr) + throw std::runtime_error("out of host memory allocating b"); + } + + if(m_cEqualsD || !pristine) + { + c = std::shared_ptr( + (CType*)std::malloc(TypeInfo::ElementSize * m_cMaxElements), + std::free); + if(c == nullptr) + throw std::runtime_error("out of host memory allocating c"); + } + else + { + c = pristine->managedC; + } + + if(m_cEqualsD) + { + d = c; + } + else if(pristine) + { + d = pristine->managedD; + } + else + { + d = std::shared_ptr( + (DType*)std::malloc(TypeInfo::ElementSize * m_dMaxElements), + std::free); + if(d == nullptr) + throw std::runtime_error("out of host memory allocating d"); + } + + auto alpha = static_cast(0); + auto beta = static_cast(0); + + if(pristine) + { + bias = pristine->managedBias; + } + else + { + bias = std::shared_ptr( + (AType*)std::malloc(TypeInfo::ElementSize * m_dMaxElements), + std::free); + if(bias == nullptr) + throw std::runtime_error("out of host memory allocating d"); + } + + auto rv = std::make_shared(a, + b, + c, + d, + std::shared_ptr(), + std::shared_ptr(), + std::shared_ptr(), + std::shared_ptr(), + m_aMaxElements, + m_bMaxElements, + m_cMaxElements, + m_dMaxElements, + alpha, + beta, + false, + bias); + + return rv; + } + + template + std::shared_ptr allocNewGPUBuffer(const char* title, size_t size) + { + static const int sizew = 10; + T* ptr = nullptr; + HIP_CHECK_EXC(hipMalloc(&ptr, size)); + auto p = std::shared_ptr(ptr, hipFree); + if(Debug::Instance().printTensorInfo()) + std::cout << "info: allocate " << title << " " << std::setw(sizew) << size + << " bytes at " << static_cast(ptr) << "\n"; + return p; + } + + std::shared_ptr allocNewGPUInputs(std::shared_ptr pristine + = nullptr) + { + if(m_curBoundsCheck != BoundsCheckMode::Disable || (pristine && !pristine->gpu)) + pristine = nullptr; + + std::shared_ptr a; + std::shared_ptr b; + std::shared_ptr c; + std::shared_ptr d; + std::shared_ptr bias; + + std::shared_ptr batch_a; + std::shared_ptr batch_b; + std::shared_ptr batch_c; + std::shared_ptr batch_d; + + std::shared_ptr ws; + static const int sizew = 10; + + std::vector> guardPage; + void* guardPagePtr; + bool enableGuardPage = (m_curBoundsCheck == BoundsCheckMode::GuardPageFront + || m_curBoundsCheck == BoundsCheckMode::GuardPageBack); + + if(pristine) + { + a = pristine->managedA; + b = pristine->managedB; + batch_a = pristine->managedBatchA; + batch_b = pristine->managedBatchB; + } + else + { + if(enableGuardPage) + { + HIP_CHECK_EXC(hipMalloc(&guardPagePtr, pageSize)); + guardPage.push_back(std::shared_ptr(guardPagePtr, hipFree)); + } + + a = allocNewGPUBuffer("a", + TypeInfo::ElementSize * m_aMaxElements); + batch_a = allocNewGPUBuffer("batchA", sizeof(AType*) * m_maxBatch); + + if(enableGuardPage) + { + HIP_CHECK_EXC(hipMalloc(&guardPagePtr, pageSize)); + guardPage.push_back(std::shared_ptr(guardPagePtr, hipFree)); + } + + b = allocNewGPUBuffer("b", + TypeInfo::ElementSize * m_bMaxElements); + batch_b = allocNewGPUBuffer("batchB", sizeof(BType*) * m_maxBatch); + } + + if(m_cEqualsD || !pristine) + { + if(enableGuardPage) + { + HIP_CHECK_EXC(hipMalloc(&guardPagePtr, pageSize)); + guardPage.push_back(std::shared_ptr(guardPagePtr, hipFree)); + } + + c = allocNewGPUBuffer("c", + TypeInfo::ElementSize * m_cMaxElements); + batch_c = allocNewGPUBuffer("batchC", sizeof(CType*) * m_maxBatch); + } + else + { + c = pristine->managedC; + batch_c = pristine->managedBatchC; + } + + if(m_cEqualsD) + { + d = c; + batch_d = batch_c; + } + else if(pristine) + { + d = pristine->managedD; + batch_d = pristine->managedBatchD; + } + else + { + if(enableGuardPage) + { + HIP_CHECK_EXC(hipMalloc(&guardPagePtr, pageSize)); + guardPage.push_back(std::shared_ptr(guardPagePtr, hipFree)); + } + + d = allocNewGPUBuffer("d", + TypeInfo::ElementSize * m_dMaxElements); + batch_d = allocNewGPUBuffer("batchD", sizeof(DType*) * m_maxBatch); + } + + if(enableGuardPage) + { + HIP_CHECK_EXC(hipMalloc(&guardPagePtr, pageSize)); + guardPage.push_back(std::shared_ptr(guardPagePtr, hipFree)); + } + + if(pristine) + { + ws = pristine->managedWS; + } + else + { + ws = allocNewGPUBuffer("ws", m_workspaceSize); + } + + auto alpha = static_cast(0); + auto beta = static_cast(0); + + if(pristine) + { + bias = pristine->managedBias; + } + else + { + bias = allocNewGPUBuffer("bias", + TypeInfo::ElementSize * m_dMaxElements); + } + auto rv = std::make_shared(a, + b, + c, + d, + batch_a, + batch_b, + batch_c, + batch_d, + m_aMaxElements, + m_bMaxElements, + m_cMaxElements, + m_dMaxElements, + alpha, + beta, + true, + bias, + ws, + m_workspaceSize); + return rv; + } + + void initializeCPUInputs(ManagedInputs& inputs, ContractionProblem const& problem) + { + if(inputs.gpu) + throw std::runtime_error("Initializing GPU inputs as CPU."); + + if(m_problem.a() != problem.a() || m_problem.b() != problem.b() + || m_problem.c() != problem.c() || (!m_cEqualsD && m_problem.d() != problem.d())) + { + if(m_problemDependentData) + { + initArray(m_aInit, inputs.managedA.get(), problem.a()); + initArray(m_bInit, inputs.managedB.get(), problem.b()); + initArray(m_cInit, inputs.managedC.get(), problem.c()); + if(!m_cEqualsD) + initArray(m_dInit, inputs.managedD.get(), problem.d()); + } + else + { + initArray(m_aInit, inputs.managedA.get(), m_aMaxElements); + initArray(m_bInit, inputs.managedB.get(), m_bMaxElements); + initArray(m_cInit, inputs.managedC.get(), m_cMaxElements); + if(!m_cEqualsD) + initArray(m_dInit, inputs.managedD.get(), m_dMaxElements); + } + + inputs.alpha = getValue(m_alphaInit); + inputs.beta = getValue(m_betaInit); + + if(problem.useBias()) + { + auto size = problem.d().sizes()[0]; + initArray(m_biasInit, inputs.managedBias.get(), size); + } + + for(int i = 0; i < getAdditionalArgNum(problem.activationType()); i++) + { + double value = 0.0; + if(m_activationAdditionalArgs.empty()) + { + value = getValueWithUpperLowerBoundFP(2.0, -2.0); + // Cannot be zero when DType = integers + if(convertDoubleTo(value) == static_cast(0)) + { + value = std::copysign(std::ceil(std::abs(value)), value); + } + } + else + { + const auto& actArgs = m_activationAdditionalArgs[0]; + value + = (i == actArgs.size()) ? actArgs[actArgs.size() - 1] : actArgs[i]; + } + inputs.activationArgs.push_back(convertDoubleTo(value)); + } + + m_problem = problem; + } + } + + void initializeCPUBadInputs(ManagedInputs& inputs) + { + if(inputs.gpu) + throw std::runtime_error("Initializing GPU inputs as CPU."); + + initArray(InitMode::BadInput, inputs.managedA.get(), m_aMaxElements); + initArray(InitMode::BadInput, inputs.managedB.get(), m_bMaxElements); + + initArray(InitMode::BadOutput, inputs.managedD.get(), m_dMaxElements); + + if(!m_cEqualsD) + initArray(InitMode::BadInput, inputs.managedC.get(), m_cMaxElements); + + inputs.alpha = getValue(m_alphaInit); + inputs.beta = getValue(m_betaInit); + + for(int i = 0; i < getAdditionalArgNum(m_problem.activationType()); i++) + { + const auto& actArgs = m_activationAdditionalArgs[0]; + double value = (i == actArgs.size()) ? actArgs[actArgs.size() - 1] : actArgs[i]; + inputs.activationArgs.push_back(convertDoubleTo(value)); + } + } + + hipMemcpyKind getCopyKind(std::shared_ptr dst, + std::shared_ptr src) + { + if(src->gpu) + { + if(dst->gpu) + { + return hipMemcpyDeviceToDevice; + } + else + { + return hipMemcpyDeviceToHost; + } + } + else + { + if(dst->gpu) + { + return hipMemcpyHostToDevice; + } + else + { + return hipMemcpyHostToHost; + } + } + } + + void copyInputBuffers(std::shared_ptr dst, + std::shared_ptr src) + { + hipMemcpyKind kind = getCopyKind(dst, src); + + if(dst->managedA != src->managedA) + HIP_CHECK_EXC(hipMemcpy(dst->managedA.get(), + src->managedA.get(), + TypeInfo::ElementSize * m_aMaxElements, + kind)); + + if(dst->managedB != src->managedB) + HIP_CHECK_EXC(hipMemcpy(dst->managedB.get(), + src->managedB.get(), + TypeInfo::ElementSize * m_bMaxElements, + kind)); + + if(dst->managedC != src->managedC) + HIP_CHECK_EXC(hipMemcpy(dst->managedC.get(), + src->managedC.get(), + TypeInfo::ElementSize * m_cMaxElements, + kind)); + + if(!m_cEqualsD && dst->managedD != src->managedD) + HIP_CHECK_EXC(hipMemcpy(dst->managedD.get(), + src->managedD.get(), + TypeInfo::ElementSize * m_dMaxElements, + kind)); + + HIP_CHECK_EXC(hipMemcpy(dst->managedBias.get(), + src->managedBias.get(), + TypeInfo::ElementSize * m_dMaxElements, + kind)); + + dst->alpha = src->alpha; + dst->beta = src->beta; + + dst->activationArgs = src->activationArgs; + } + + void copyInputs(std::shared_ptr dst, + std::shared_ptr src, + std::shared_ptr bad, + ContractionProblem const& problem) + { + hipMemcpyKind kind = getCopyKind(dst, src); + + if(m_curBoundsCheck == BoundsCheckMode::NaN) + { + if(!bad) + throw std::runtime_error( + "bad inputs must be initialized for bounds check!"); + if(bad->gpu != src->gpu) + throw std::runtime_error( + "bad inputs must be in the same location as the source"); + if(dst->managedA == src->managedA) + throw std::runtime_error("A pointers are equal for bounds check!"); + if(dst->managedB == src->managedB) + throw std::runtime_error("B pointers are equal for bounds check!"); + if(dst->managedC == src->managedC) + throw std::runtime_error("C pointers are equal for bounds check!"); + if(dst->managedD == src->managedD) + throw std::runtime_error("D pointers are equal for bounds check!"); + + copyInputBuffers(dst, bad); + + { + ptrdiff_t aPadding = dst->aElements - problem.a().totalAllocatedElements(); + dst->a = dst->managedA.get() + aPadding / 2; + } + + { + ptrdiff_t bPadding = dst->bElements - problem.b().totalAllocatedElements(); + dst->b = dst->managedB.get() + bPadding / 2; + } + + { + ptrdiff_t cPadding = dst->cElements - problem.c().totalAllocatedElements(); + dst->c = dst->managedC.get() + cPadding / 2; + } + + { + ptrdiff_t dPadding = dst->dElements - problem.d().totalAllocatedElements(); + dst->d = dst->managedD.get() + dPadding / 2; + } + + Tensile::hip::CopyTensor(const_cast(dst->a), src->a, problem.a(), kind); + Tensile::hip::CopyTensor(const_cast(dst->b), src->b, problem.b(), kind); + + Tensile::hip::CopyTensor(const_cast(dst->c), src->c, problem.c(), kind); + + if(!m_cEqualsD) + Tensile::hip::CopyTensor(dst->d, src->d, problem.d(), kind); + + memcpy(const_cast(dst->bias), src->bias, problem.d().sizes()[0] * TypeInfo::ElementSize); + + dst->alpha = src->alpha; + dst->beta = src->beta; + + dst->activationArgs = src->activationArgs; + } + else if(m_curBoundsCheck == BoundsCheckMode::GuardPageBack) + { + { + ptrdiff_t aPadding = dst->aElements - problem.a().totalAllocatedElements(); + dst->a = dst->managedA.get() + aPadding; + } + + { + ptrdiff_t bPadding = dst->bElements - problem.b().totalAllocatedElements(); + dst->b = dst->managedB.get() + bPadding; + } + + { + ptrdiff_t cPadding = dst->cElements - problem.c().totalAllocatedElements(); + dst->c = dst->managedC.get() + cPadding; + } + + { + ptrdiff_t dPadding = dst->dElements - problem.d().totalAllocatedElements(); + dst->d = dst->managedD.get() + dPadding; + } + HIP_CHECK_EXC(hipMemcpy(const_cast(dst->a), + src->a, + TypeInfo::ElementSize + * problem.a().totalAllocatedElements(), + kind)); + HIP_CHECK_EXC(hipMemcpy(const_cast(dst->b), + src->b, + TypeInfo::ElementSize + * problem.b().totalAllocatedElements(), + kind)); + HIP_CHECK_EXC(hipMemcpy(const_cast(dst->c), + src->c, + TypeInfo::ElementSize + * problem.c().totalAllocatedElements(), + kind)); + + if(!m_cEqualsD) + HIP_CHECK_EXC(hipMemcpy(const_cast(dst->d), + src->d, + TypeInfo::ElementSize + * problem.d().totalAllocatedElements(), + kind)); + + HIP_CHECK_EXC(hipMemcpy(const_cast(dst->bias), + src->bias, + TypeInfo::ElementSize + * problem.d().sizes()[0], + kind)); + + dst->alpha = src->alpha; + dst->beta = src->beta; + + dst->activationArgs = src->activationArgs; + } + else + { + copyInputBuffers(dst, src); + } + } + + void copyD(std::shared_ptr dst, std::shared_ptr src) + { + hipMemcpyKind kind = getCopyKind(dst, src); + + HIP_CHECK_EXC(hipMemcpy(dst->managedD.get(), + src->managedD.get(), + TypeInfo::ElementSize * m_dMaxElements, + kind)); + } + + private: + /** + * Depending on user configuration, the actual pointers within these inputs structs may not + * all point to separately allocated buffers. + */ + + std::shared_ptr m_cpuConvInputs; + std::shared_ptr m_cpuInputsPristine; //< Untouched copies of the inputs + std::shared_ptr + m_cpuInputs; //< Inputs used for CPU reference calculations + std::shared_ptr + m_cpuBadInputs; //< Inputs containing 'bad' values for bounds checking + std::shared_ptr m_gpuInputsPristine; //< Untouched copies of the inputs + std::shared_ptr m_gpuInputs; //< Inputs to be sent in to GPU kernels + std::shared_ptr m_gpuBadInputs; //< GPU copies of 'bad' values + ContractionProblem + m_problem; //< Contraction problem for which current inputs are initialized + }; + + // Commonly used managed contraction input type groupings + // Naming: _[Ti_To_Tc]_: + // S=float, D=double, C=complex, Z=complex, + // H=Half, B=BF16, I8x4=Int8x4, I32=int32_t + using ManagedContractionInputs_S_S_S = ManagedContractionInputs; + using ManagedContractionInputs_D_D_D = ManagedContractionInputs; + using ManagedContractionInputs_C_C_C = ManagedContractionInputs>; + using ManagedContractionInputs_Z_Z_Z = ManagedContractionInputs>; +#ifdef TENSILE_USE_HALF + using ManagedContractionInputs_H_H_H = ManagedContractionInputs; + using ManagedContractionInputs_H_H_S + = ManagedContractionInputs; + using ManagedContractionInputs_H_S_S = ManagedContractionInputs; +#endif // TENSILE_USE_HALF + using ManagedContractionInputs_I8x4_I32_I32 + = ManagedContractionInputs; + using ManagedContractionInputs_I8_I8_I32 + = ManagedContractionInputs; + using ManagedContractionInputs_I8_I32_I32 + = ManagedContractionInputs; + using ManagedContractionInputs_I32_I32_I32 = ManagedContractionInputs; +#ifdef TENSILE_USE_BF16 + using ManagedContractionInputs_B_B_S + = ManagedContractionInputs; + using ManagedContractionInputs_B_S_S + = ManagedContractionInputs; +#endif // TENSILE_USE_BF16 + + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/HardwareMonitor.hpp b/tensilelite/Tensile/Source/client/include/HardwareMonitor.hpp new file mode 100644 index 0000000000..2146305b19 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/HardwareMonitor.hpp @@ -0,0 +1,149 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace Tensile +{ + namespace Client + { + /** + * Monitors properties of a particular GPU in a separate thread. + * + * The thread is manually managed because the thread creation overhead is too + * high to create a thread every time. + * + * The interface to this class is not thread-safe. + */ + class HardwareMonitor + { + public: + /** Translates the Hip device index into the corresponding device index for + * ROCm-SMI. */ + static uint32_t GetROCmSMIIndex(int hipDeviceIndex); + + using rsmi_temperature_type_t = int; + using clock = std::chrono::steady_clock; + + // Monitor at the maximum possible rate. + HardwareMonitor(int hipDeviceIndex); + // Limit collection to once per minPeriod. + HardwareMonitor(int hipDeviceIndex, clock::duration minPeriod); + + ~HardwareMonitor(); + + void addTempMonitor(rsmi_temperature_type_t sensorType = 0, + rsmi_temperature_metric_t metric = RSMI_TEMP_CURRENT); + void addClockMonitor(rsmi_clk_type_t clockType); + void addFanSpeedMonitor(uint32_t sensorIndex = 0); + + double getAverageTemp(rsmi_temperature_type_t sensorIndex = 0, + rsmi_temperature_metric_t metric = RSMI_TEMP_CURRENT); + double getAverageClock(rsmi_clk_type_t clockType); + double getAverageFanSpeed(uint32_t sensorIndex = 0); + int getDeviceIndex() + { + return m_hipDeviceIndex; + } + size_t getSamples() + { + return m_dataPoints; + } + + /// Begins monitoring until stop() is called. + void start(); + + /// Sends a signal to the monitoring thread to end monitoring. + void stop(); + + /// Begins monitoring immediately, until the event has occurred. + void runUntilEvent(hipEvent_t event); + + /// Monitoring will occur from startEvent until stopEvent. + void runBetweenEvents(hipEvent_t startEvent, hipEvent_t stopEvent); + + /// Waits until monitoring has finished. + /// Throws an exception if monitoring was started without a stop event + /// and stop() has not been called. + void wait(); + + private: + static void InitROCmSMI(); + + void assertActive(); + void assertNotActive(); + + void clearValues(); + void collectOnce(); + void sleepIfNecessary(); + + void initThread(); + void runLoop(); + void collect(hipEvent_t startEvent, hipEvent_t stopEvent); + + clock::time_point m_lastCollection; + clock::time_point m_nextCollection; + clock::duration m_minPeriod; + + std::thread m_thread; + + std::mutex m_mutex; + std::condition_variable m_cv; + + using Task = std::packaged_task; + Task m_task; + std::future m_future; + std::atomic m_exit; + std::atomic m_stop; + bool m_hasStopEvent = false; + + int m_hipDeviceIndex; + uint32_t m_smiDeviceIndex; + + size_t m_dataPoints; + + std::vector> + m_tempMetrics; + std::vector m_tempValues; + + std::vector m_clockMetrics; + std::vector m_clockValues; + + std::vector m_fanMetrics; + std::vector m_fanValues; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/HardwareMonitorListener.hpp b/tensilelite/Tensile/Source/client/include/HardwareMonitorListener.hpp new file mode 100644 index 0000000000..9645a16249 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/HardwareMonitorListener.hpp @@ -0,0 +1,110 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "RunListener.hpp" + +#include +#include +#include + +#include + +#include "HardwareMonitor_fwd.hpp" + +namespace Tensile +{ + namespace Client + { + namespace po = boost::program_options; + + class HardwareMonitorListener : public RunListener + { + public: + HardwareMonitorListener(po::variables_map const& args); + + virtual bool needMoreBenchmarkRuns() const override + { + return false; + }; + virtual void preBenchmarkRun() override{}; + virtual void postBenchmarkRun() override{}; + virtual void preProblem(ContractionProblem const& problem) override{}; + virtual void postProblem() override{}; + virtual void preSolution(ContractionSolution const& solution) override{}; + virtual void postSolution() override{}; + virtual bool needMoreRunsInSolution() const override + { + return false; + }; + + virtual size_t numWarmupRuns() override + { + return 0; + }; + virtual void setNumWarmupRuns(size_t count) override{}; + virtual void preWarmup() override{}; + virtual void postWarmup() override{}; + virtual void validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override{}; + + virtual size_t numSyncs() override + { + return 0; + }; + virtual void setNumSyncs(size_t count) override{}; + virtual void preSyncs() override{}; + virtual void postSyncs() override{}; + + virtual size_t numEnqueuesPerSync() override + { + return 0; + }; + virtual void setNumEnqueuesPerSync(size_t count) override{}; + virtual void preEnqueues() override; + virtual void postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + virtual void validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + + virtual void finalizeReport() override{}; + + virtual int error() const override + { + return 0; + }; + + private: + bool m_active; + bool m_useGPUTimer; + + std::shared_ptr m_monitor; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/HardwareMonitor_fwd.hpp b/tensilelite/Tensile/Source/client/include/HardwareMonitor_fwd.hpp new file mode 100644 index 0000000000..5374b4930a --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/HardwareMonitor_fwd.hpp @@ -0,0 +1,35 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +namespace Tensile +{ + namespace Client + { + class HardwareMonitor; + } +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/LibraryUpdateReporter.hpp b/tensilelite/Tensile/Source/client/include/LibraryUpdateReporter.hpp new file mode 100644 index 0000000000..efad7c4206 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/LibraryUpdateReporter.hpp @@ -0,0 +1,89 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "CSVStackFile.hpp" +#include "ResultReporter.hpp" + +#include + +#include + +namespace Tensile +{ + namespace Client + { + namespace po = boost::program_options; + + class LibraryUpdateReporter : public ResultReporter + { + public: + static std::shared_ptr Default(po::variables_map const& args); + + static std::shared_ptr FromFilename(std::string const& filename, + bool addComment); + + /// This one will not close the stream. Useful for writing to cout. + LibraryUpdateReporter(std::ostream& stream, bool addComment); + /// This one has shared ownership of the stream. + LibraryUpdateReporter(std::shared_ptr stream, bool addComment); + + virtual void reportValue_string(std::string const& key, + std::string const& value) override; + virtual void reportValue_uint(std::string const& key, uint64_t value) override; + virtual void reportValue_int(std::string const& key, int64_t value) override; + virtual void reportValue_double(std::string const& key, double value) override; + virtual void reportValue_sizes(std::string const& key, + std::vector const& value) override; + + virtual void postProblem() override; + virtual void postSolution() override; + + void finalizeReport() override; + + private: + template + void reportValue(std::string const& key, T const& value); + + std::ostream& m_stream; + std::shared_ptr m_ownedStream; + + std::vector m_problemSizes; + + bool m_addComment = false; + + int64_t m_curSolutionIdx = -1; + std::string m_curSolutionName; + double m_curSolutionSpeed = -1.0; + bool m_curSolutionPassed = false; + + int64_t m_fastestSolutionIdx = -1; + std::string m_fastestSolutionName; + double m_fastestSolutionSpeed = -1.0; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/LogReporter.hpp b/tensilelite/Tensile/Source/client/include/LogReporter.hpp new file mode 100644 index 0000000000..c7dd6847f7 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/LogReporter.hpp @@ -0,0 +1,329 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "CSVStackFile.hpp" +#include "ResultReporter.hpp" + +#include +#include +#include + +#include +#include + +namespace po = boost::program_options; + +namespace Tensile +{ + namespace Client + { + class LogReporter : public ResultReporter + { + public: + LogReporter(LogLevel level, + std::initializer_list keys, + std::ostream& stream, + bool dumpTensors) + : m_level(level) + , m_stream(stream) + , m_csvOutput(stream, ",") + , m_dumpTensors(dumpTensors) + { + for(auto const& key : keys) + m_csvOutput.setHeaderForKey(key, key); + } + + LogReporter(LogLevel level, + std::initializer_list keys, + std::ostream& stream, + bool dumpTensors) + : m_level(level) + , m_stream(stream) + , m_csvOutput(stream, ",") + , m_dumpTensors(dumpTensors) + { + for(auto const& key : keys) + m_csvOutput.setHeaderForKey(key, key); + } + + LogReporter(LogLevel level, + std::initializer_list keys, + std::shared_ptr stream, + bool dumpTensors) + : m_level(level) + , m_stream(*stream) + , m_ownedStream(stream) + , m_csvOutput(stream, ",") + , m_dumpTensors(dumpTensors) + { + for(auto const& key : keys) + m_csvOutput.setHeaderForKey(key, key); + } + + template + static std::shared_ptr Default(po::variables_map const& args, + Stream& stream, + LogLevel level = LogLevel::Count) + { + bool dumpTensors = args["dump-tensors"].as(); + using namespace ResultKey; + if(level == LogLevel::Count) + level = args["log-level"].as(); + std::cout << "Log level: " << level << std::endl; + + PerformanceMetric metric = args["performance-metric"].as(); + // Default to 'DeviceEfficiency' benchmarking if CUEfficiency not specified + const std::string perfUnit + = (metric == PerformanceMetric::CUEfficiency ? SpeedGFlopsPerCu : SpeedGFlops); + + return std::shared_ptr(new LogReporter(level, + {BenchmarkRunNumber, + ProblemProgress, + SolutionProgress, + OperationIdentifier, + ProblemSizes, + ActivationType, + SolutionName, + Validation, + TimeUS, + perfUnit, + Empty, + TotalGranularity, + TilesPerCu, + NumCus, + Tile0Granularity, + Tile1Granularity, + CuGranularity, + WaveGranularity, + MemReadBytes, + MemWriteBytes, + TempEdge, + ClockRateSys, + ClockRateSOC, + ClockRateMem, + FanSpeedRPMs, + HardwareSampleCount, + EnqueueTime}, + stream, + dumpTensors)); + } + + static std::shared_ptr Default(po::variables_map const& args) + { + return Default(args, std::cout); + } + + virtual void reportValue_string(std::string const& key, + std::string const& value) override + { + if(key == ResultKey::Validation) + acceptValidation(value); + + m_csvOutput.setValueForKey(key, value); + } + + virtual void reportValue_uint(std::string const& key, uint64_t value) override + { + m_csvOutput.setValueForKey(key, value); + } + + virtual void reportValue_int(std::string const& key, int64_t value) override + { + m_csvOutput.setValueForKey(key, value); + } + + virtual void reportValue_double(std::string const& key, double value) override + { + m_csvOutput.setValueForKey(key, value); + } + + virtual void reportValue_sizes(std::string const& key, + std::vector const& value) override + { + std::ostringstream msg; + msg << "("; + streamJoin(msg, value, ","); + msg << ")"; + reportValue_string(key, msg.str()); + } + + void acceptValidation(std::string const& value) + { + if(value == "PASSED" || value == "NO_CHECK") + m_rowLevel = LogLevel::Normal; + else if(value == "FAILED" || value == "FAILED_CONV") + m_rowLevel = LogLevel::Error; + else if(value == "WRONG_HARDWARE") + m_rowLevel = LogLevel::Terse; + else if(value == "DID_NOT_SATISFY_ASSERTS") + m_rowLevel = LogLevel::Terse; + else if(value == "INVALID") + m_rowLevel = LogLevel::Error; + } + + virtual bool logAtLevel(LogLevel level) override + { + return level <= m_level; + } + + virtual void logMessage(LogLevel level, std::string const& message) override + { + if(logAtLevel(level)) + { + m_stream << message; + m_stream.flush(); + } + } + + template + void logTensorTyped(LogLevel level, + std::string const& name, + T const* data, + TensorDescriptor const& tensor, + T const* ptrVal) + { + if(logAtLevel(level)) + { + if(m_dumpTensors) + { + std::string fname = concatenate("tensor_", name, ".bin"); + std::ofstream ofile(fname.c_str()); + ofile.write(reinterpret_cast(data), + tensor.totalAllocatedBytes()); + + m_stream << "Dumped tensor to file " << fname << std::endl; + } + else + { + m_stream << name << ": " << tensor << std::endl; + WriteTensor(m_stream, data, tensor, ptrVal); + } + } + } + + virtual void logTensor(LogLevel level, + std::string const& name, + void const* data, + TensorDescriptor const& tensor, + void const* ptrVal) override + { + if(logAtLevel(level)) + { + if(tensor.dataType() == DataType::Float) + logTensorTyped(level, + name, + reinterpret_cast(data), + tensor, + reinterpret_cast(ptrVal)); + else if(tensor.dataType() == DataType::Half) + logTensorTyped(level, + name, + reinterpret_cast(data), + tensor, + reinterpret_cast(ptrVal)); + else if(tensor.dataType() == DataType::ComplexFloat) + logTensorTyped(level, + name, + reinterpret_cast const*>(data), + tensor, + reinterpret_cast const*>(ptrVal)); + else if(tensor.dataType() == DataType::Double) + logTensorTyped(level, + name, + reinterpret_cast(data), + tensor, + reinterpret_cast(ptrVal)); + else if(tensor.dataType() == DataType::Int32) + logTensorTyped(level, + name, + reinterpret_cast(data), + tensor, + reinterpret_cast(ptrVal)); + else if(tensor.dataType() == DataType::Int8) + logTensorTyped(level, + name, + reinterpret_cast(data), + tensor, + reinterpret_cast(ptrVal)); + else if(tensor.dataType() == DataType::BFloat16) + logTensorTyped(level, + name, + reinterpret_cast(data), + tensor, + reinterpret_cast(ptrVal)); + else + throw std::runtime_error( + concatenate("Can't log tensor of type ", tensor.dataType())); + } + } + + /// RunListener interface functions + + virtual void setReporter(std::shared_ptr reporter) override {} + + virtual void preProblem(ContractionProblem const& problem) override + { + m_csvOutput.push(); + } + + virtual void preSolution(ContractionSolution const& solution) override + { + m_csvOutput.push(); + m_rowLevel = LogLevel::Normal; + } + + virtual void postSolution() override + { + if(m_rowLevel <= m_level) + m_csvOutput.writeCurrentRow(); + m_csvOutput.pop(); + } + + virtual void postProblem() override + { + m_csvOutput.pop(); + } + + virtual void finalizeReport() override {} + + private: + LogLevel m_level; + + std::ostream& m_stream; + std::shared_ptr m_ownedStream; + + bool m_firstRun = true; + bool m_inSolution = false; + bool m_dumpTensors = false; + + LogLevel m_rowLevel; + + CSVStackFile m_csvOutput; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/MetaResultReporter.hpp b/tensilelite/Tensile/Source/client/include/MetaResultReporter.hpp new file mode 100644 index 0000000000..cdad328b59 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/MetaResultReporter.hpp @@ -0,0 +1,264 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "RunListener.hpp" + +#include + +namespace Tensile +{ + namespace Client + { + class MetaResultReporter : public ResultReporter + { + public: + virtual void addReporter(std::shared_ptr reporter) + { + if(reporter != nullptr) + m_reporters.push_back(reporter); + } + + virtual void reportValue_string(std::string const& key, + std::string const& value) override + { + for(auto r : m_reporters) + r->reportValue_string(key, value); + } + + virtual void reportValue_uint(std::string const& key, uint64_t value) override + { + for(auto r : m_reporters) + r->reportValue_uint(key, value); + } + + virtual void reportValue_int(std::string const& key, int64_t value) override + { + for(auto r : m_reporters) + r->reportValue_int(key, value); + } + + virtual void reportValue_double(std::string const& key, double value) override + { + for(auto r : m_reporters) + r->reportValue_double(key, value); + } + + virtual void reportValue_sizes(std::string const& key, + std::vector const& value) override + { + for(auto r : m_reporters) + r->reportValue_sizes(key, value); + } + + virtual bool logAtLevel(LogLevel level) override + { + for(auto r : m_reporters) + if(r->logAtLevel(level)) + return true; + return false; + } + + virtual void logMessage(LogLevel level, std::string const& message) override + { + for(auto r : m_reporters) + r->logMessage(level, message); + } + + virtual void logTensor(LogLevel level, + std::string const& name, + void const* data, + TensorDescriptor const& tensor, + void const* ptrValue) override + { + for(auto r : m_reporters) + r->logTensor(level, name, data, tensor, ptrValue); + } + + /// RunListener interface functions + + virtual bool needMoreBenchmarkRuns() const override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + if((*iter)->needMoreBenchmarkRuns()) + return true; + + return false; + } + + virtual void preBenchmarkRun() override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->preBenchmarkRun(); + } + + virtual void postBenchmarkRun() override + { + for(auto iter = m_reporters.rbegin(); iter != m_reporters.rend(); iter++) + (*iter)->postBenchmarkRun(); + } + + virtual void preProblem(ContractionProblem const& problem) override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->preProblem(problem); + } + + virtual void postProblem() override + { + for(auto iter = m_reporters.rbegin(); iter != m_reporters.rend(); iter++) + (*iter)->postProblem(); + } + + virtual void preSolution(ContractionSolution const& solution) override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->preSolution(solution); + } + + virtual void postSolution() override + { + for(auto iter = m_reporters.rbegin(); iter != m_reporters.rend(); iter++) + (*iter)->postSolution(); + } + + virtual bool needMoreRunsInSolution() const override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + if((*iter)->needMoreRunsInSolution()) + return true; + + return false; + } + + virtual size_t numWarmupRuns() override + { + return 0; + } + virtual void setNumWarmupRuns(size_t count) override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->setNumWarmupRuns(count); + } + + virtual void preWarmup() override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->preWarmup(); + } + + virtual void postWarmup() override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->postWarmup(); + } + + virtual void validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->validateWarmups(inputs, startEvents, stopEvents); + } + + virtual size_t numSyncs() override + { + return 0; + } + + virtual void setNumSyncs(size_t count) override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->setNumSyncs(count); + } + + virtual void preSyncs() override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->preSyncs(); + } + + virtual void postSyncs() override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->postSyncs(); + } + + virtual size_t numEnqueuesPerSync() override + { + return 0; + } + + virtual void setNumEnqueuesPerSync(size_t count) override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->setNumEnqueuesPerSync(count); + } + + virtual void preEnqueues() override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->preEnqueues(); + } + virtual void postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->postEnqueues(startEvents, stopEvents); + } + + virtual void validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->validateEnqueues(inputs, startEvents, stopEvents); + } + + virtual void finalizeReport() override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + (*iter)->finalizeReport(); + } + + virtual int error() const override + { + for(auto iter = m_reporters.begin(); iter != m_reporters.end(); iter++) + { + int rv = (*iter)->error(); + if(rv) + return rv; + } + + return 0; + } + + private: + std::vector> m_reporters; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/MetaRunListener.hpp b/tensilelite/Tensile/Source/client/include/MetaRunListener.hpp new file mode 100644 index 0000000000..a7a88d2f13 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/MetaRunListener.hpp @@ -0,0 +1,93 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include "ResultReporter.hpp" +#include "RunListener.hpp" + +#include + +namespace Tensile +{ + namespace Client + { + class MetaRunListener : public RunListener + { + public: + void addListener(std::shared_ptr listener); + + virtual void setReporter(std::shared_ptr reporter) override; + + virtual bool needMoreBenchmarkRuns() const override; + virtual void preBenchmarkRun() override; + virtual void postBenchmarkRun() override; + + virtual void preProblem(ContractionProblem const& problem) override; + virtual void postProblem() override; + + virtual void preSolution(ContractionSolution const& solution) override; + virtual void postSolution() override; + virtual bool needMoreRunsInSolution() const override; + + virtual size_t numWarmupRuns() override; + virtual void setNumWarmupRuns(size_t count) override; + virtual void preWarmup() override; + virtual void postWarmup() override; + virtual void validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + + virtual size_t numSyncs() override; + virtual void setNumSyncs(size_t count) override; + virtual void preSyncs() override; + virtual void postSyncs() override; + + virtual size_t numEnqueuesPerSync() override; + virtual void setNumEnqueuesPerSync(size_t count) override; + virtual void preEnqueues() override; + virtual void postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + virtual void validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + + virtual void finalizeReport() override; + + virtual int error() const override; + + private: + bool m_firstProblem = true; + + std::vector> m_listeners; + std::vector> m_reporters; + }; + + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/PerformanceReporter.hpp b/tensilelite/Tensile/Source/client/include/PerformanceReporter.hpp new file mode 100644 index 0000000000..4945add976 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/PerformanceReporter.hpp @@ -0,0 +1,113 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include "ResultReporter.hpp" + +#include +#include +#include +#include + +#include +#include + +namespace Tensile +{ + namespace Client + { + namespace po = boost::program_options; + + class PerformanceReporter : public ResultReporter + { + public: + static std::shared_ptr Default(po::variables_map const& args); + + PerformanceReporter(int deviceIndex, + double l2ReadHits, + double l2WriteHits, + double l2ReadBwMultiplier, + double readEff); + + virtual void reportValue_int(std::string const& key, int64_t value) override; + + virtual void reportValue_uint(std::string const& key, uint64_t value) override; + + virtual void reportValue_double(std::string const& key, double value) override; + + virtual void preSolution(ContractionSolution const& solution) override; + + virtual void preProblem(ContractionProblem const& problem) override; + + virtual void postSolution() override; + + void setPerfModel(double l2ReadHits, + double l2WriteHits, + double l2ReadBwMul, + double readEff); + void setNumCUs(); + void setMemoryBusWidth(); + void setClockMhz(double value); + void setMemClockMhz(double value); + void setMemBandwidthMBps(); + + int getNumCUs(); + int getMagicNum(); + double getMemClockMhz(); + double getClockMhz(); + double getL2ReadBwMultiplier(); + double getL2ReadHits(); + double getL2WriteHits(); + double getReadEff(); + double getMemBandwidthMBps(); + + template + void reportValue_numeric(std::string const& key, T value); + virtual void reportValue_string(std::string const& key, + std::string const& value) override; + virtual void reportValue_sizes(std::string const& key, + std::vector const& value) override; + void finalizeReport() override; + + protected: + hipDeviceProp_t m_props; + double m_clockMhz = std::numeric_limits::quiet_NaN(); + double m_memClockMhz = std::numeric_limits::quiet_NaN(); + double m_gFlops = std::numeric_limits::quiet_NaN(); + int m_numCUs; + int m_memoryBusWidth; + bool m_deviceProps = false; + double m_l2ReadHits; + double m_l2WriteHits; + double m_l2ReadBwMul; + double m_readEff; + double m_memBandwidthMBps; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/ProgressListener.hpp b/tensilelite/Tensile/Source/client/include/ProgressListener.hpp new file mode 100644 index 0000000000..40bbf3ef20 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/ProgressListener.hpp @@ -0,0 +1,96 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "RunListener.hpp" + +#include "ResultReporter.hpp" + +#include + +#include + +namespace Tensile +{ + namespace Client + { + namespace po = boost::program_options; + + class ProgressListener : public RunListener + { + public: + ProgressListener(po::variables_map const& args); + + virtual bool needMoreBenchmarkRuns() const override; + + virtual void preBenchmarkRun() override; + + virtual void postBenchmarkRun() override; + + virtual void preProblem(ContractionProblem const& problem) override; + + virtual void postProblem() override; + + virtual void preSolution(ContractionSolution const& solution) override; + + virtual void postSolution() override; + + virtual bool needMoreRunsInSolution() const override; + + virtual size_t numWarmupRuns() override; + virtual void setNumWarmupRuns(size_t count) override; + virtual void preWarmup() override; + + virtual void postWarmup() override; + virtual void validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + + virtual size_t numSyncs() override; + virtual void setNumSyncs(size_t count) override; + virtual void preSyncs() override; + virtual void postSyncs() override; + + virtual size_t numEnqueuesPerSync() override; + virtual void setNumEnqueuesPerSync(size_t count) override; + virtual void preEnqueues() override; + virtual void postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + virtual void validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + + virtual void finalizeReport() override; + + virtual int error() const override; + + private: + size_t m_benchmarkRun = 0; + bool m_runOnce = false; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/Reference.hpp b/tensilelite/Tensile/Source/client/include/Reference.hpp new file mode 100644 index 0000000000..878847d636 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/Reference.hpp @@ -0,0 +1,128 @@ +/** + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include + +#include + +namespace Tensile +{ + namespace Client + { + template + inline bool AlmostEqual(T a, T b); + + template <> + inline bool AlmostEqual(Half a, Half b) + { + Half absA = (a > 0) ? a : -a; + Half absB = (b > 0) ? b : -b; + // this avoids NaN when inf is compared against inf in the alternative code + // path + if(static_cast(absA) == std::numeric_limits::infinity() + || // numeric_limits is yet to + // support _Float16 type + // properly; + static_cast(absB) + == std::numeric_limits::infinity()) // however promoting it to + // float works just as fine + { + return a == b; + } + Half absDiff = (a - b > 0) ? a - b : b - a; + return absDiff / (absA + absB + 1) < 0.01; + } + + template <> + inline bool AlmostEqual(BFloat16 a, BFloat16 b) + { + BFloat16 absA = (a > static_cast(0.0f)) ? a : static_cast(0.0f) - a; + BFloat16 absB = (b > static_cast(0.0f)) ? b : static_cast(0.0f) - b; + BFloat16 absDiff = (a - b > static_cast(0.0f)) ? a - b : b - a; + return absDiff / (absA + absB + static_cast(1.0f)) + < static_cast(0.1f); + } + + template <> + inline bool AlmostEqual(float a, float b) + { + return std::fabs(a - b) / (std::fabs(a) + std::fabs(b) + 1) + < 0.0001; // 7 digits of precision - 2 + } + + template <> + inline bool AlmostEqual(double a, double b) + { + return std::fabs(a - b) / (std::fabs(a) + std::fabs(b) + 1) + < 0.000000000001; // 15 digits of precision - 2 + } + template <> + inline bool AlmostEqual(int8_t a, int8_t b) + { + return a == b; + } + template <> + inline bool AlmostEqual(int a, int b) + { + return a == b; + } + template <> + inline bool AlmostEqual(unsigned int a, unsigned int b) + { + return a == b; + } + template <> + inline bool AlmostEqual(std::complex a, std::complex b) + { + return AlmostEqual(a.real(), b.real()) && AlmostEqual(a.imag(), b.imag()); + } + + template <> + inline bool AlmostEqual(std::complex a, std::complex b) + { + return AlmostEqual(a.real(), b.real()) && AlmostEqual(a.imag(), b.imag()); + } + + template + struct ReferenceSolution + { + static void SolveCPU(ContractionProblem const& contraction, + Inputs const& inputs, + size_t validationStride = 1); + static void SolveCPUConvolution(ConvolutionProblem const& convProblem, + ContractionProblem const& problem, + Inputs const& inputs); + }; + + void SolveCPU(ContractionProblem const& contraction, + ContractionInputs const& inputs, + size_t validationStride = 1); + void SolveCPUConvolution(ConvolutionProblem const& convProblem, + ContractionProblem const& problem, + ContractionInputs& inputs); + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/ReferenceValidator.hpp b/tensilelite/Tensile/Source/client/include/ReferenceValidator.hpp new file mode 100644 index 0000000000..f390db587c --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/ReferenceValidator.hpp @@ -0,0 +1,160 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "RunListener.hpp" + +#include + +#include +#include +#include + +#include "DataInitialization.hpp" + +#include + +namespace Tensile +{ + namespace Client + { + namespace po = boost::program_options; + + class ReferenceValidator : public RunListener + { + public: + ReferenceValidator(po::variables_map const& args, + std::shared_ptr dataInit); + + virtual bool needMoreBenchmarkRuns() const override; + virtual void preBenchmarkRun() override; + virtual void postBenchmarkRun() override; + + virtual void preProblem(ContractionProblem const& problem) override; + virtual void postProblem() override; + + virtual void preSolution(ContractionSolution const& solution) override; + virtual void postSolution() override; + + virtual bool needMoreRunsInSolution() const override; + + virtual size_t numWarmupRuns() override; + virtual void setNumWarmupRuns(size_t count) override; + virtual void preWarmup() override; + virtual void postWarmup() override; + virtual void validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override; + + virtual size_t numSyncs() override + { + return 0; + } + virtual void setNumSyncs(size_t count) override {} + virtual void preSyncs() override {} + virtual void postSyncs() override {} + + virtual size_t numEnqueuesPerSync() override + { + return 0; + } + virtual void setNumEnqueuesPerSync(size_t count) override {} + virtual void preEnqueues() override {} + virtual void postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + } + virtual void validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + } + + /** + * Helper which does dynamic_cast of inputs and m_referenceInputs + * to ManagedInputs and then calls ValidateTyped(). + */ + template + bool validateSolutionCast(std::shared_ptr inputs); + + template + bool validateTyped(TypedInputs const& reference, TypedInputs const& result); + + template + bool checkResultsTyped(ManagedInputs const& reference, + ManagedInputs const& result, + CompareValid& compareValid, + CompareInvalid& compareInvalid); + + template + void printTensorsTyped(TypedInputs const& reference, TypedInputs const& result); + + virtual void finalizeReport() override; + + virtual int error() const override; + + private: + void allocateResultBuffer(size_t bytes); + + std::shared_ptr m_dataInit; + std::shared_ptr m_referenceInputs; + + ConvolutionProblem m_convolutionProblem; + + size_t m_cpuResultBufferSize = 0; + std::shared_ptr m_cpuResultBuffer; + + ContractionProblem m_problem; + + bool m_enabled; + + int m_elementsToValidate; + bool m_printValids; + int m_printMax; + int m_validationStride; + + bool m_printTensorA; + bool m_printTensorB; + bool m_printTensorC; + bool m_printTensorD; + bool m_printTensorRef; + bool m_printAny; + + bool m_convolutionVsContraction; + + int m_numBenchmarkRuns = 0; + + bool m_validatedSolution = false; + bool m_errorInSolution = false; + bool m_error = false; + bool m_errorInConvolutionVsContraction = false; + size_t m_errorsReported = 0; + + bool validateSolution(std::shared_ptr inputs); + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/ResultComparison.hpp b/tensilelite/Tensile/Source/client/include/ResultComparison.hpp new file mode 100644 index 0000000000..f0324f24b1 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/ResultComparison.hpp @@ -0,0 +1,353 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "Reference.hpp" +#include + +namespace Tensile +{ + namespace Client + { + template + struct NullComparison + { + inline void + operator()(T referenceValue, T resultValue, size_t elemIndex, size_t elemNumber) + { + } + + template + inline void before(T value, size_t elemIndex, size_t elemCount) + { + } + + inline void inside(T value, size_t elemIndex, size_t elemCount) {} + + template + inline void after(T value, size_t elemIndex, size_t elemCount) + { + } + + void report() const {} + + bool error() const + { + return false; + } + }; + + template + class PointwiseComparison + { + public: + PointwiseComparison(bool printValids, size_t printMax, bool printReport) + : m_printValids(printValids) + , m_printMax(printMax) + , m_doPrint(printMax > 0) + , m_printReport(printReport) + { + } + + inline void + operator()(T referenceValue, T resultValue, size_t elemIndex, size_t elemNumber) + { + m_values++; + bool match = AlmostEqual(referenceValue, resultValue); + if(!match) + m_errors++; + + if(!match || m_printValids) + { + if(m_doPrint) + { + if(m_printed == 0) + { + std::cout << "Index: Device | Reference" << std::endl; + } + constexpr bool needCast = std::is_same(); + using castT = std::conditional_t; + std::cout << "[" << (m_printed) << "] " + << " elem=" << elemNumber << " idx=" << elemIndex << ": " + << static_cast(resultValue) << (match ? "==" : "!=") + << static_cast(referenceValue) << std::endl; + + m_printed++; + + if(m_printMax >= 0 && m_printed >= m_printMax) + m_doPrint = false; + } + } + } + + void report() const + { + if(0 && m_printReport) + std::cout << "Found " << m_errors << " incorrect values in " << m_values + << " total values compared." << std::endl; + } + + bool error() const + { + return m_errors != 0; + } + + private: + size_t m_errors = 0; + size_t m_values = 0; + bool m_printValids = 0; + size_t m_printMax = 0; + size_t m_printed = 0; + bool m_doPrint = false; + bool m_printReport = false; + }; + + template + struct Magnitude + { + inline static T abs(T val) + { + return std::abs(val); + } + }; + + template + struct Magnitude> + { + inline static T abs(std::complex val) + { + return std::abs(val); + } + }; + + template <> + struct Magnitude + { + inline static Half abs(Half val) + { + return static_cast(std::abs(static_cast(val))); + } + }; + + template + class RMSComparison + { + public: + RMSComparison(double threshold, bool printReport) + : m_threshold(threshold) + , m_printReport(printReport) + { + } + + inline void + operator()(T referenceValue, T resultValue, size_t elemIndex, size_t elemNumber) + { + m_values++; + + using m = Magnitude; + + m_maxReference + = std::max(m_maxReference, static_cast(m::abs(referenceValue))); + m_maxResult = std::max(m_maxResult, static_cast(m::abs(resultValue))); + auto diff = m::abs(referenceValue - resultValue); + m_squareDifference += static_cast(diff * diff); + } + + inline void report() const + { + if(m_printReport) + { + std::cout << "Max reference value: " << m_maxReference + << ", max result value: " << m_maxResult << " (" << m_values + << " values)" << std::endl; + std::cout << "RMS Error: " << errorValue() << " (threshold: " << m_threshold + << ")" << std::endl; + } + } + + bool error() const + { + auto value = errorValue(); + return value > m_threshold; + } + + double errorValue() const + { + double maxMagnitude + = std::max({m_maxReference, m_maxResult, std::numeric_limits::min()}); + double denom = std::sqrt(static_cast(m_values)) * maxMagnitude; + return std::sqrt(m_squareDifference) / denom; + } + + private: + size_t m_values = 0; + bool m_printReport = false; + + double m_maxReference = 0; + double m_maxResult = 0; + double m_squareDifference = 0; + double m_threshold = 1e-7; + }; + + template + class InvalidComparison + { + public: + InvalidComparison(size_t printMax, bool printReport) + : m_printMax(printMax) + , m_printReport(printReport) + , m_doPrintBefore(printMax > 0) + , m_doPrintInside(printMax > 0) + , m_doPrintAfter(printMax > 0) + { + } + + inline void before(T value, size_t elemIndex, size_t elemCount) + { + m_checkedBefore++; + + if(!DataInitialization::isBadOutput(value)) + { + m_errorsBefore++; + + if(m_doPrintBefore) + { + if(m_printedBefore == 0) + { + std::cout << "Value written before output buffer:" << std::endl; + m_printedBefore++; + } + + std::cout << "Index " << elemIndex << " / " << elemCount << ": found " + << value << " instead of " + << DataInitialization::getValue() + << std::endl; + + if(m_printedBefore >= m_printMax) + m_doPrintBefore = false; + } + } + } + + inline void inside(T value, size_t elemIndex, size_t elemCount) + { + m_checkedInside++; + + if(!DataInitialization::isBadOutput(value)) + { + m_errorsInside++; + + if(m_doPrintInside) + { + if(m_printedInside == 0) + { + std::cout << "Value written inside output buffer, ouside tensor:" + << std::endl; + m_printedInside++; + } + + std::cout << "Index " << elemIndex << " / " << elemCount << ": found " + << value << " instead of " + << DataInitialization::getValue() + << std::endl; + + if(m_printedInside >= m_printMax) + m_doPrintInside = false; + } + } + } + + inline void after(T value, size_t elemIndex, size_t elemCount) + { + m_checkedAfter++; + + if(!DataInitialization::isBadOutput(value)) + { + m_errorsAfter++; + + if(m_doPrintAfter) + { + if(m_printedAfter == 0) + { + std::cout << "Value written after output buffer:" << std::endl; + m_printedAfter++; + } + + std::cout << "Index " << elemIndex << " / " << elemCount << ": found " + << value << " instead of " + << DataInitialization::getValue() + << std::endl; + + if(m_printedAfter >= m_printMax) + m_doPrintAfter = false; + } + } + } + + void report() const + { + if(m_printReport + && (m_checkedBefore > 0 || m_checkedInside > 0 || m_checkedAfter > 0)) + { + std::cout << "BOUNDS CHECK:" << std::endl; + std::cout << "Before buffer: found " << m_errorsBefore << " errors in " + << m_checkedBefore << " values checked." << std::endl; + std::cout << "Inside buffer: found " << m_errorsInside << " errors in " + << m_checkedInside << " values checked." << std::endl; + std::cout << "After buffer: found " << m_errorsAfter << " errors in " + << m_checkedAfter << " values checked." << std::endl; + } + } + + bool error() const + { + return m_errorsBefore != 0 || m_errorsInside != 0 || m_errorsAfter != 0; + } + + private: + size_t m_printMax = 0; + bool m_printReport = false; + + size_t m_checkedBefore = 0; + size_t m_checkedInside = 0; + size_t m_checkedAfter = 0; + + size_t m_errorsBefore = 0; + size_t m_errorsInside = 0; + size_t m_errorsAfter = 0; + + size_t m_printedBefore = 0; + size_t m_printedInside = 0; + size_t m_printedAfter = 0; + + bool m_doPrintBefore = false; + bool m_doPrintInside = false; + bool m_doPrintAfter = false; + }; + + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/ResultFileReporter.hpp b/tensilelite/Tensile/Source/client/include/ResultFileReporter.hpp new file mode 100644 index 0000000000..ff188b6df6 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/ResultFileReporter.hpp @@ -0,0 +1,87 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "CSVStackFile.hpp" +#include "ResultReporter.hpp" + +#include + +#include + +namespace Tensile +{ + namespace Client + { + namespace po = boost::program_options; + + class ResultFileReporter : public ResultReporter + { + public: + static std::shared_ptr Default(po::variables_map const& args); + + ResultFileReporter(std::string const& filename, + bool exportExtraCols, + bool mergeSameProblems, + PerformanceMetric performanceMetric); + + virtual void reportValue_string(std::string const& key, + std::string const& value) override; + virtual void reportValue_uint(std::string const& key, uint64_t value) override; + virtual void reportValue_int(std::string const& key, int64_t value) override; + virtual void reportValue_double(std::string const& key, double value) override; + virtual void reportValue_sizes(std::string const& key, + std::vector const& value) override; + + virtual void postProblem() override; + virtual void postSolution() override; + + void finalizeReport() override; + + private: + template + void reportValue(std::string const& key, T const& value); + void mergeRow(std::unordered_map& newRow); + + CSVStackFile m_output; + std::string m_solutionName; + bool m_invalidSolution = false; + bool m_extraCol; + bool m_mergeSameProblems; + PerformanceMetric m_performanceMetric; + // for extra columns + std::string m_winnerSolution; + int64_t m_currSolutionIdx = -1; + int64_t m_winnerSolutionIdx = -1; + double m_fastestGflops = -1.0; + double m_fasterTimeUS = -1.0; + // for merge rows + int64_t m_currProbID = -1; + std::map> m_probMap; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/ResultReporter.hpp b/tensilelite/Tensile/Source/client/include/ResultReporter.hpp new file mode 100644 index 0000000000..3becf28887 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/ResultReporter.hpp @@ -0,0 +1,272 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "RunListener.hpp" + +#include +#include + +namespace Tensile +{ + namespace Client + { + enum class LogLevel + { + Error = 0, + Terse, + Normal, + Verbose, + Debug, + Count + }; + + std::string ToString(LogLevel level); + std::ostream& operator<<(std::ostream& stream, LogLevel level); + std::istream& operator>>(std::istream& stream, LogLevel& level); + + namespace ResultKey + { + const std::string BenchmarkRunNumber = "run"; + + // Problem definition + const std::string ProblemIndex = "problem-index"; + const std::string ProblemCount = "problem-count"; + const std::string ProblemProgress = "problem-progress"; + const std::string OperationIdentifier = "operation"; + + const std::string ASizes = "a-sizes"; + const std::string BSizes = "b-sizes"; + const std::string CSizes = "c-sizes"; + const std::string DSizes = "d-sizes"; + + const std::string AStrides = "a-strides"; + const std::string BStrides = "b-strides"; + const std::string CStrides = "c-strides"; + const std::string DStrides = "d-strides"; + + const std::string LDA = "lda"; + const std::string LDB = "ldb"; + const std::string LDC = "ldc"; + const std::string LDD = "ldd"; + + const std::string TotalFlops = "total-flops"; + const std::string ProblemSizes = "problem-sizes"; + + const std::string ActivationType = "activation-type"; + + // Solution information + const std::string SolutionName = "solution"; + const std::string SolutionIndex = "solution-index"; + const std::string SolutionLibraryIndex = "solution-library-index"; + const std::string SolutionProgress = "solution-progress"; + const std::string SolutionWinnerIdx = "solution-winner-idx"; + const std::string SolutionWinner = "solution-winner"; + + // Performance-related + const std::string Validation = "validation"; + const std::string TimeUS = "time-us"; + const std::string SpeedGFlops = "gflops"; + const std::string SpeedGFlopsPerCu = "gflops-per-cu"; + const std::string EnqueueTime = "enqueue-time"; + const std::string FastestGFlops = "fastest-gflops"; + + // Performance estimation and granularity + const std::string Tile0Granularity = "tile0-gran"; + const std::string Tile1Granularity = "tile1-gran"; + const std::string CuGranularity = "cu-gran"; + const std::string WaveGranularity = "wave-gran"; + const std::string TotalGranularity = "total-gran"; + const std::string TilesPerCu = "tiles-per-cu"; + + const std::string MemReadBytes = "mem-read-bytes"; + const std::string MemWriteBytes = "mem-write-bytes"; + const std::string MemGlobalReads = "mem-global-reads"; + const std::string MemGlobalWrites = "mem-global-writes"; + + const std::string L2ReadHits = "perf-l2-read-hits"; + const std::string L2WriteHits = "perf-l2-write-hits"; + const std::string L2ReadBwMul = "perf-l2-read-bw-mul"; + const std::string L2BandwidthMBps = "perf-l2-bandwidth-mbps"; + + const std::string Empty = "empty"; + const std::string Efficiency = "efficiency"; + const std::string NumCus = "num-cus"; + const std::string PeakGFlops = "peak-gflops"; + + // Hardware monitoring + const std::string TempEdge = "temp-edge"; + const std::string ClockRateSys = "clock-sys"; // GPU clock in Mhz + const std::string ClockRateSOC = "clock-soc"; // Soc clock in Mhz + const std::string ClockRateMem = "clock-mem"; // Mem clock in Mhz + const std::string DeviceIndex = "device-idx"; + const std::string FanSpeedRPMs = "fan-rpm"; + const std::string HardwareSampleCount = "hardware-samples"; + }; // namespace ResultKey + + class ResultReporter : public RunListener + { + public: + /** + * Reports the value for a key, related to the current state of the run. + */ + void report(std::string const& key, std::string const& value) + { + reportValue_string(key, value); + } + + void report(std::string const& key, uint64_t value) + { + reportValue_uint(key, value); + } + + void report(std::string const& key, int value) + { + reportValue_int(key, value); + } + + void report(std::string const& key, int64_t value) + { + reportValue_int(key, value); + } + + void report(std::string const& key, double value) + { + reportValue_double(key, value); + } + + void report(std::string const& key, std::vector const& value) + { + reportValue_sizes(key, value); + } + + virtual void reportValue_string(std::string const& key, std::string const& value) = 0; + virtual void reportValue_uint(std::string const& key, uint64_t value) = 0; + virtual void reportValue_int(std::string const& key, int64_t value) = 0; + virtual void reportValue_double(std::string const& key, double value) = 0; + virtual void reportValue_sizes(std::string const& key, std::vector const& value) + = 0; + + virtual bool logAtLevel(LogLevel level) + { + return false; + }; + + /** + * Records an informative message. This may or may not actually get printed + * anywhere depending on settings. + */ + template + void log(LogLevel level, T const& object) + { + if(logAtLevel(level)) + { + std::ostringstream msg; + msg << object; + logMessage(level, msg.str()); + } + } + + virtual void logMessage(LogLevel level, std::string const& message) {} + virtual void logTensor(LogLevel level, + std::string const& name, + void const* data, + TensorDescriptor const& tensor, + void const* ptrVal) + { + } + + /// RunListener interface functions + + virtual void setReporter(std::shared_ptr reporter) override {} + + virtual bool needMoreBenchmarkRuns() const override + { + return false; + } + virtual void preBenchmarkRun() override {} + virtual void postBenchmarkRun() override {} + + virtual void preProblem(ContractionProblem const& problem) override {} + virtual void postProblem() override {} + + virtual void preSolution(ContractionSolution const& solution) override {} + virtual void postSolution() override {} + + virtual bool needMoreRunsInSolution() const override + { + return false; + } + + virtual size_t numWarmupRuns() override + { + return 0; + } + virtual void setNumWarmupRuns(size_t count) override {} + virtual void preWarmup() override {} + virtual void postWarmup() override {} + virtual void validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + } + + virtual size_t numSyncs() override + { + return 0; + } + virtual void setNumSyncs(size_t count) override {} + virtual void preSyncs() override {} + virtual void postSyncs() override {} + + virtual size_t numEnqueuesPerSync() override + { + return 0; + } + virtual void setNumEnqueuesPerSync(size_t count) override {} + virtual void preEnqueues() override {} + virtual void postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + } + virtual void validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + } + + // finalizeReport() deliberately left out of here to force it to be + // implemented in subclasses. + + virtual int error() const override + { + return 0; + } + }; + + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/ResultReporter_fwd.hpp b/tensilelite/Tensile/Source/client/include/ResultReporter_fwd.hpp new file mode 100644 index 0000000000..e1fd40589b --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/ResultReporter_fwd.hpp @@ -0,0 +1,35 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +namespace Tensile +{ + namespace Client + { + class ResultReporter; + } +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/RunListener.hpp b/tensilelite/Tensile/Source/client/include/RunListener.hpp new file mode 100644 index 0000000000..d1024e831c --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/RunListener.hpp @@ -0,0 +1,134 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include "ResultReporter_fwd.hpp" +#include "TimingEvents.hpp" + +#include + +namespace Tensile +{ + namespace Client + { + class RunListener + { + public: + virtual ~RunListener() = default; + + virtual void setReporter(std::shared_ptr reporter) + { + m_reporter = reporter; + } + + /********************************** + * Benchmark run - Outermost loop. + **********************************/ + + /// A benchmark run is a loop over all the problems. + /// Return true if we need to loop once more. + /// Note that it's not guaranteed that each listener gets this call. + virtual bool needMoreBenchmarkRuns() const = 0; + + /// Called at the beginning of each benchmark run. + virtual void preBenchmarkRun() = 0; + + /// Called at the end of each benchmark run. + virtual void postBenchmarkRun() = 0; + + /********** + * Problem + **********/ + + /// Called at the beginning of each problem. + virtual void preProblem(ContractionProblem const& problem) = 0; + + /// Called at end of each problem. + virtual void postProblem() = 0; + + /*********** + * Solution + ***********/ + + /// Called at the beginning of each solution. + virtual void preSolution(ContractionSolution const& solution) = 0; + + /// Called at end of each solution. + virtual void postSolution() = 0; + + /// Loop condition. Return true if we need another run for this solution. + /// Note that it's not guaranteed that each listener gets this call. + virtual bool needMoreRunsInSolution() const = 0; + + /*********************************************************************** + * Kernel invocation run - Innermost loop. + * + * Within a solution, we will have zero or more warmup runs followed by + * zero or more benchmark runs. + ***********************************************************************/ + + virtual size_t numWarmupRuns() = 0; + virtual void setNumWarmupRuns(size_t count) = 0; + virtual void preWarmup() = 0; + virtual void postWarmup() = 0; + virtual void validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) + = 0; + + virtual size_t numSyncs() = 0; + virtual void setNumSyncs(size_t count) = 0; + virtual void preSyncs() = 0; + virtual void postSyncs() = 0; + + virtual size_t numEnqueuesPerSync() = 0; + virtual void setNumEnqueuesPerSync(size_t count) = 0; + virtual void preEnqueues() = 0; + virtual void postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) + = 0; + virtual void validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) + = 0; + + /// Called at end of program execution. Print out a summary of the runs. + virtual void finalizeReport() = 0; + + /// Called at end of program execution. Return a non-zero value if a + /// non-fatal error was previously recorded. + virtual int error() const = 0; + + protected: + std::shared_ptr m_reporter; + }; + + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/SolutionIterator.hpp b/tensilelite/Tensile/Source/client/include/SolutionIterator.hpp new file mode 100644 index 0000000000..ed1ce017b7 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/SolutionIterator.hpp @@ -0,0 +1,185 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include + +#include +#include + +#include "RunListener.hpp" + +namespace Tensile +{ + namespace Client + { + namespace po = boost::program_options; + + /** + * Not an iterator by the traditional definition but I can't think of a better + * name + */ + class SolutionIterator : public RunListener + { + public: + static std::shared_ptr + Default(std::shared_ptr> library, + std::shared_ptr hardware, + po::variables_map const& args); + + virtual bool needMoreBenchmarkRuns() const override + { + return false; + } + virtual void preBenchmarkRun() override {} + virtual void postBenchmarkRun() override {} + + virtual bool needMoreRunsInSolution() const override + { + return false; + } + + virtual size_t numWarmupRuns() override + { + return 0; + } + virtual void setNumWarmupRuns(size_t count) override {} + virtual void preWarmup() override {} + virtual void postWarmup() override {} + virtual void validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + } + + virtual size_t numSyncs() override + { + return 0; + } + virtual void setNumSyncs(size_t count) override {} + virtual void preSyncs() override {} + virtual void postSyncs() override {} + + virtual size_t numEnqueuesPerSync() override + { + return 0; + } + virtual void setNumEnqueuesPerSync(size_t count) override {} + virtual void preEnqueues() override {} + virtual void postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + } + virtual void validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) override + { + } + + virtual void finalizeReport() override {} + + virtual int error() const override + { + return 0; + } + + virtual bool moreSolutionsInProblem() const = 0; + virtual std::shared_ptr getSolution() = 0; + virtual bool runCurrentSolution(); + + virtual void preProblem(ContractionProblem const& problem) override; + + protected: + SolutionIterator(std::shared_ptr> library, + std::shared_ptr hardware); + + virtual bool checkSolution(ContractionSolution const& solution); + + std::shared_ptr> m_library; + std::shared_ptr m_hardware; + ContractionProblem m_problem; + }; + + class AllSolutionsIterator : public SolutionIterator + { + public: + using RunCriteria = std::vector>; + + static RunCriteria + CreateCriteria(std::shared_ptr> library, + std::shared_ptr hardware, + po::variables_map const& args); + + AllSolutionsIterator(std::shared_ptr> library, + std::shared_ptr hardware, + int firstSolutionIdx, + int numSolutions, + RunCriteria runCriteria = RunCriteria()); + + virtual void preProblem(ContractionProblem const& problem) override; + virtual void postProblem() override; + + virtual void preSolution(ContractionSolution const& solution) override; + virtual void postSolution() override; + + virtual bool moreSolutionsInProblem() const override; + virtual std::shared_ptr getSolution() override; + virtual bool runCurrentSolution() override; + + private: + int m_firstSolutionIdx; + int m_lastSolutionIdx; + + int m_currentSolutionIdx; + + RunCriteria m_runCriteria; + }; + + class BestSolutionIterator : public SolutionIterator + { + public: + BestSolutionIterator(std::shared_ptr> library, + std::shared_ptr hardware); + + virtual void preProblem(ContractionProblem const& problem) override; + virtual void postProblem() override; + + virtual void preSolution(ContractionSolution const& solution) override; + virtual void postSolution() override; + + virtual bool moreSolutionsInProblem() const override; + virtual std::shared_ptr getSolution() override; + + private: + std::shared_ptr m_currentSolution; + bool m_usedCurrentSolution = false; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/include/TimingEvents.hpp b/tensilelite/Tensile/Source/client/include/TimingEvents.hpp new file mode 100644 index 0000000000..4b88dc73c9 --- /dev/null +++ b/tensilelite/Tensile/Source/client/include/TimingEvents.hpp @@ -0,0 +1,53 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include +#include + +namespace Tensile +{ + namespace Client + { + class TimingEvents + { + public: + TimingEvents(size_t numInvocations, size_t numKernels); + ~TimingEvents(); + + std::vector const& operator[](size_t index) const; + + std::vector> const& operator*() const; + std::vector> const* operator->() const; + + private: + std::vector> m_events; + }; + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/main.cpp b/tensilelite/Tensile/Source/client/main.cpp new file mode 100644 index 0000000000..6f38b8d13a --- /dev/null +++ b/tensilelite/Tensile/Source/client/main.cpp @@ -0,0 +1,701 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "BenchmarkTimer.hpp" +#include "ClientProblemFactory.hpp" +#include "DataInitialization.hpp" +#include "HardwareMonitorListener.hpp" +#include "MetaRunListener.hpp" +#include "ProgressListener.hpp" +#include "ReferenceValidator.hpp" +#include "SolutionIterator.hpp" +#include "TimingEvents.hpp" + +#include "LibraryUpdateReporter.hpp" +#include "LogReporter.hpp" +#include "MetaResultReporter.hpp" +#include "PerformanceReporter.hpp" +#include "ResultFileReporter.hpp" +#include "ResultReporter.hpp" + +#include +#include +#include + +#include + +namespace po = boost::program_options; + +namespace Tensile +{ + namespace Client + { + + template + po::typed_value* value_default(std::string const& desc) + { + return po::value()->default_value(T(), desc); + } + + template + po::typed_value* value_default() + { + return po::value()->default_value(T()); + } + + template + po::typed_value>* vector_default_empty() + { + return value_default>("[]"); + } + + po::options_description all_options() + { + po::options_description options("Tensile client options"); + + // clang-format off + options.add_options() + ("help,h", "Show help message.") + + ("config-file", vector_default_empty(), "INI config file(s) to read.") + + ("library-file,l", po::value(), "Load a (YAML) solution library. If not specified, we will use " + "the embedded library, if available.") + ("code-object,c", vector_default_empty(), "Code object file with kernel(s). If none are " + "specified, we will use the embedded code " + "object(s) if available.") + + ("performance-metric", po::value()->default_value(PerformanceMetric::DeviceEfficiency), "Metric for benchmarking results") + + ("problem-identifier", po::value(), "Problem identifer (Einstein notation). Either " + "this or free/batch/bound must be specified.") + ("free", value_default("[]"), "Free index. Order: a,b,ca,cb,da,db") + ("batch", value_default("[]"), "Batch index. Order: a,b,c,d") + ("bound", value_default("[]"), "Bound/summation index. Order: a,b") + + ("type", po::value()->default_value(DataType::None), "Data type") + ("a-type", po::value()->default_value(DataType::None), "A data type") + ("b-type", po::value()->default_value(DataType::None), "B data type") + ("c-type", po::value()->default_value(DataType::None), "C data type") + ("d-type", po::value()->default_value(DataType::None), "D data type") + ("alpha-type", po::value()->default_value(DataType::None), "alpha data type") + ("beta-type", po::value()->default_value(DataType::None), "beta data type") + ("high-precision-accumulate", po::value()->default_value(false), "Use high-precision accumulate.") + ("strided-batched", po::value()->default_value(true), "Use strided-batched or general batched") + ("kernel-language", po::value()->default_value(KernelLanguage::Any), "Select kernel language.") + ("deterministic-mode", po::value()->default_value(false), "Enforce deterministic summation patterns" + "by not splitting U among workgroups") + ("arithmetic-unit", po::value()->default_value(ArithmeticUnit::Any), "Select arithmetic unit.") + + ("init-a", po::value()->default_value(InitMode::Random), "Initialization for A") + ("init-b", po::value()->default_value(InitMode::Random), "Initialization for B") + ("init-c", po::value()->default_value(InitMode::Random), "Initialization for C") + ("init-d", po::value()->default_value(InitMode::Zero), "Initialization for D") + ("init-alpha", po::value()->default_value(InitMode::Two), "Initialization for alpha") + ("init-beta", po::value()->default_value(InitMode::Two), "Initialization for beta") + ("init-bias", po::value()->default_value(InitMode::One), "Initialization for bias") + ("pristine-on-gpu", po::value()->default_value(true), "Keep a pristine copy of inputs on GPU for performance") + ("c-equal-d", po::value()->default_value(false), "C equals D") + ("offset-a", po::value()->default_value(0), "buffer a start offset") + ("offset-b", po::value()->default_value(0), "buffer b start offset") + ("offset-c", po::value()->default_value(0), "buffer c start offset") + ("offset-d", po::value()->default_value(0), "buffer d start offset") + ("print-valids", po::value()->default_value(false), "Print values that pass validation") + ("print-max", po::value()->default_value(-1), "Max number of values to print") + ("num-elements-to-validate", po::value()->default_value(0), "Number of elements to validate") + ("bounds-check", po::value()->default_value(BoundsCheckMode::Disable), + "1:Use sentinel values to check memory boundaries." + "2:Memory bound check by front guard page" + "3:Memory bound check by back guard page" + "4:Memory bound check by both side guard page") + + ("print-tensor-a", po::value()->default_value(false), "Print tensor A.") + ("print-tensor-b", po::value()->default_value(false), "Print tensor B.") + ("print-tensor-c", po::value()->default_value(false), "Print tensor C.") + ("print-tensor-d", po::value()->default_value(false), "Print tensor D.") + ("print-tensor-ref", po::value()->default_value(false), "Print reference tensor D.") + + ("dump-tensors", po::value()->default_value(false), "Binary dump tensors instead of printing.") + + ("convolution-identifier", po::value(), "Convolution problem identifer: ConvolutionType_ActFormat_FilterFormat_Filter_Stride_Dilation_Groups. Example: ConvolutionBackwardWeights_NCHW_filter:3x3_stride:1x1_dilation:1x1_groups:1. Batch count, spacial dimensions (H,W,D), Cin and Cout filters are determined by the problem dimensions.") + ("convolution-vs-contraction", po::value()->default_value(false), "Compare reference convolution against contraction.") + + ("device-idx", po::value()->default_value(0), "Device index") + ("use-default-stream", po::value()->default_value(false), "Use default Hip stream to run kernels.") + ("platform-idx", po::value()->default_value(0), "OpenCL Platform Index") + + ("num-warmups", po::value()->default_value(0), "Number of warmups to run") + ("sync-after-warmups", po::value()->default_value(true), "Synchronize GPU after warmup kernel runs") + ("num-benchmarks", po::value()->default_value(1), "Number of benchmarks to run") + ("num-enqueues-per-sync", po::value()->default_value(1), "Enqueues per sync, will affect by min-flops-per-sync") + ("num-syncs-per-benchmark", po::value()->default_value(1), "Syncs per benchmark") + ("min-flops-per-sync", po::value()->default_value(0), "Minimum number of flops per sync to increase stability for small problems.") + ("use-gpu-timer", po::value()->default_value(true), "Use GPU timer") + ("sleep-percent", po::value()->default_value(0), "Sleep percentage") + ("hardware-monitor", po::value()->default_value(true), "Use hardware monitor.") + + ("perf-l2-read-hits", po::value()->default_value(0.0), "L2 read hits") + ("perf-l2-write-hits", po::value()->default_value(0.5), "L2 write hits") + ("perf-l2-read-bw-mul", po::value()->default_value(2.0), "L2 read bandwidth multiplier") + ("perf-read-efficiency", po::value()->default_value(0.85), "Read efficiency") + ("perf-ops-per-cycle", po::value()->default_value(64), "Ops per cycle") + ("csv-export-extra-cols", po::value()->default_value(false), "CSV exports winner information") + ("csv-merge-same-problems", po::value()->default_value(false), "CSV merge rows of same problem id") + + ("problem-size,p", vector_default_empty(), "Specify a problem size. Comma-separated list of " + "sizes, in the order of the Einstein notation.") + + ("a-strides", vector_default_empty(), "Unspecified means default stride " + "(prev_dim_stride*prev_dim_size)" + "specifying once applies to all problem sizes, " + "otherwise specify once per problem size.") + + ("b-strides", vector_default_empty(), "Unspecified means default stride " + "(prev_dim_stride*prev_dim_size)" + "specifying once applies to all problem sizes, " + "otherwise specify once per problem size.") + + ("c-strides", vector_default_empty(), "Unspecified means default stride " + "(prev_dim_stride*prev_dim_size)" + "specifying once applies to all problem sizes, " + "otherwise specify once per problem size.") + + ("d-strides", vector_default_empty(), "Unspecified means default stride " + "(prev_dim_stride*prev_dim_size)" + "specifying once applies to all problem sizes, " + "otherwise specify once per problem size.") + + ("convolution-problem", vector_default_empty(), "Specify a Convolution problem size. Comma-separated list of sizes:" + "Spatial(w,h,d),Filter(x,y,z),Stride(v,u,#)," + "Dilation(j,l,^),Pad start(q,p,$),Pad end(q_,p_,$_)") + + ("a-zero-pads", vector_default_empty(), "Comma-separated tuple(s) of anchor dim," + "summation dim, leading pad, trailing pad." + "Each tuple must be separated with a semi-colon.") + + ("b-zero-pads", vector_default_empty(), "Comma-separated tuple(s) of anchor dim," + "summation dim, leading pad, trailing pad." + "Each tuple must be separated with a semi-colon.") + + ("a-ops", vector_default_empty(), "Operations applied to A.") + ("b-ops", vector_default_empty(), "Operations applied to B.") + ("c-ops", vector_default_empty(), "Operations applied to C.") + ("d-ops", vector_default_empty(), "Operations applied to D.") + + ("problem-start-idx", po::value()->default_value(0), "First problem to run") + ("num-problems", po::value()->default_value(-1), "Number of problems to run") + + ("solution-start-idx", po::value()->default_value(-1), "First solution to run") + ("num-solutions", po::value()->default_value(-1), "Number of solutions to run") + ("best-solution", po::value()->default_value(false), "Best solution benchmark mode") + + ("results-file", po::value()->default_value("results.csv"), "File name to write results.") + ("log-file", po::value(), "File name for output log.") + ("log-file-append", po::value()->default_value(false), "Append to log file.") + ("log-level", po::value()->default_value(LogLevel::Debug), "Log level") + + ("library-update-file", po::value()->default_value(""), "File name for writing indices " + "and speeds suitable for updating " + "an existing library logic file.") + ("library-update-comment", po::value()->default_value(false), "Include solution name as a " + "comment in library update " + "file.") + + + ("exit-on-error", po::value()->default_value(false), "Exit run early on failed kernels or other errors.") + ("selection-only", po::value()->default_value(false), "Don't run any solutions, only print kernel selections.") + ("max-workspace-size", po::value()->default_value(32*1024*1024), "Max workspace for training") + ("granularity-threshold", po::value()->default_value(0.0), "Don't run a solution if total granularity is below") + + ("activation-type", po::value()->default_value(ActivationType::None), "An activation type") + ("activation-hpa", po::value()->default_value(false), "Use the same data type as high precision accumulate.") + ("activation-additional-args",vector_default_empty(), "Activation additional floating-point number arguments.") + ("activation-enum-args", po::value>()->default_value(std::vector(1, ActivationType::None), "[]"), "Activation enum argument.") + ("use-bias", po::value()->default_value(false), "Use bias.") + ; + // clang-format on + + return options; + } + + std::shared_ptr GetHardware(po::variables_map const& args) + { + int deviceCount = 0; + HIP_CHECK_EXC(hipGetDeviceCount(&deviceCount)); + + int deviceIdx = args["device-idx"].as(); + + if(deviceIdx >= deviceCount) + throw std::runtime_error(concatenate( + "Invalid device index ", deviceIdx, " (", deviceCount, " total found.)")); + + HIP_CHECK_EXC(hipSetDevice(deviceIdx)); + + return hip::GetCurrentDevice(); + } + + hipStream_t GetStream(po::variables_map const& args) + { + if(args["use-default-stream"].as()) + return 0; + + hipStream_t stream; + HIP_CHECK_EXC(hipStreamCreate(&stream)); + return stream; + } + + std::shared_ptr> + LoadSolutionLibrary(po::variables_map const& args) + { + auto filename = args["library-file"]; + if(!filename.empty()) + { + return std::dynamic_pointer_cast>( + LoadLibraryFile(filename.as())); + } + + auto embeddedLibrary + = std::dynamic_pointer_cast>( + EmbeddedLibrary::Get()); + + if(embeddedLibrary != nullptr) + return embeddedLibrary; + + throw std::runtime_error("Client must be linked with an embedded library or " + "a library must be specified at runtime."); + } + + void LoadCodeObjects(po::variables_map const& args, hip::SolutionAdapter& adapter) + { + auto const& filenames = args["code-object"].as>(); + auto logLevel = args["log-level"].as(); + + if(filenames.empty()) + { + adapter.loadEmbeddedCodeObjects(); + } + else + { + //only trigger exception when failed to load all code objects. + bool loaded = false; + hipError_t retError = hipSuccess; + + for(auto const& filename : filenames) + { + hipError_t ret; + + if(logLevel >= LogLevel::Verbose) + std::cout << "Loading " << filename << std::endl; + ret = adapter.loadCodeObjectFile(filename); + + if(ret == hipSuccess) + loaded = true; + else + retError = ret; + } + + if(!loaded) + HIP_CHECK_EXC(retError); + } + } + + template + std::vector split_nums(std::string const& value) + { + std::vector parts; + boost::split(parts, value, boost::algorithm::is_any_of(",;")); + + std::vector rv; + rv.reserve(parts.size()); + + for(auto const& part : parts) + if(part != "") + rv.push_back(boost::lexical_cast(part)); + + return rv; + } + + template + void parse_arg_nums(po::variables_map& args, std::string const& name) + { + auto inValue = args[name].as>(); + + std::vector> outValue; + outValue.reserve(inValue.size()); + for(auto const& str : inValue) + outValue.push_back(split_nums(str)); + + boost::any v(outValue); + + args.at(name).value() = v; + } + + void parse_arg_ints(po::variables_map& args, std::string const& name) + { + parse_arg_nums(args, name); + } + + void parse_arg_double(po::variables_map& args, std::string const& name) + { + parse_arg_nums(args, name); + } + + void parse_activation_enum_args(po::variables_map& args, std::string const& name) + { + auto type = args[name].as>(); + args.at(name).value() = boost::any(type); + } + + void parse_activation_int(po::variables_map& args, std::string const& name) + { + auto type = args[name].as(); + + args.at(name).value() = boost::any(type); + } + + void fix_data_types(po::variables_map& args) + { + auto type = args["type"].as(); + + // These types use the same data type for all inputs/outputs, so we allow + // using the overarching 'type' parameter. + if(type == DataType::Float || type == DataType::Double || type == DataType::ComplexFloat + || type == DataType::ComplexDouble || type == DataType::Int32) + { + args.at("a-type").value() = boost::any(type); + args.at("b-type").value() = boost::any(type); + args.at("c-type").value() = boost::any(type); + args.at("d-type").value() = boost::any(type); + args.at("alpha-type").value() = boost::any(type); + args.at("beta-type").value() = boost::any(type); + } + } + + po::variables_map parse_args(int argc, const char* argv[]) + { + auto options = all_options(); + + po::variables_map args; + po::store(po::parse_command_line(argc, argv, options), args); + po::notify(args); + + if(args.count("help")) + { + std::cout << options << std::endl; + exit(1); + } + + if(args.count("config-file")) + { + auto configFiles = args["config-file"].as>(); + for(auto filename : configFiles) + { + std::cout << "loading config file " << filename << std::endl; + std::ifstream file(filename.c_str()); + if(file.bad()) + throw std::runtime_error(concatenate("Could not open ", filename)); + po::store(po::parse_config_file(file, options), args); + } + } + + fix_data_types(args); + + parse_arg_ints(args, "problem-size"); + parse_arg_ints(args, "a-strides"); + parse_arg_ints(args, "b-strides"); + parse_arg_ints(args, "c-strides"); + parse_arg_ints(args, "d-strides"); + parse_arg_ints(args, "a-zero-pads"); + parse_arg_ints(args, "b-zero-pads"); + + if(args["convolution-vs-contraction"].as()) + parse_arg_ints(args, "convolution-problem"); + parse_activation_int(args, "activation-type"); + parse_activation_enum_args(args, "activation-enum-args"); + parse_arg_double(args, "activation-additional-args"); + return args; + } + + size_t getMaxWorkspace(std::shared_ptr>& library, + std::shared_ptr& hardware, + po::variables_map& args, + std::vector& problems, + int firstProblemIdx, + int lastProblemIdx) + { + // get max workspace size + size_t maxWorkspaceSize = 0; + + auto solutionIterator = SolutionIterator::Default(library, hardware, args); + MetaRunListener listeners; + listeners.addListener(solutionIterator); + auto reporters = std::make_shared(); + listeners.setReporter(reporters); + + listeners.preBenchmarkRun(); + + for(int problemIdx = firstProblemIdx; problemIdx <= lastProblemIdx; problemIdx++) + { + auto& problem = problems[problemIdx]; + + problem.setWorkspaceSize(std::numeric_limits::max()); + + listeners.preProblem(problem); + + while(solutionIterator->moreSolutionsInProblem()) + { + auto solution = solutionIterator->getSolution(); + + listeners.preSolution(*solution); + + if(solutionIterator->runCurrentSolution()) + { + maxWorkspaceSize + = std::max(maxWorkspaceSize, + solution->requiredWorkspaceSize(problems[problemIdx])); + } + + listeners.postSolution(); + } + + listeners.postProblem(); + } + + listeners.postBenchmarkRun(); + + return maxWorkspaceSize; + } + + } // namespace Client +} // namespace Tensile + +int main(int argc, const char* argv[]) +{ + using namespace Tensile; + using namespace Tensile::Client; + + auto args = parse_args(argc, argv); + + ClientProblemFactory problemFactory(args); + + auto hardware = GetHardware(args); + hipStream_t stream = GetStream(args); + + auto library = LoadSolutionLibrary(args); + Tensile::hip::SolutionAdapter adapter; + LoadCodeObjects(args, adapter); + + auto problems = problemFactory.problems(); + int firstProblemIdx = args["problem-start-idx"].as(); + int numProblems = args["num-problems"].as(); + if(numProblems < 0) + numProblems = problems.size(); + int lastProblemIdx = firstProblemIdx + numProblems - 1; + + int firstSolutionIdx = args["solution-start-idx"].as(); + int numSolutions = args["num-solutions"].as(); + bool gpuTimer = args["use-gpu-timer"].as(); + bool runKernels = !args["selection-only"].as(); + bool exitOnError = args["exit-on-error"].as(); + + if(firstSolutionIdx < 0) + firstSolutionIdx = library->solutions.begin()->first; + + if(numSolutions < 0) + { + auto iter = library->solutions.end(); + iter--; + } + + size_t maxWorkspaceSizeLimit = args["max-workspace-size"].as(); + size_t maxWorkspaceSize + = getMaxWorkspace(library, hardware, args, problems, firstProblemIdx, lastProblemIdx); + maxWorkspaceSize = std::min(maxWorkspaceSize, maxWorkspaceSizeLimit); + + auto dataInit = DataInitialization::Get(args, problemFactory, maxWorkspaceSize); + + auto solutionIterator = SolutionIterator::Default(library, hardware, args); + + MetaRunListener listeners; + + listeners.addListener(dataInit); + listeners.addListener(solutionIterator); + listeners.addListener(std::make_shared(args)); + if(runKernels) + { + listeners.addListener(std::make_shared(args, dataInit)); + listeners.addListener(std::make_shared(args, *hardware)); + listeners.addListener(std::make_shared(args)); + } + + auto reporters = std::make_shared(); + reporters->addReporter(PerformanceReporter::Default(args)); + + // PerformanceReporter needs to be called before these two, or else values + // will be missing + reporters->addReporter(LogReporter::Default(args)); + reporters->addReporter(ResultFileReporter::Default(args)); + reporters->addReporter(LibraryUpdateReporter::Default(args)); + + if(args.count("log-file")) + { + std::string filename = args["log-file"].as(); + auto logFile = std::make_shared( + filename.c_str(), args["log-file-append"].as() ? std::ios::app : std::ios::out); + + reporters->addReporter(LogReporter::Default(args, logFile, LogLevel::Normal)); + } + + listeners.setReporter(reporters); + + // ReferenceValidator validator(args, dataInit); + // BenchmarkTimer timer(args); + + reporters->report(ResultKey::ProblemCount, problemFactory.problems().size()); + + while(listeners.needMoreBenchmarkRuns()) + { + listeners.preBenchmarkRun(); + + for(int problemIdx = firstProblemIdx; problemIdx <= lastProblemIdx; problemIdx++) + { + auto& problem = problems[problemIdx]; + problem.setWorkspaceSize(dataInit->workspaceSize()); + + reporters->report(ResultKey::ProblemIndex, problemIdx); + reporters->report(ResultKey::ProblemProgress, + concatenate(problemIdx, "/", lastProblemIdx)); + + // std::cout << "Problem: " << problem.operationDescription() << + // std::endl; std::cout << "a: " << problem.a() << std::endl; std::cout << + // "b: " << problem.b() << std::endl; std::cout << "c: " << problem.c() << + // std::endl; std::cout << "d: " << problem.d() << std::endl; + + listeners.preProblem(problem); + + while(solutionIterator->moreSolutionsInProblem()) + { + auto solution = solutionIterator->getSolution(); + + listeners.preSolution(*solution); + + if(solutionIterator->runCurrentSolution() && runKernels) + { + try + { + while(listeners.needMoreRunsInSolution()) + { + auto inputs = dataInit->prepareGPUInputs(problem); + + auto kernels = solution->solve(problem, *inputs, *hardware); + + size_t warmupInvocations = listeners.numWarmupRuns(); + size_t eventCount = gpuTimer ? kernels.size() : 0; + TimingEvents warmupStartEvents(warmupInvocations, eventCount); + TimingEvents warmupStopEvents(warmupInvocations, eventCount); + + for(int i = 0; i < warmupInvocations; i++) + { + listeners.preWarmup(); + if(gpuTimer) + HIP_CHECK_EXC(adapter.launchKernels(kernels, + stream, + warmupStartEvents[i], + warmupStopEvents[i])); + else + HIP_CHECK_EXC( + adapter.launchKernels(kernels, stream, nullptr, nullptr)); + listeners.postWarmup(); + } + + listeners.validateWarmups(inputs, warmupStartEvents, warmupStopEvents); + + size_t syncs = listeners.numSyncs(); + size_t enq = listeners.numEnqueuesPerSync(); + + listeners.preSyncs(); + + for(int i = 0; i < syncs; i++) + { + TimingEvents startEvents(enq, eventCount); + TimingEvents stopEvents(enq, eventCount); + + listeners.preEnqueues(); + + for(int j = 0; j < enq; j++) + { + if(gpuTimer) + HIP_CHECK_EXC(adapter.launchKernels( + kernels, stream, startEvents[j], stopEvents[j])); + else + HIP_CHECK_EXC(adapter.launchKernels( + kernels, stream, nullptr, nullptr)); + } + + listeners.postEnqueues(startEvents, stopEvents); + listeners.validateEnqueues(inputs, startEvents, stopEvents); + } + + listeners.postSyncs(); + } + } + catch(std::runtime_error const& err) + { + reporters->report(ResultKey::Validation, "INVALID"); + reporters->log(LogLevel::Error, + concatenate("Exception occurred: ", err.what(), "\n")); + } + } + + listeners.postSolution(); + + if(exitOnError && listeners.error() > 0) + { + // error range in shell is [0-255] + return std::min(listeners.error(), 255); + } + } + + listeners.postProblem(); + } + + listeners.postBenchmarkRun(); + } + + listeners.finalizeReport(); + + // error range in shell is [0-255] + return std::min(listeners.error(), 255); +} diff --git a/tensilelite/Tensile/Source/client/source/BenchmarkTimer.cpp b/tensilelite/Tensile/Source/client/source/BenchmarkTimer.cpp new file mode 100644 index 0000000000..82a94cfc7a --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/BenchmarkTimer.cpp @@ -0,0 +1,242 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "BenchmarkTimer.hpp" +#include "PerformanceReporter.hpp" +#include "ResultReporter.hpp" + +#include "Reference.hpp" + +#include + +#include +#include + +namespace Tensile +{ + namespace Client + { + static_assert(BenchmarkTimer::clock::is_steady, "Clock must be steady."); + + BenchmarkTimer::BenchmarkTimer(po::variables_map const& args, Hardware const& hardware) + : m_numWarmups(args["num-warmups"].as()) + , m_syncAfterWarmups(args["sync-after-warmups"].as()) + , m_numBenchmarks(args["num-benchmarks"].as()) + , m_numEnqueuesPerSync(args["num-enqueues-per-sync"].as()) + , m_minFlopsPerSync(args["min-flops-per-sync"].as()) + , m_numSyncsPerBenchmark(args["num-syncs-per-benchmark"].as()) + , m_hardware(hardware) + , m_numEnqueuesPerSolution(m_numEnqueuesPerSync * m_numSyncsPerBenchmark) + , m_useGPUTimer(args["use-gpu-timer"].as()) + , m_sleepPercent(args["sleep-percent"].as()) + , m_timeInSolution(0) + , m_totalGPUTime(0) + { + } + + bool BenchmarkTimer::needMoreBenchmarkRuns() const + { + return m_numBenchmarksRun < m_numBenchmarks; + } + + void BenchmarkTimer::preBenchmarkRun() {} + + void BenchmarkTimer::postBenchmarkRun() + { + m_numBenchmarksRun++; + } + + void BenchmarkTimer::preProblem(ContractionProblem const& problem) + { + m_problem = problem; + } + + void BenchmarkTimer::postProblem() {} + + void BenchmarkTimer::preSolution(ContractionSolution const& solution) + { + m_numEnqueuesInSolution = 0; + m_timeInSolution = double_millis::zero(); + + ContractionSolution::ProjectedPerformance pp + = solution.projectedPerformance(m_problem, m_hardware); + m_solution = solution; + + m_reporter->report(ResultKey::Tile0Granularity, pp.granularities.tile0Granularity); + m_reporter->report(ResultKey::Tile1Granularity, pp.granularities.tile1Granularity); + m_reporter->report(ResultKey::CuGranularity, pp.granularities.cuGranularity); + m_reporter->report(ResultKey::WaveGranularity, pp.granularities.waveGranularity); + m_reporter->report(ResultKey::TotalGranularity, pp.granularities.totalGranularity); + + m_reporter->report(ResultKey::NumCus, perf.CUs); + m_reporter->report(ResultKey::TilesPerCu, pp.granularities.tilesPerCu); + m_reporter->report(ResultKey::MemReadBytes, pp.staticModel.memReadBytes); + m_reporter->report(ResultKey::MemWriteBytes, pp.staticModel.memWriteBytesD); + } + + void BenchmarkTimer::postSolution() + { + double timePerEnqueue_us + = double_micros(m_timeInSolution).count() / m_numEnqueuesInSolution; + + ContractionSolution::ProjectedPerformance pp + = m_solution.projectedPerformance(m_problem, m_hardware); + + double gflops = m_problem.flopCount() / (timePerEnqueue_us) / 1000.0; + int tiles = pp.granularities.tilesPerCu * perf.CUs; + int usedCus = std::min(tiles, perf.CUs); + double gflopsPerCu = gflops / usedCus; + + m_reporter->report(ResultKey::TimeUS, timePerEnqueue_us); + m_reporter->report(ResultKey::SpeedGFlopsPerCu, gflopsPerCu); + m_reporter->report(ResultKey::SpeedGFlops, gflops); + + m_timeInSolution = double_millis::zero(); + m_numEnqueuesInSolution = 0; + } + + bool BenchmarkTimer::needMoreRunsInSolution() const + { + return m_numEnqueuesInSolution < m_numEnqueuesPerSolution; + } + + size_t BenchmarkTimer::numWarmupRuns() + { + return m_numWarmups; + } + + void BenchmarkTimer::setNumWarmupRuns(size_t count) + { + if(count < m_numWarmups) + throw std::runtime_error(concatenate( + "Expected at least", m_numWarmups, " warmup runs, got ", count, ".")); + } + + void BenchmarkTimer::preWarmup() {} + + void BenchmarkTimer::postWarmup() {} + + void BenchmarkTimer::validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + if(m_syncAfterWarmups && (stopEvents->size() > 0) && (stopEvents->back().size() > 0)) + HIP_CHECK_EXC(hipEventSynchronize(stopEvents->back().back())); + } + + size_t BenchmarkTimer::numSyncs() + { + return m_numSyncsPerBenchmark; + } + + void BenchmarkTimer::setNumSyncs(size_t count) + { + m_numSyncsInBenchmark = count; + } + + void BenchmarkTimer::preSyncs() {} + + void BenchmarkTimer::postSyncs() {} + + size_t BenchmarkTimer::numEnqueuesPerSync() + { + size_t enqueuesByFlops = 0; + if(m_minFlopsPerSync > 0) + { + size_t flopsInProblem = m_problem.flopCount(); + enqueuesByFlops = CeilDivide(m_minFlopsPerSync, flopsInProblem); + } + + return std::max(m_numEnqueuesPerSync, enqueuesByFlops); + } + + void BenchmarkTimer::setNumEnqueuesPerSync(size_t count) + { + m_curNumEnqueuesPerSync = count; + } + + void BenchmarkTimer::preEnqueues() + { + if(!m_useGPUTimer) + { + m_startTime = clock::now(); + } + } + + void BenchmarkTimer::postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + if(!m_useGPUTimer) + { + HIP_CHECK_EXC(hipDeviceSynchronize()); + m_endTime = clock::now(); + } + } + + void BenchmarkTimer::validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + double_millis totalTime(0.0); + + if(m_useGPUTimer) + { + HIP_CHECK_EXC(hipEventSynchronize(stopEvents->back().back())); + for(size_t i = 0; i < startEvents->size(); i++) + { + float enqTime = 0.0f; + + HIP_CHECK_EXC(hipEventElapsedTime( + &enqTime, startEvents->at(i).front(), stopEvents->at(i).back())); + + totalTime += double_millis(enqTime); + } + } + else + { + totalTime = double_millis(m_endTime - m_startTime); + } + + m_timeInSolution += totalTime; + m_totalGPUTime += totalTime; + m_numEnqueuesInSolution += startEvents->size(); + + if(m_sleepPercent > 0) + { + auto sleepTime = totalTime * (m_sleepPercent / 100.0); + + std::this_thread::sleep_for(sleepTime); + } + } + + void BenchmarkTimer::finalizeReport() {} + + int BenchmarkTimer::error() const + { + return 0; + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/CSVStackFile.cpp b/tensilelite/Tensile/Source/client/source/CSVStackFile.cpp new file mode 100644 index 0000000000..f5a11bfb9c --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/CSVStackFile.cpp @@ -0,0 +1,195 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include + +#include + +namespace Tensile +{ + namespace Client + { + CSVStackFile::CSVStackFile(std::string const& filename, std::string const& separator) + : m_stream(new std::ofstream(filename.c_str())) + , m_separator(separator) + { + } + + void null_deleter(void* ptr) {} + + CSVStackFile::CSVStackFile(std::ostream& stream, std::string const& separator) + : m_stream(&stream, null_deleter) + , m_separator(separator) + { + } + + CSVStackFile::CSVStackFile(std::shared_ptr stream, + std::string const& separator) + : m_stream(stream) + , m_separator(separator) + { + } + + CSVStackFile::~CSVStackFile() {} + + void CSVStackFile::setHeaderForKey(std::string const& key, std::string const& header) + { + if(m_headers.find(key) == m_headers.end()) + m_keyOrder.push_back(key); + + m_headers[key] = header; + } + + void CSVStackFile::push() + { + m_stack.push_back(m_currentRow); + } + + void CSVStackFile::pop() + { + m_currentRow = m_stack.back(); + m_stack.pop_back(); + } + + void CSVStackFile::writeCurrentRow() + { + if(m_firstRow && !m_headers.empty()) + writeRow(m_headers); + + m_firstRow = false; + + writeRow(m_currentRow); + + if(m_stack.empty()) + m_currentRow.clear(); + else + m_currentRow = m_stack.back(); + } + + void CSVStackFile::readCurrentRow(std::unordered_map& outMap) + { + // we still write the header to csv first, then read the data to map + if(m_firstRow && !m_headers.empty()) + writeRow(m_headers); + + m_firstRow = false; + + // only copy the fields that are in headers + for(auto const& key : m_keyOrder) + { + std::string value = ""; + + auto it = m_currentRow.find(key); + if(it != m_currentRow.end()) + value = escape(it->second); + outMap[key] = value; + } + } + + void CSVStackFile::clearCurrentRow() + { + m_currentRow.clear(); + } + + void CSVStackFile::writeRow(std::unordered_map const& row) + { + bool firstCol = true; + for(auto const& key : m_keyOrder) + { + if(!firstCol) + (*m_stream) << m_separator; + + std::string value = ""; + + auto it = row.find(key); + if(it != row.end()) + value = escape(it->second); + + (*m_stream) << value; + + firstCol = false; + } + + (*m_stream) << std::endl; + } + + std::string CSVStackFile::escape(std::string const& value) + { + // An actual quote needs more attention. + if(value.find('"') != std::string::npos) + return escapeQuote(value); + + bool needQuote = false; + + std::string badValues = ",\n\r"; + for(char c : badValues) + { + if(value.find(c) != std::string::npos) + { + needQuote = true; + break; + } + } + + if(needQuote) + return concatenate("\"", value, "\""); + + return value; + } + + std::string CSVStackFile::escapeQuote(std::string const& value) + { + std::ostringstream rv; + rv << '"'; + + for(char c : value) + { + if(c == '"') + rv << "\"\""; + else + rv << c; + } + + rv << '"'; + + return rv.str(); + } + + void CSVStackFile::setValueForKey(std::string const& key, std::string const& value) + { + m_currentRow[key] = value; + } + + void CSVStackFile::setValueForKey(std::string const& key, double const& value) + { + + std::ostringstream ss; + ss << std::setprecision(6) << value; + setValueForKey(key, ss.str()); + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/ClientProblemFactory.cpp b/tensilelite/Tensile/Source/client/source/ClientProblemFactory.cpp new file mode 100644 index 0000000000..a5cb7d28a6 --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/ClientProblemFactory.cpp @@ -0,0 +1,239 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "ClientProblemFactory.hpp" +#include "DataInitialization.hpp" + +#include + +namespace Tensile +{ + namespace Client + { + ClientProblemFactory::ClientProblemFactory(po::variables_map const& args) + : m_freeIndices(args["free"].as()) + , m_batchIndices(args["batch"].as()) + , m_boundIndices(args["bound"].as()) + , m_problemSizes(args["problem-size"].as>>()) + , m_aZeroPads(args["a-zero-pads"].as>>()) + , m_bZeroPads(args["b-zero-pads"].as>>()) + , m_aType(DataType::Float) + , m_bType(DataType::Float) + , m_cType(DataType::Float) + , m_dType(DataType::Float) + , m_alphaType(DataType::Float) + , m_betaType(DataType::Float) + , m_stridedBatched(args["strided-batched"].as()) + , m_highPrecisionAccumulate(args["high-precision-accumulate"].as()) + , m_kernelLanguage(args["kernel-language"].as()) + , m_performanceMetric(args["performance-metric"].as()) + , m_deterministicMode(args["deterministic-mode"].as()) + , m_cEqualsD(args["c-equal-d"].as()) + , m_arithmeticUnit(args["arithmetic-unit"].as()) + , m_aStrides(args["a-strides"].as>>()) + , m_bStrides(args["b-strides"].as>>()) + , m_cStrides(args["c-strides"].as>>()) + , m_dStrides(args["d-strides"].as>>()) + , m_aOps(args["a-ops"].as()) + , m_bOps(args["b-ops"].as()) + , m_cOps(args["c-ops"].as()) + , m_dOps(args["d-ops"].as()) + , m_aOffset(args["offset-a"].as()) + , m_bOffset(args["offset-b"].as()) + , m_cOffset(args["offset-c"].as()) + , m_dOffset(args["offset-d"].as()) + , m_activationType(ActivationType::None) + , m_activationHPA(false) + , m_activationEnumArg(std::vector(1, ActivationType::None)) + { + if(args.count("problem-identifier")) + ContractionProblem::IdentifierToIndices( + args["problem-identifier"].as(), + m_freeIndices, + m_batchIndices, + m_boundIndices, + m_aOps, + m_bOps, + m_cOps, + m_dOps); + + if(args.count("type")) + { + m_aType = m_bType = m_cType = m_dType = m_alphaType = m_betaType + = args["type"].as(); + } + + if(args.count("a-type")) + m_aType = args["a-type"].as(); + if(args.count("b-type")) + m_bType = args["b-type"].as(); + if(args.count("c-type")) + m_cType = args["c-type"].as(); + if(args.count("d-type")) + m_dType = args["d-type"].as(); + if(args.count("alpha-type")) + m_alphaType = args["alpha-type"].as(); + if(args.count("beta-type")) + m_betaType = args["beta-type"].as(); + + m_beta = DataInitialization::getValue(args["init-beta"].as()); + m_alpha = DataInitialization::getValue(args["init-alpha"].as()); + + if(args["convolution-vs-contraction"].as()) + m_convProblemSizes + = args["convolution-problem"].as>>(); + + if(args.count("activation-type")) + m_activationType = args["activation-type"].as(); + if(args.count("activation-hpa")) + m_activationHPA = args["activation-hpa"].as(); + if(args.count("activation-enum-args")) + m_activationEnumArg + = args["activation-enum-args"].as>(); + if(args.count("use-bias")) + m_useBias = args["use-bias"].as(); + m_problems = createProblems(); + } + + ClientProblemFactory::~ClientProblemFactory() = default; + + std::vector const& ClientProblemFactory::problems() const + { + return m_problems; + } + + std::vector ClientProblemFactory::createProblems() + { + std::vector rv; + int activationSize = std::max(1, (int)m_activationEnumArg.size()); + rv.reserve(m_problemSizes.size() * activationSize); + + std::vector aStrides, bStrides, cStrides, dStrides; + + if(m_aStrides.size() == 1) + aStrides = m_aStrides[0]; + if(m_bStrides.size() == 1) + bStrides = m_bStrides[0]; + if(m_cStrides.size() == 1) + cStrides = m_cStrides[0]; + if(m_dStrides.size() == 1) + dStrides = m_dStrides[0]; + for(int j = 0; j < activationSize; j++) + { + for(int i = 0; i < m_problemSizes.size(); i++) + { + if(m_aStrides.size() == m_problemSizes.size()) + aStrides = m_aStrides[i]; + if(m_bStrides.size() == m_problemSizes.size()) + bStrides = m_bStrides[i]; + if(m_cStrides.size() == m_problemSizes.size()) + cStrides = m_cStrides[i]; + if(m_dStrides.size() == m_problemSizes.size()) + dStrides = m_dStrides[i]; + + rv.push_back(ContractionProblem::FromIndexSizes(m_freeIndices, + m_batchIndices, + m_boundIndices, + m_problemSizes[i], + m_aType, + aStrides, + m_aOps, + m_aOffset, + m_bType, + bStrides, + m_bOps, + m_bOffset, + m_cType, + cStrides, + m_cOps, + m_cOffset, + m_dType, + dStrides, + m_dOps, + m_dOffset, + m_beta)); + + rv.back().setAlphaRestriction(toScalarValueEnum(m_alpha)); + rv.back().setCEqualsD(m_cEqualsD); + + if(i < m_aZeroPads.size()) + { + const auto& zp = m_aZeroPads[i]; + if(zp.size() % 4 != 0) + throw std::runtime_error("zero-pad must contain tuples of 4 values"); + for(int zi = 0; zi < zp.size(); zi += 4) + { + rv.back().addAZeroPad( + ContractionProblem::ZeroPad({static_cast(zp[zi + 0]), + static_cast(zp[zi + 1]), + static_cast(zp[zi + 2]), + static_cast(zp[zi + 3])})); + } + } + if(i < m_bZeroPads.size()) + { + const auto& zp = m_bZeroPads[i]; + if(zp.size() % 4 != 0) + throw std::runtime_error("zero-pad must contain tuples of 4 values"); + for(int zi = 0; zi < zp.size(); zi += 4) + { + rv.back().addBZeroPad( + ContractionProblem::ZeroPad({static_cast(zp[zi + 0]), + static_cast(zp[zi + 1]), + static_cast(zp[zi + 2]), + static_cast(zp[zi + 3])})); + } + } + rv.back().setAlphaType(m_alphaType); + rv.back().setBetaType(m_betaType); + rv.back().setStridedBatched(m_stridedBatched); + rv.back().setHighPrecisionAccumulate(m_highPrecisionAccumulate); + rv.back().setUseBias(m_useBias); + rv.back().setKernelLanguage(m_kernelLanguage); + rv.back().setPerformanceMetric(m_performanceMetric); + rv.back().setDeterministicMode(m_deterministicMode); + rv.back().setArithmeticUnit(m_arithmeticUnit); + rv.back().setFp16AltImpl(m_fp16AltImpl); + rv.back().setActivationType(m_activationType); + if(j < m_activationEnumArg.size()) + { + rv.back().setActivationEnumArg(m_activationEnumArg[j]); + } + else + { + rv.back().setActivationType(m_activationType); + } + rv.back().setActivationHPA(m_activationHPA); + + if(m_convProblemSizes.size()) + rv.back().setConvProblemSizes(m_convProblemSizes[i]); + } + } + + return rv; + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/ConvolutionProblem.cpp b/tensilelite/Tensile/Source/client/source/ConvolutionProblem.cpp new file mode 100644 index 0000000000..fb8a9c554d --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/ConvolutionProblem.cpp @@ -0,0 +1,598 @@ +/** + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +namespace Tensile +{ + const size_t ConvolutionProblem::InvalidPos = -1; + ConvolutionProblem::ActivationFormat::ActivationFormat() {} + void ConvolutionProblem::ActivationFormat::FromIdentifier(std::string identifier, + size_t formatNumSpatialDims, + size_t numSpatialDims, + std::vector* filters) + { + // summation dimensions immediately follow the spatial dim(s) + m_formatIdentifier = identifier; + m_filterPositions.clear(); + m_spatialPositions.clear(); + if(identifier == "NCHW") + { + assert(formatNumSpatialDims == 2); + m_format = TensorFormat::NCHW; + + size_t position = 0; + + if(filters) + for(int fi = 0; fi < filters->size(); fi++) + { + if((*filters)[fi] != 1) + m_filterPositions.push_back(position++); + else + m_filterPositions.push_back(InvalidPos); + } + for(auto si = 0; si < numSpatialDims; si++) + m_spatialPositions.push_back(position++); + + m_channelPosition = position++; + m_batchPosition = position++; + } + else if(identifier == "NHWC") + { + assert(formatNumSpatialDims == 2); + m_format = TensorFormat::NHWC; + + m_channelPosition = 0; + + size_t position = m_channelPosition + 1; + if(filters) + for(int fi = 0; fi < filters->size(); fi++) + { + if((*filters)[fi] != 1) + m_filterPositions.push_back(position++); + else + m_filterPositions.push_back(InvalidPos); + } + + // assume spatial dimensions are collapsed here: + std::cout << "FIXME\n"; + for(auto si = 0; si < numSpatialDims; si++) + m_spatialPositions.push_back(position++); + m_batchPosition = position++; + } + else if(identifier == "CNHW") + { + assert(formatNumSpatialDims == 2); + m_format = TensorFormat::CNHW; + + size_t position = 0; + if(filters) + for(int fi = 0; fi < filters->size(); fi++) + { + if((*filters)[fi] != 1) + m_filterPositions.push_back(position++); + else + m_filterPositions.push_back(InvalidPos); + } + // assume spatial dimensions are collapsed here: + std::cout << "FIXME\n"; + for(auto si = 0; si < numSpatialDims; si++) + m_spatialPositions.push_back(position++); + m_batchPosition = position++; + m_channelPosition = position++; + } + else + { + throw std::runtime_error(std::string("Invalid tensor format in convolution identifier:") + + identifier); + } + } + + std::string ConvolutionProblem::ActivationFormat::description() const + { + std::ostringstream rv; + rv << m_formatIdentifier << "_" + << " batchPosition=" << m_batchPosition << " channelPosition=" << m_channelPosition; + rv << " spatialPositions[]="; + for(auto i = 0; i < m_spatialPositions.size(); i++) + { + if(i != 0) + rv << ","; + rv << static_cast(m_spatialPositions[i]); + } + + rv << " filterPositions[]="; + for(auto i = 0; i < m_filterPositions.size(); i++) + { + if(i != 0) + rv << ","; + rv << static_cast(m_filterPositions[i]); + } + + return rv.str(); + } + + ConvolutionProblem::WeightFormat::WeightFormat() {} + + void ConvolutionProblem::WeightFormat::FromIdentifier(std::string identifier, + bool transposeCK, + size_t formatNumSpatialDims, + std::vector* filters) + { + m_formatIdentifier = identifier; + if(identifier == "KCYX") + m_format = TensorFormat::KCYX; + else if(identifier == "CKYX") + m_format = TensorFormat::CKYX; + else + throw std::runtime_error(std::string("Invalid weight format in convolution identifier:") + + identifier); + + m_filterPositions.clear(); + if((identifier == "KCYX" && !transposeCK) || (identifier == "CKYX" && transposeCK)) + { + + assert(formatNumSpatialDims == 2); + + size_t position = 0; + if(filters) + // Weight dims are assigned in reverse order for optimal Tensile summation + // processing + for(int fi = 0; fi < filters->size(); fi++) + { + if((*filters)[fi] != 1) + m_filterPositions.push_back(position++); + else + m_filterPositions.push_back(InvalidPos); + } + m_cinPosition = position++; + m_coutPosition = position; + } + else if((identifier == "CKYX" && !transposeCK) || (identifier == "KCYX" && transposeCK)) + { + assert(formatNumSpatialDims == 2); + size_t position = 0; + if(filters) + for(int fi = 0; fi < filters->size(); fi++) + { + if((*filters)[fi] != 1) + m_filterPositions.push_back(position++); + else + m_filterPositions.push_back(InvalidPos); + } + m_coutPosition = position; + m_cinPosition = m_coutPosition + 1; + } + else + { + throw std::runtime_error(std::string("Invalid weight format in convolution identifier:") + + identifier); + } + } + std::string ConvolutionProblem::WeightFormat::description() const + { + std::ostringstream rv; + rv << m_formatIdentifier << "_" + << " coutPosition=" << m_coutPosition << " cinPosition=" << m_cinPosition + << " filterPositions[]="; + for(auto i = 0; i < m_filterPositions.size(); i++) + { + if(i != 0) + rv << ","; + rv << static_cast(m_filterPositions[i]); + } + return rv.str(); + } + void ConvolutionProblem::LoopCounts::setupFormat(ConvolutionProblem const& convProblem) + { + std::string operationIdentifier = convProblem.operationIdentifier(); + if(operationIdentifier == "ConvolutionForward" + || operationIdentifier == "ConvolutionBackwardData") + { + m_formatA.FromIdentifier(convProblem.AIdentifier(), + convProblem.numFormatSpatialDims(), + convProblem.numSpatialDims(), + &filterCount); + m_formatB.weightsW().FromIdentifier(convProblem.BIdentifier(), + operationIdentifier == "ConvolutionBackwardData", + convProblem.numFormatSpatialDims(), + &filterCount); + m_formatD.activationW().FromIdentifier(convProblem.DIdentifier(), + convProblem.numFormatSpatialDims(), + convProblem.numSpatialDims(), + nullptr); + } + else if(operationIdentifier == "ConvolutionBackwardWeights") + { + } + else + { + throw std::runtime_error(std::string("Invalid operation identifier:") + + operationIdentifier); + } + } + + // Setup for forward or backward data + void ConvolutionProblem::LoopCounts::setupForData(ConvolutionProblem const& convProblem, + ContractionProblem const& problem) + { + size_t numSpatial = convProblem.numFormatSpatialDims(); + std::vector convProblemSizes = problem.convProblemSizes(); + std::vector::iterator it = convProblemSizes.begin(); + + //convolution problem size must be six times of numSpatial + assert(convProblemSizes.size() == 6 * numSpatial); + + spatialCount.assign(it, it + numSpatial); + filterCount.assign(it += numSpatial, it + numSpatial); + strideCount.assign(it += numSpatial, it + numSpatial); + dilationCount.assign(it += numSpatial, it + numSpatial); + padStartCount.assign(it += numSpatial, it + numSpatial); + padEndCount.assign(it += numSpatial, it + numSpatial); + + setupFormat(convProblem); + + batchCount = problem.a().sizes()[m_formatA.batchPosition()]; + cinCount = problem.a().sizes()[m_formatA.channelPosition()]; + coutCount = problem.b().sizes()[m_formatB.weights().coutPosition()]; + for(int si = 0; si < m_formatA.spatialPositions().size(); si++) + { + auto spatialPositionA = m_formatA.spatialPositions()[si]; + auto const problemSpatialSize = problem.a().sizes()[spatialPositionA]; + scount[si] = problemSpatialSize; + } + } + + std::string ConvolutionProblem::LoopCounts::description() const + { + std::ostringstream rv; + rv << " formatA = " << m_formatA.description() << std::endl; + rv << " formatB = " << m_formatB.weights().description() << std::endl; + rv << " batchCount=" << batchCount << " coutCount=" << coutCount + << " scalarCount_dhw=" << scount[2] << "x" << scount[1] << "x" << scount[0] + << " cinCount=" << cinCount << std::endl; + + rv << " spatialCount "; + for(auto i = spatialCount.begin(); i != spatialCount.end(); ++i) + rv << *i << ','; + rv << "filterCount "; + for(auto i = filterCount.begin(); i != filterCount.end(); ++i) + rv << *i << ','; + rv << "strideCount "; + for(auto i = strideCount.begin(); i != strideCount.end(); ++i) + rv << *i << ','; + rv << "dilationCount "; + for(auto i = dilationCount.begin(); i != dilationCount.end(); ++i) + rv << *i << ','; + rv << "padStartCount "; + for(auto i = padStartCount.begin(); i != padStartCount.end(); ++i) + rv << *i << ','; + rv << "padEndCount "; + for(auto i = padEndCount.begin(); i != padEndCount.end(); ++i) + rv << *i << ','; + return rv.str(); + } + + void ConvolutionProblem::FromIdentifier(std::string identifier) + { + // example identifier: + // ConvolutionForward_NCHW_KCHW_NCHW_filter:3x3x1_stride:1x1x1_dilation:1x1x1_groups:1 + std::vector parts; + boost::split(parts, identifier, boost::algorithm::is_any_of("_")); + + if(parts.size() < 4) + // id, formatA, formatB, outputTensor + throw std::runtime_error( + std::string("Invalid convolution identifier- must have at least 3 sections:") + + identifier); + + m_operationIdentifier = parts[0]; + m_AIdentifier = parts[1]; + m_BIdentifier = parts[2]; + m_DIdentifier = parts[3]; + size_t formatNumSpatialDims = parts[1].size() - 2; + m_numFormatSpatialDims = formatNumSpatialDims; + + for(auto part = parts.begin() + 4; part != parts.end(); part++) + { + std::vector flags; + boost::split(flags, *part, boost::algorithm::is_any_of(":")); + assert(flags.size() == 2); // must be key:value pair + + if(flags[0] == "spatialDims") + m_numSpatialDims = boost::lexical_cast(flags[1]); + else if(flags[0] == "indices") + { + } + else if(flags[0] == "groups") + { + m_groups = boost::lexical_cast(flags[1]); + assert(m_groups == 1); // not supported yet + } + else + throw std::runtime_error(std::string("Invalid flag in convolution identifier:") + + flags[0]); + + std::cout << flags[0] << ":::" << flags[1] << "\n"; + }; + } + + TensorDescriptor + ConvolutionProblem::setupDataActivation(ConvolutionProblem::LoopCounts const& counts, + ContractionProblem const& problem) const + { + // Mimic the expected dimension order in formatA: + std::vector activationDims; + std::vector activationStri; + int64_t batchStride; + int64_t realSpatialSize = 1; + + auto formatA = counts.formatA(); + switch(formatA.format()) + { + case ConvolutionProblem::TensorFormat::NCHW: + for(int fi = 0; fi < counts.filterCount.size(); fi++) + if(formatA.filterPositions()[fi] != ConvolutionProblem::InvalidPos) + { + activationDims.push_back(counts.filterCount[fi]); + activationStri.push_back(fi == 0 ? counts.dilationCount[fi] + : counts.dilationCount[fi] + * counts.spatialCount[fi - 1]); + } + for(int si = 0; si < formatA.spatialPositions().size(); si++) + { + activationDims.push_back(counts.scount[si]); + activationStri.push_back(si == 0 ? counts.strideCount[si] + : counts.strideCount[si] + * counts.spatialCount[si - 1]); + } + for(int si = 0; si < m_numFormatSpatialDims; si++) + realSpatialSize *= counts.spatialCount[si]; + activationDims.push_back(problem.a().sizes()[formatA.channelPosition()]); + activationStri.push_back(realSpatialSize); + activationDims.push_back(problem.a().sizes()[formatA.batchPosition()]); + batchStride = realSpatialSize * problem.a().sizes()[formatA.channelPosition()]; + activationStri.push_back(batchStride); + break; + case ConvolutionProblem::TensorFormat::NHWC: + assert(0); // need strides + activationDims.push_back(problem.a().sizes()[formatA.channelPosition()]); + for(int fi = 0; fi < counts.filterCount.size(); fi++) + if(formatA.filterPositions()[fi] != ConvolutionProblem::InvalidPos) + activationDims.push_back(counts.filterCount[fi]); + for(int si = 0; si < formatA.spatialPositions().size(); si++) + activationDims.push_back(counts.scount[si]); + activationDims.push_back(problem.a().sizes()[formatA.batchPosition()]); + case ConvolutionProblem::TensorFormat::CNHW: + assert(0); // need strides + for(int fi = 0; fi < counts.filterCount.size(); fi++) + if(formatA.filterPositions()[fi] != ConvolutionProblem::InvalidPos) + activationDims.push_back(counts.filterCount[fi]); + for(int si = 0; si < formatA.spatialPositions().size(); si++) + activationDims.push_back(counts.scount[si]); + activationDims.push_back(problem.a().sizes()[formatA.batchPosition()]); + activationDims.push_back(problem.a().sizes()[formatA.channelPosition()]); + break; + default: + throw std::runtime_error("unknown formatA"); + }; + TensorDescriptor rv(problem.a().dataType(), + activationDims.begin(), + activationDims.end(), + activationStri.begin(), + activationStri.end(), + problem.a().offset()); + return rv; + } + + TensorDescriptor + ConvolutionProblem::setupDataOutput(ConvolutionProblem::LoopCounts const& counts, + ContractionProblem const& problem) const + { + std::vector outputDims; + auto formatA = counts.formatA(); + auto formatD = counts.formatD(); + switch(formatD.activation().format()) + { + case ConvolutionProblem::TensorFormat::NCHW: + for(int si = 0; si < formatA.spatialPositions().size(); si++) + outputDims.push_back(counts.scount[si]); + outputDims.push_back(problem.d().sizes()[formatD.activation().channelPosition()]); + outputDims.push_back(problem.d().sizes()[formatD.activation().batchPosition()]); + break; + case ConvolutionProblem::TensorFormat::NHWC: + outputDims.push_back(problem.d().sizes()[formatD.activation().channelPosition()]); + for(int si = 0; si < formatA.spatialPositions().size(); si++) + outputDims.push_back(counts.scount[si]); + outputDims.push_back(problem.d().sizes()[formatD.activation().batchPosition()]); + break; + case ConvolutionProblem::TensorFormat::CNHW: + for(int si = 0; si < formatA.spatialPositions().size(); si++) + outputDims.push_back(counts.scount[si]); + outputDims.push_back(problem.d().sizes()[formatD.activation().batchPosition()]); + outputDims.push_back(problem.d().sizes()[formatD.activation().channelPosition()]); + break; + default: + throw std::runtime_error("unknown formatD"); + }; + TensorDescriptor rv( + problem.d().dataType(), outputDims.begin(), outputDims.end(), problem.d().offset()); + return rv; + } + + TensorDescriptor + ConvolutionProblem::setupForwardWeights(ConvolutionProblem::LoopCounts const& counts, + ContractionProblem const& problem) const + { + std::vector filterDims; + auto formatB = counts.formatB(); + switch(formatB.weights().format()) + { + case ConvolutionProblem::TensorFormat::KCYX: + for(int fi = 0; fi < counts.filterCount.size(); fi++) + if(formatB.weights().filterPositions()[fi] != ConvolutionProblem::InvalidPos) + filterDims.push_back(counts.filterCount[fi]); + filterDims.push_back(problem.b().sizes()[formatB.weights().cinPosition()]); + filterDims.push_back(problem.b().sizes()[formatB.weights().coutPosition()]); + break; + case ConvolutionProblem::TensorFormat::CKYX: + for(int fi = 0; fi < counts.filterCount.size(); fi++) + if(formatB.weights().filterPositions()[fi] != ConvolutionProblem::InvalidPos) + filterDims.push_back(counts.filterCount[fi]); + filterDims.push_back(problem.b().sizes()[formatB.weights().coutPosition()]); + filterDims.push_back(problem.b().sizes()[formatB.weights().cinPosition()]); + break; + default: + throw std::runtime_error("unknown formatB"); + }; + TensorDescriptor rv( + problem.b().dataType(), filterDims.begin(), filterDims.end(), problem.b().offset()); + return rv; + } + + void ConvolutionProblem::validate(const ContractionProblem& problem, + const ConvolutionProblem::LoopCounts& counts) const + { + if(1) + { + std::cout << "validate::\n"; + + std::cout << " freeAIndices: "; + for(auto i : problem.freeIndicesA()) + std::cout << i << ","; + + std::cout << "\n freeBIndices"; + for(auto i : problem.freeIndicesB()) + std::cout << i << ","; + + std::cout << "\n batchIndices: "; + for(auto i : problem.batchIndices()) + std::cout << i << ","; + + std::cout << "\n summationIndicies: "; + for(auto i : problem.boundIndices()) + std::cout << i << ","; + + std::cout << "\n"; + + if(m_operationIdentifier != "ConvolutionForward") + { + throw std::runtime_error(std::string("Unsupported operation identifier for check") + + m_operationIdentifier); + } + } + + // Ensure positions are where we expect them to be in the convolution tensor + // description: + auto formatA = counts.formatA(); + auto formatB = counts.formatB(); + auto formatD = counts.formatD(); + + assert(problem.batchIndices().end() + != std::find_if(problem.batchIndices().begin(), + problem.batchIndices().end(), + [formatA](const ContractionProblem::BatchIndex& bi) { + return bi.a == formatA.batchPosition(); + })); + + assert(problem.boundIndices().end() + != std::find_if(problem.boundIndices().begin(), + problem.boundIndices().end(), + [formatA](const ContractionProblem::BoundIndex& bi) { + return bi.a == formatA.channelPosition(); + })); + if(m_operationIdentifier == "ConvolutionForward") + { + for(int i = 0; i < formatA.filterPositions().size(); i++) + { + auto const filterPositionA = formatA.filterPositions()[i]; + if(filterPositionA != ConvolutionProblem::InvalidPos) + assert(problem.boundIndices().end() + != std::find_if( + problem.boundIndices().begin(), + problem.boundIndices().end(), + [filterPositionA](const ContractionProblem::BoundIndex& bi) { + return bi.a == filterPositionA; + })); + } + for(int i = 0; i < formatB.weights().filterPositions().size(); i++) + { + auto const filterPositionB = formatB.weights().filterPositions()[i]; + if(filterPositionB != ConvolutionProblem::InvalidPos) + assert(problem.boundIndices().end() + != std::find_if( + problem.boundIndices().begin(), + problem.boundIndices().end(), + [filterPositionB](const ContractionProblem::BoundIndex& bi) { + return bi.b == filterPositionB; + })); + } + for(auto s : formatA.spatialPositions()) + assert(problem.freeIndicesA().end() + != std::find_if(problem.freeIndicesA().begin(), + problem.freeIndicesA().end(), + [s](const ContractionProblem::FreeIndex& bi) { + return bi.isA && bi.i == s; + })); + } + else + throw std::runtime_error(std::string("Unsupported operation identifier for check") + + m_operationIdentifier); + } + + template + static std::string delimitedVector(const std::vector& v, const std::string& delimiter) + { + std::ostringstream rv; + + std::string delim; + for(auto e : v) + { + rv << delim << e; + delim = delimiter; + } + return rv.str(); + } + + std::string ConvolutionProblem::description() const + { + std::ostringstream rv; + + rv << operationIdentifier(); + + return rv.str(); + } + + TENSILE_API std::ostream& operator<<(std::ostream& stream, + ConvolutionProblem const& convolution) + { + return stream << convolution.description(); + } + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/DataInitialization.cpp b/tensilelite/Tensile/Source/client/source/DataInitialization.cpp new file mode 100644 index 0000000000..11b0edd099 --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/DataInitialization.cpp @@ -0,0 +1,439 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "DataInitialization.hpp" +#include "DataInitializationTyped.hpp" + +#include + +#include + +namespace Tensile +{ + namespace Client + { + std::string ToString(InitMode mode) + { + switch(mode) + { + case InitMode::Zero: + return "Zero"; + case InitMode::One: + return "One"; + case InitMode::Two: + return "Two"; + case InitMode::Random: + return "Random"; + case InitMode::NaN: + return "NaN"; + case InitMode::Inf: + return "Inf"; + case InitMode::BadInput: + return "BadInput"; + case InitMode::BadOutput: + return "BadOutput"; + case InitMode::SerialIdx: + return "SerialIdx"; + case InitMode::SerialDim0: + return "SerialDim0"; + case InitMode::SerialDim1: + return "SerialDim1"; + case InitMode::Identity: + return "Identity"; + case InitMode::TrigSin: + return "TrigSin"; + case InitMode::TrigCos: + return "TrigCos"; + case InitMode::TrigAbsSin: + return "TrigAbsSin"; + case InitMode::TrigAbsCos: + return "TrigAbsCos"; + case InitMode::RandomNarrow: + return "RandomNarrow"; + case InitMode::NegOne: + return "NegOne"; + case InitMode::Max: + return "Max"; + case InitMode::DenormMin: + return "DenormMin"; + case InitMode::DenormMax: + return "DenormMax"; + case InitMode::RandomNegPosLimited: + return "RandomNegPosLimited"; + + case InitMode::Count: + break; + } + + throw std::runtime_error( + concatenate("Invalid InitMode value: ", static_cast(mode))); + } + + std::ostream& operator<<(std::ostream& stream, InitMode const& mode) + { + return stream << ToString(mode); + } + + std::istream& operator>>(std::istream& stream, InitMode& mode) + { + std::string strValue; + stream >> strValue; + + if(strValue == ToString(InitMode::Zero)) + mode = InitMode::Zero; + else if(strValue == ToString(InitMode::One)) + mode = InitMode::One; + else if(strValue == ToString(InitMode::Two)) + mode = InitMode::Two; + else if(strValue == ToString(InitMode::Random)) + mode = InitMode::Random; + else if(strValue == ToString(InitMode::NaN)) + mode = InitMode::NaN; + else if(strValue == ToString(InitMode::Inf)) + mode = InitMode::Inf; + else if(strValue == ToString(InitMode::BadInput)) + mode = InitMode::BadInput; + else if(strValue == ToString(InitMode::BadOutput)) + mode = InitMode::BadOutput; + else if(strValue == ToString(InitMode::SerialIdx)) + mode = InitMode::SerialIdx; + else if(strValue == ToString(InitMode::SerialDim0)) + mode = InitMode::SerialDim0; + else if(strValue == ToString(InitMode::SerialDim1)) + mode = InitMode::SerialDim1; + else if(strValue == ToString(InitMode::Identity)) + mode = InitMode::Identity; + else if(strValue == ToString(InitMode::TrigSin)) + mode = InitMode::TrigSin; + else if(strValue == ToString(InitMode::TrigCos)) + mode = InitMode::TrigCos; + else if(strValue == ToString(InitMode::TrigAbsSin)) + mode = InitMode::TrigAbsSin; + else if(strValue == ToString(InitMode::TrigAbsCos)) + mode = InitMode::TrigAbsCos; + else if(strValue == ToString(InitMode::RandomNarrow)) + mode = InitMode::RandomNarrow; + else if(strValue == ToString(InitMode::NegOne)) + mode = InitMode::NegOne; + else if(strValue == ToString(InitMode::Max)) + mode = InitMode::Max; + else if(strValue == ToString(InitMode::DenormMin)) + mode = InitMode::DenormMin; + else if(strValue == ToString(InitMode::DenormMax)) + mode = InitMode::DenormMax; + else if(strValue == ToString(InitMode::RandomNegPosLimited)) + mode = InitMode::RandomNegPosLimited; + else if(std::all_of(strValue.begin(), strValue.end(), isdigit)) + { + int value = atoi(strValue.c_str()); + if(value >= 0 && value < static_cast(InitMode::Count)) + mode = static_cast(value); + else + throw std::runtime_error( + concatenate("Can't convert ", strValue, " to InitMode.")); + } + else + { + throw std::runtime_error(concatenate("Can't convert ", strValue, " to InitMode.")); + } + + return stream; + } + + std::ostream& operator<<(std::ostream& stream, BoundsCheckMode const& mode) + { + std::string strValue; + + if(mode == BoundsCheckMode::Disable) + strValue = "Disable"; + else if(mode == BoundsCheckMode::NaN) + strValue = "NaN"; + else if(mode == BoundsCheckMode::GuardPageFront) + strValue = "GuardPageFront"; + else if(mode == BoundsCheckMode::GuardPageBack) + strValue = "GuardPageBack"; + else if(mode == BoundsCheckMode::GuardPageAll) + strValue = "GuardPageAll"; + else + throw std::runtime_error( + concatenate("Invalid BoundsCheckMode value: ", static_cast(mode))); + + return stream << strValue; + } + + std::istream& operator>>(std::istream& stream, BoundsCheckMode& mode) + { + std::string strValue; + stream >> strValue; + + if(strValue == "Disable") + mode = BoundsCheckMode::Disable; + else if(strValue == "NaN") + mode = BoundsCheckMode::NaN; + else if(strValue == "GuardPageFront") + mode = BoundsCheckMode::GuardPageFront; + else if(strValue == "GuardPageBack") + mode = BoundsCheckMode::GuardPageBack; + else if(strValue == "GuardPageAll") + mode = BoundsCheckMode::GuardPageAll; + else if(std::all_of(strValue.begin(), strValue.end(), isdigit)) + { + int value = atoi(strValue.c_str()); + if(value >= 0 && value < static_cast(BoundsCheckMode::MaxMode)) + mode = static_cast(value); + else + throw std::runtime_error( + concatenate("Can't convert ", strValue, " to BoundsCheckMode.")); + } + else + { + throw std::runtime_error( + concatenate("Can't convert ", strValue, " to BoundsCheckMode.")); + } + + return stream; + } + + double DataInitialization::GetRepresentativeBetaValue(po::variables_map const& args) + { + auto argValue = args["init-beta"].as(); + + if(argValue == 0) + return 0.0; + + if(argValue == 1) + return 1.0; + + return 1.5; + } + + template + std::shared_ptr> + DataInitialization::GetTyped(po::variables_map const& args, + ClientProblemFactory const& problemFactory, + size_t maxWorkspaceSize) + { + auto* ptr + = new TypedDataInitialization(args, problemFactory, maxWorkspaceSize); + + return std::shared_ptr>(ptr); + } + + std::shared_ptr + DataInitialization::Get(po::variables_map const& args, + ClientProblemFactory const& problemFactory, + size_t maxWorkspaceSize) + { + auto aType = args["a-type"].as(); + auto bType = args["b-type"].as(); + auto cType = args["c-type"].as(); + auto dType = args["d-type"].as(); + auto alphaType = args["alpha-type"].as(); + auto betaType = args["beta-type"].as(); + + auto contractionInputsTypeId + = ContractionInputs::TypeId(aType, bType, cType, dType, alphaType, betaType); + + switch(contractionInputsTypeId) + { + case ContractionInputs_S_S_S::TypeId(): + { + return GetTyped(args, problemFactory, maxWorkspaceSize); + } + case ContractionInputs_D_D_D::TypeId(): + { + return GetTyped(args, problemFactory, maxWorkspaceSize); + } + case ContractionInputs_C_C_C::TypeId(): + { + return GetTyped(args, problemFactory, maxWorkspaceSize); + } + case ContractionInputs_Z_Z_Z::TypeId(): + { + return GetTyped(args, problemFactory, maxWorkspaceSize); + } +#ifdef TENSILE_USE_HALF + case ContractionInputs_H_H_H::TypeId(): + { + return GetTyped(args, problemFactory, maxWorkspaceSize); + } + case ContractionInputs_H_H_S::TypeId(): + { + return GetTyped(args, problemFactory, maxWorkspaceSize); + } + case ContractionInputs_H_S_S::TypeId(): + { + return GetTyped(args, problemFactory, maxWorkspaceSize); + } +#endif // TENSILE_USE_HALF + case ContractionInputs_I8x4_I32_I32::TypeId(): + { + return GetTyped( + args, problemFactory, maxWorkspaceSize); + } + case ContractionInputs_I32_I32_I32::TypeId(): + { + return GetTyped( + args, problemFactory, maxWorkspaceSize); + } + case ContractionInputs_I8_I8_I32::TypeId(): + { + return GetTyped( + args, problemFactory, maxWorkspaceSize); + } + case ContractionInputs_I8_I32_I32::TypeId(): + { + return GetTyped( + args, problemFactory, maxWorkspaceSize); + } +#ifdef TENSILE_USE_BF16 + case ContractionInputs_B_B_S::TypeId(): + { + return GetTyped(args, problemFactory, maxWorkspaceSize); + } + case ContractionInputs_B_S_S::TypeId(): + { + return GetTyped(args, problemFactory, maxWorkspaceSize); + } +#endif // TENSILE_USE_BF16 + default:; + } + + throw std::runtime_error(concatenate("Invalid combination of data types: ", + "a: ", + aType, + ", b: ", + bType, + ", c: ", + cType, + ", d: ", + dType, + ", alpha: ", + alphaType, + ", beta: ", + betaType)); + } + + DataInitialization::DataInitialization(po::variables_map const& args, + ClientProblemFactory const& problemFactory, + size_t maxWorkspaceSize) + : m_aInit(args["init-a"].as()) + , m_bInit(args["init-b"].as()) + , m_cInit(args["init-c"].as()) + , m_dInit(args["init-d"].as()) + , m_alphaInit(args["init-alpha"].as()) + , m_betaInit(args["init-beta"].as()) + , m_biasInit(args["init-bias"].as()) + , m_activationType(ActivationType::None) + , m_activationHPA(false) + , m_aBufferOffset(args["offset-a"].as()) + , m_bBufferOffset(args["offset-b"].as()) + , m_cBufferOffset(args["offset-c"].as()) + , m_dBufferOffset(args["offset-d"].as()) + , m_aMaxElements(0) + , m_bMaxElements(0) + , m_cMaxElements(0) + , m_dMaxElements(0) + , m_maxBatch(0) + , m_stridedBatched(args["strided-batched"].as()) + , m_cEqualsD(args["c-equal-d"].as()) + , m_elementsToValidate(args["num-elements-to-validate"].as()) + , m_keepPristineCopyOnGPU(args["pristine-on-gpu"].as()) + , m_workspaceSize(maxWorkspaceSize) + { + m_boundsCheck = args["bounds-check"].as(); + m_curBoundsCheck = m_boundsCheck; + + if(m_boundsCheck == BoundsCheckMode::GuardPageAll) + { + //GuardPageAll needs 2 runs per solution. + //First run perform front side guard page checking. + m_curBoundsCheck = BoundsCheckMode::GuardPageFront; + m_numRunsPerSolution = 2; + } + + if(args.count("convolution-vs-contraction")) + m_convolutionVsContraction = args["convolution-vs-contraction"].as(); + + for(auto const& problem : problemFactory.problems()) + { + m_aMaxElements = std::max(m_aMaxElements, problem.a().totalAllocatedElements()); + m_bMaxElements = std::max(m_bMaxElements, problem.b().totalAllocatedElements()); + m_cMaxElements = std::max(m_cMaxElements, problem.c().totalAllocatedElements()); + m_dMaxElements = std::max(m_dMaxElements, problem.d().totalAllocatedElements()); + + size_t numOfBatch = 1; + for(size_t i = 0; i < problem.batchIndices().size(); i++) + numOfBatch *= problem.batchSize(i); + m_maxBatch = std::max(m_maxBatch, numOfBatch); + } + + m_aMaxElements += m_aBufferOffset; + m_bMaxElements += m_bBufferOffset; + m_cMaxElements += m_cBufferOffset; + m_dMaxElements += m_dBufferOffset; + + if(m_curBoundsCheck == BoundsCheckMode::NaN) + { + m_aMaxElements += 1024; + m_bMaxElements += 1024; + m_cMaxElements += 1024; + m_dMaxElements += 1024; + } + else if(m_curBoundsCheck == BoundsCheckMode::GuardPageFront + || m_curBoundsCheck == BoundsCheckMode::GuardPageBack) + { + unsigned int aRoundUpSize + = pageSize / DataTypeInfo::Get(args["a-type"].as()).elementSize; + unsigned int bRoundUpSize + = pageSize / DataTypeInfo::Get(args["b-type"].as()).elementSize; + unsigned int cRoundUpSize + = pageSize / DataTypeInfo::Get(args["c-type"].as()).elementSize; + unsigned int dRoundUpSize + = pageSize / DataTypeInfo::Get(args["d-type"].as()).elementSize; + + m_aMaxElements = RoundUpToMultiple(m_aMaxElements, aRoundUpSize); + m_bMaxElements = RoundUpToMultiple(m_bMaxElements, bRoundUpSize); + m_cMaxElements = RoundUpToMultiple(m_cMaxElements, cRoundUpSize); + m_dMaxElements = RoundUpToMultiple(m_dMaxElements, dRoundUpSize); + } + m_problemDependentData = IsProblemDependent(m_aInit) || IsProblemDependent(m_bInit) + || IsProblemDependent(m_cInit) || IsProblemDependent(m_dInit); + + if(args.count("activation-type")) + m_activationType = args["activation-type"].as(); + if(args.count("activation-hpa")) + m_activationHPA = args["activation-hpa"].as(); + if(args.count("activation-additional-args")) + m_activationAdditionalArgs + = args["activation-additional-args"].as>>(); + if(args.count("use-bias")) + m_useBias = args["use-bias"].as(); + } + + DataInitialization::~DataInitialization() {} + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/HardwareMonitor.cpp b/tensilelite/Tensile/Source/client/source/HardwareMonitor.cpp new file mode 100644 index 0000000000..8275e95195 --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/HardwareMonitor.cpp @@ -0,0 +1,444 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "HardwareMonitor.hpp" + +#include +#include +#include +#include + +#include + +#include + +#include "ResultReporter.hpp" + +#define RSMI_CHECK_EXC(expr) \ + do \ + { \ + rsmi_status_t e = (expr); \ + if(e) \ + { \ + const char* errName = nullptr; \ + rsmi_status_string(e, &errName); \ + std::ostringstream msg; \ + msg << "Error " << e << "(" << errName << ") " << __FILE__ << ":" << __LINE__ << ": " \ + << std::endl \ + << #expr << std::endl; \ + throw std::runtime_error(msg.str()); \ + } \ + } while(0) + +namespace Tensile +{ + namespace Client + { + uint32_t HardwareMonitor::GetROCmSMIIndex(int hipDeviceIndex) + { + InitROCmSMI(); + + hipDeviceProp_t props; + + HIP_CHECK_EXC(hipGetDeviceProperties(&props, hipDeviceIndex)); +#if HIP_VERSION >= 50220730 + int hip_version; + HIP_CHECK_EXC(hipRuntimeGetVersion(&hip_version)); + if(hip_version >= 50220730) + { + HIP_CHECK_EXC(hipDeviceGetAttribute(&props.multiProcessorCount, + hipDeviceAttributePhysicalMultiProcessorCount, + hipDeviceIndex)); + } +#endif + + uint64_t hipPCIID = 0; + hipPCIID |= props.pciDeviceID & 0xFF; + hipPCIID |= ((props.pciBusID & 0xFF) << 8); + hipPCIID |= (props.pciDomainID) << 16; + + uint32_t smiCount = 0; + + RSMI_CHECK_EXC(rsmi_num_monitor_devices(&smiCount)); + + std::ostringstream msg; + msg << "PCI IDs: [" << std::endl; + + for(uint32_t smiIndex = 0; smiIndex < smiCount; smiIndex++) + { + uint64_t rsmiPCIID = 0; + + RSMI_CHECK_EXC(rsmi_dev_pci_id_get(smiIndex, &rsmiPCIID)); + + msg << smiIndex << ": " << rsmiPCIID << std::endl; + + if(hipPCIID == rsmiPCIID) + return smiIndex; + } + + msg << "]" << std::endl; + std::time_t now + = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + msg << std::put_time(gmtime(&now), "%F %T %z"); + + throw std::runtime_error(concatenate("RSMI Can't find a device with PCI ID ", + hipPCIID, + "(", + props.pciDomainID, + "-", + props.pciBusID, + "-", + props.pciDeviceID, + ")\n", + msg.str())); + } + + void HardwareMonitor::InitROCmSMI() + { + static rsmi_status_t status = rsmi_init(0); + RSMI_CHECK_EXC(status); + } + + HardwareMonitor::HardwareMonitor(int hipDeviceIndex, clock::duration minPeriod) + : m_minPeriod(minPeriod) + , m_hipDeviceIndex(hipDeviceIndex) + , m_smiDeviceIndex(GetROCmSMIIndex(hipDeviceIndex)) + , m_dataPoints(0) + { + InitROCmSMI(); + + initThread(); + } + + HardwareMonitor::HardwareMonitor(int hipDeviceIndex) + : m_minPeriod(clock::duration::zero()) + , m_hipDeviceIndex(hipDeviceIndex) + , m_smiDeviceIndex(GetROCmSMIIndex(hipDeviceIndex)) + , m_dataPoints(0) + { + InitROCmSMI(); + + initThread(); + } + + HardwareMonitor::~HardwareMonitor() + { + m_stop = true; + m_exit = true; + + m_cv.notify_all(); + m_thread.join(); + } + + void HardwareMonitor::initThread() + { + m_stop = false; + m_exit = false; + m_thread = std::thread([=]() { this->runLoop(); }); + } + + void HardwareMonitor::addTempMonitor(rsmi_temperature_type_t sensorType, + rsmi_temperature_metric_t metric) + { + assertNotActive(); + + m_tempMetrics.emplace_back(sensorType, metric); + m_tempValues.resize(m_tempMetrics.size()); + } + + void HardwareMonitor::addClockMonitor(rsmi_clk_type_t clockType) + { + assertNotActive(); + + m_clockMetrics.push_back(clockType); + m_clockValues.resize(m_clockMetrics.size()); + } + + void HardwareMonitor::addFanSpeedMonitor(uint32_t sensorIndex) + { + assertNotActive(); + + m_fanMetrics.push_back(sensorIndex); + m_fanValues.resize(m_fanMetrics.size()); + } + + double HardwareMonitor::getAverageTemp(rsmi_temperature_type_t sensorType, + rsmi_temperature_metric_t metric) + { + assertNotActive(); + + if(m_dataPoints == 0) + throw std::runtime_error("No data points collected!"); + + for(size_t i = 0; i < m_tempMetrics.size(); i++) + { + if(m_tempMetrics[i] == std::make_tuple(sensorType, metric)) + { + int64_t rawValue = m_tempValues[i]; + if(rawValue == std::numeric_limits::max()) + return std::numeric_limits::quiet_NaN(); + + return static_cast(rawValue) / (1000.0 * m_dataPoints); + } + } + + throw std::runtime_error(concatenate( + "Can't read temp value that wasn't requested: ", sensorType, " - ", metric)); + } + + double HardwareMonitor::getAverageClock(rsmi_clk_type_t clockType) + { + assertNotActive(); + + if(m_dataPoints == 0) + throw std::runtime_error("No data points collected!"); + + for(size_t i = 0; i < m_clockMetrics.size(); i++) + { + if(m_clockMetrics[i] == clockType) + { + uint64_t rawValue = m_clockValues[i]; + if(rawValue == std::numeric_limits::max()) + return std::numeric_limits::quiet_NaN(); + + return static_cast(rawValue) / (1e6 * m_dataPoints); + } + } + + throw std::runtime_error( + concatenate("Can't read clock value that wasn't requested: ", clockType)); + } + + double HardwareMonitor::getAverageFanSpeed(uint32_t sensorIndex) + { + assertNotActive(); + + if(m_dataPoints == 0) + throw std::runtime_error("No data points collected!"); + + for(size_t i = 0; i < m_fanMetrics.size(); i++) + { + if(m_fanMetrics[i] == sensorIndex) + { + int64_t rawValue = m_fanValues[i]; + if(rawValue == std::numeric_limits::max()) + return std::numeric_limits::quiet_NaN(); + + return static_cast(rawValue) / m_dataPoints; + } + } + + throw std::runtime_error( + concatenate("Can't read fan value that wasn't requested: ", sensorIndex)); + } + + void HardwareMonitor::start() + { + runBetweenEvents(nullptr, nullptr); + } + + void HardwareMonitor::stop() + { + assertActive(); + + m_stop = true; + } + + void HardwareMonitor::runUntilEvent(hipEvent_t event) + { + runBetweenEvents(nullptr, event); + } + + void HardwareMonitor::runBetweenEvents(hipEvent_t startEvent, hipEvent_t stopEvent) + { + assertNotActive(); + + { + std::unique_lock lock(m_mutex); + + m_hasStopEvent = stopEvent != nullptr; + + m_task = std::move(Task([=]() { this->collect(startEvent, stopEvent); })); + m_future = m_task.get_future(); + + m_stop = false; + m_exit = false; + } + m_cv.notify_all(); + } + + void HardwareMonitor::clearValues() + { + m_dataPoints = 0; + + for(auto& v : m_tempValues) + v = 0; + for(auto& v : m_clockValues) + v = 0; + for(auto& v : m_fanValues) + v = 0; + + m_lastCollection = clock::time_point(); + m_nextCollection = clock::time_point(); + } + + void HardwareMonitor::collectOnce() + { + for(int i = 0; i < m_tempMetrics.size(); i++) + { + // if an error occurred previously, don't overwrite it. + if(m_tempValues[i] == std::numeric_limits::max()) + continue; + + rsmi_temperature_type_t sensorType; + rsmi_temperature_metric_t metric; + std::tie(sensorType, metric) = m_tempMetrics[i]; + + int64_t newValue = 0; + auto status + = rsmi_dev_temp_metric_get(m_smiDeviceIndex, sensorType, metric, &newValue); + if(status != RSMI_STATUS_SUCCESS) + m_tempValues[i] = std::numeric_limits::max(); + else + m_tempValues[i] += newValue; + } + + for(int i = 0; i < m_clockMetrics.size(); i++) + { + // if an error occurred previously, don't overwrite it. + if(m_clockValues[i] == std::numeric_limits::max()) + continue; + + rsmi_frequencies_t freq; + + auto status = rsmi_dev_gpu_clk_freq_get(m_smiDeviceIndex, m_clockMetrics[i], &freq); + if(status != RSMI_STATUS_SUCCESS) + { + m_clockValues[i] = std::numeric_limits::max(); + } + else + { + m_clockValues[i] += freq.frequency[freq.current]; + } + } + + for(int i = 0; i < m_fanMetrics.size(); i++) + { + // if an error occurred previously, don't overwrite it. + if(m_fanValues[i] == std::numeric_limits::max()) + continue; + + rsmi_frequencies_t freq; + + int64_t newValue = 0; + auto status = rsmi_dev_fan_rpms_get(m_smiDeviceIndex, m_fanMetrics[i], &newValue); + if(status != RSMI_STATUS_SUCCESS) + m_fanValues[i] = std::numeric_limits::max(); + else + m_fanValues[i] += newValue; + } + + m_dataPoints++; + } + + void HardwareMonitor::sleepIfNecessary() + { + auto now = clock::now(); + + if(now < m_nextCollection) + { + std::this_thread::sleep_until(m_nextCollection); + m_lastCollection = clock::now(); + } + else + { + m_lastCollection = now; + } + + m_nextCollection = m_lastCollection + m_minPeriod; + } + + void HardwareMonitor::runLoop() + { + hipSetDevice(m_hipDeviceIndex); + + std::unique_lock lock(m_mutex); + while(!m_exit) + { + while(!m_task.valid() && !m_exit) + m_cv.wait(lock); + + if(m_exit) + return; + + m_task(); + m_task = std::move(Task()); + } + } + + void HardwareMonitor::collect(hipEvent_t startEvent, hipEvent_t stopEvent) + { + clearValues(); + + if(startEvent != nullptr) + HIP_CHECK_EXC(hipEventSynchronize(startEvent)); + + do + { + collectOnce(); + sleepIfNecessary(); + + if(stopEvent != nullptr && hipEventQuery(stopEvent) == hipSuccess) + return; + } while(!m_stop && !m_exit); + } + + void HardwareMonitor::wait() + { + if(!m_future.valid()) + return; + + if(!m_hasStopEvent && !m_stop) + throw std::runtime_error("Waiting for monitoring to stop with no end condition."); + + m_future.wait(); + m_future = std::move(std::future()); + } + + void HardwareMonitor::assertActive() + { + if(!m_future.valid()) + throw std::runtime_error("Monitor is not active."); + } + + void HardwareMonitor::assertNotActive() + { + if(m_future.valid()) + throw std::runtime_error("Monitor is active."); + } + + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/HardwareMonitorListener.cpp b/tensilelite/Tensile/Source/client/source/HardwareMonitorListener.cpp new file mode 100644 index 0000000000..5e556be56f --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/HardwareMonitorListener.cpp @@ -0,0 +1,105 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "HardwareMonitor.hpp" + +#include + +#include + +#include + +#include "HardwareMonitorListener.hpp" +#include "ResultReporter.hpp" + +namespace Tensile +{ + namespace Client + { + HardwareMonitorListener::HardwareMonitorListener(po::variables_map const& args) + : m_useGPUTimer(args["use-gpu-timer"].as()) + , m_active(args["hardware-monitor"].as()) + { + if(!m_active) + return; + + m_monitor = std::make_shared(args["device-idx"].as()); + m_monitor->addTempMonitor(0); + + m_monitor->addClockMonitor(RSMI_CLK_TYPE_SYS); + m_monitor->addClockMonitor(RSMI_CLK_TYPE_SOC); + m_monitor->addClockMonitor(RSMI_CLK_TYPE_MEM); + + m_monitor->addFanSpeedMonitor(); + } + + void HardwareMonitorListener::preEnqueues() + { + if(m_active && !m_useGPUTimer) + m_monitor->start(); + } + + void HardwareMonitorListener::postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + if(!m_active) + return; + + if(m_useGPUTimer) + { + m_monitor->runBetweenEvents(startEvents->front().front(), + stopEvents->back().back()); + } + else + { + m_monitor->stop(); + } + } + + void HardwareMonitorListener::validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + if(!m_active) + return; + + m_monitor->wait(); + + m_reporter->report(ResultKey::DeviceIndex, m_monitor->getDeviceIndex()); + m_reporter->report(ResultKey::TempEdge, m_monitor->getAverageTemp(0)); + + m_reporter->report(ResultKey::ClockRateSys, + m_monitor->getAverageClock(RSMI_CLK_TYPE_SYS)); + m_reporter->report(ResultKey::ClockRateSOC, + m_monitor->getAverageClock(RSMI_CLK_TYPE_SOC)); + m_reporter->report(ResultKey::ClockRateMem, + m_monitor->getAverageClock(RSMI_CLK_TYPE_MEM)); + + m_reporter->report(ResultKey::FanSpeedRPMs, m_monitor->getAverageFanSpeed()); + m_reporter->report(ResultKey::HardwareSampleCount, m_monitor->getSamples()); + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/LibraryUpdateReporter.cpp b/tensilelite/Tensile/Source/client/source/LibraryUpdateReporter.cpp new file mode 100644 index 0000000000..3cac11e7c1 --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/LibraryUpdateReporter.cpp @@ -0,0 +1,182 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include + +#include + +namespace Tensile +{ + namespace Client + { + std::shared_ptr + LibraryUpdateReporter::Default(po::variables_map const& args) + { + auto filename = args["library-update-file"].as(); + auto comment = args["library-update-comment"].as(); + if(filename != "") + { + return FromFilename(filename, comment); + } + return std::shared_ptr(); + } + + std::shared_ptr + LibraryUpdateReporter::FromFilename(std::string const& filename, bool addComment) + { + auto file = std::make_shared(filename); + return std::make_shared(file, addComment); + } + + LibraryUpdateReporter::LibraryUpdateReporter(std::ostream& stream, bool addComment) + : m_stream(stream) + , m_addComment(addComment) + { + } + + LibraryUpdateReporter::LibraryUpdateReporter(std::shared_ptr stream, + bool addComment) + : m_stream(*stream.get()) + , m_ownedStream(stream) + , m_addComment(addComment) + { + if(!stream) + throw std::runtime_error("Invalid stream! nullptr is not allowed."); + } + + template + void LibraryUpdateReporter::reportValue(std::string const& key, T const& value) + { + std::string valueStr = boost::lexical_cast(value); + //m_stream << key << " = " << valueStr << std::endl; + + if(key == ResultKey::Validation) + { + m_curSolutionPassed = (valueStr == "PASSED" || valueStr == "NO_CHECK"); + } + else if(key == ResultKey::SolutionLibraryIndex) + { + m_curSolutionIdx = std::stoi(valueStr); + } + else if(key == ResultKey::SolutionName) + { + m_curSolutionName = valueStr; + } + else if(key == ResultKey::SpeedGFlops) + { + try + { + double speed = std::stod(valueStr); + m_curSolutionSpeed = speed; + //m_stream << "Fastest: " << m_fastestSolutionSpeed << std::endl; + } + catch(std::out_of_range const& exc) + { + } + } + } + + void LibraryUpdateReporter::reportValue_string(std::string const& key, + std::string const& value) + { + reportValue(key, value); + } + + void LibraryUpdateReporter::reportValue_uint(std::string const& key, uint64_t value) + { + reportValue(key, value); + } + + void LibraryUpdateReporter::reportValue_int(std::string const& key, int64_t value) + { + reportValue(key, value); + } + + void LibraryUpdateReporter::reportValue_double(std::string const& key, double value) + { + reportValue(key, value); + } + + void LibraryUpdateReporter::reportValue_sizes(std::string const& key, + std::vector const& value) + { + if(key == ResultKey::ProblemSizes) + { + m_problemSizes = value; + } + } + + void LibraryUpdateReporter::postProblem() + { + if(m_fastestSolutionIdx < 0) + { + m_stream << "# ["; + streamJoin(m_stream, m_problemSizes, ", "); + m_stream << "] no valid solutions." << std::endl; + } + else + { + // - - [1024, 4096, 1, 6336] + // - [289, 4853.07] + m_stream << " - - ["; + streamJoin(m_stream, m_problemSizes, ", "); + m_stream << "]" << std::endl; + m_stream << " - [" << m_fastestSolutionIdx << ", " << m_fastestSolutionSpeed + << "]"; + if(m_addComment) + m_stream << " # " << m_fastestSolutionName; + m_stream << std::endl; + } + + // reset + m_fastestSolutionIdx = -1; + m_fastestSolutionName = ""; + m_fastestSolutionSpeed = -1.0; + } + + void LibraryUpdateReporter::postSolution() + { + // cascade from BenchmarkTimer, SpeedGFlops second + if(m_curSolutionPassed && m_curSolutionSpeed > m_fastestSolutionSpeed) + { + m_fastestSolutionIdx = m_curSolutionIdx; + m_fastestSolutionName = m_curSolutionName; + m_fastestSolutionSpeed = m_curSolutionSpeed; + } + + m_curSolutionName = ""; + m_curSolutionIdx = -1; + m_curSolutionSpeed = -1.0; + m_curSolutionPassed = false; + } + + void LibraryUpdateReporter::finalizeReport() + { + // Close file if we're the last owner. + m_ownedStream.reset(); + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/MetaRunListener.cpp b/tensilelite/Tensile/Source/client/source/MetaRunListener.cpp new file mode 100644 index 0000000000..31f5766f47 --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/MetaRunListener.cpp @@ -0,0 +1,241 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "MetaRunListener.hpp" + +#include +#include + +#include "ResultReporter.hpp" +#include "RunListener.hpp" + +#include + +namespace Tensile +{ + namespace Client + { + void MetaRunListener::addListener(std::shared_ptr listener) + { + if(!m_firstProblem) + throw std::runtime_error("Can't add listeners after benchmarking has begun."); + + listener->setReporter(m_reporter); + + m_listeners.push_back(listener); + } + + void MetaRunListener::setReporter(std::shared_ptr reporter) + { + if(!m_firstProblem) + throw std::runtime_error("Can't set reporter after benchmarking has begun."); + if(m_reporter) + throw std::runtime_error("Can't set reporter more than once."); + + m_reporter = reporter; + for(auto const& listener : m_listeners) + listener->setReporter(reporter); + + m_listeners.insert(m_listeners.begin(), reporter); + } + + bool MetaRunListener::needMoreBenchmarkRuns() const + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + if((*iter)->needMoreBenchmarkRuns()) + return true; + + return false; + } + + void MetaRunListener::preBenchmarkRun() + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->preBenchmarkRun(); + } + + void MetaRunListener::postBenchmarkRun() + { + for(auto iter = m_listeners.rbegin(); iter != m_listeners.rend(); iter++) + (*iter)->postBenchmarkRun(); + } + + void MetaRunListener::preProblem(ContractionProblem const& problem) + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->preProblem(problem); + } + + void MetaRunListener::postProblem() + { + for(auto iter = m_listeners.rbegin(); iter != m_listeners.rend(); iter++) + (*iter)->postProblem(); + } + + void MetaRunListener::preSolution(ContractionSolution const& solution) + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->preSolution(solution); + } + + void MetaRunListener::postSolution() + { + for(auto iter = m_listeners.rbegin(); iter != m_listeners.rend(); iter++) + (*iter)->postSolution(); + } + + bool MetaRunListener::needMoreRunsInSolution() const + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + if((*iter)->needMoreRunsInSolution()) + return true; + + return false; + } + + size_t MetaRunListener::numWarmupRuns() + { + size_t count = 0; + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + count = std::max(count, (*iter)->numWarmupRuns()); + + setNumWarmupRuns(count); + + return count; + } + + void MetaRunListener::setNumWarmupRuns(size_t count) + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->setNumWarmupRuns(count); + } + + void MetaRunListener::preWarmup() + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->preWarmup(); + } + + void MetaRunListener::postWarmup() + { + for(auto iter = m_listeners.rbegin(); iter != m_listeners.rend(); iter++) + (*iter)->postWarmup(); + } + + void MetaRunListener::validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->validateWarmups(inputs, startEvents, stopEvents); + } + + size_t MetaRunListener::numSyncs() + { + size_t count = 0; + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + count = std::max(count, (*iter)->numSyncs()); + + setNumSyncs(count); + + return count; + } + + void MetaRunListener::setNumSyncs(size_t count) + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->setNumSyncs(count); + } + + void MetaRunListener::preSyncs() + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->preSyncs(); + } + + void MetaRunListener::postSyncs() + { + for(auto iter = m_listeners.rbegin(); iter != m_listeners.rend(); iter++) + (*iter)->postSyncs(); + } + + size_t MetaRunListener::numEnqueuesPerSync() + { + size_t count = 0; + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + count = std::max(count, (*iter)->numEnqueuesPerSync()); + + setNumEnqueuesPerSync(count); + + return count; + } + + void MetaRunListener::setNumEnqueuesPerSync(size_t count) + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->setNumEnqueuesPerSync(count); + } + + void MetaRunListener::preEnqueues() + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->preEnqueues(); + } + + void MetaRunListener::postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + for(auto iter = m_listeners.rbegin(); iter != m_listeners.rend(); iter++) + (*iter)->postEnqueues(startEvents, stopEvents); + } + + void MetaRunListener::validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->validateEnqueues(inputs, startEvents, stopEvents); + } + + void MetaRunListener::finalizeReport() + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + (*iter)->finalizeReport(); + } + + int MetaRunListener::error() const + { + for(auto iter = m_listeners.begin(); iter != m_listeners.end(); iter++) + { + int rv = (*iter)->error(); + if(rv) + return rv; + } + + return 0; + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/PerformanceReporter.cpp b/tensilelite/Tensile/Source/client/source/PerformanceReporter.cpp new file mode 100644 index 0000000000..92b16b87bb --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/PerformanceReporter.cpp @@ -0,0 +1,199 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2020-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "PerformanceReporter.hpp" + +#include +#include +#include +#include +#include + +namespace Tensile +{ + namespace Client + { + std::shared_ptr + PerformanceReporter::Default(po::variables_map const& args) + { + int deviceIndex = args["device-idx"].as(); + double l2ReadHits = args["perf-l2-read-hits"].as(); + double l2WriteHits = args["perf-l2-write-hits"].as(); + double l2ReadBwMultiplier = args["perf-l2-read-bw-mul"].as(); + double readEff = args["perf-read-efficiency"].as(); + + return std::make_shared( + deviceIndex, l2ReadHits, l2WriteHits, l2ReadBwMultiplier, readEff); + } + + PerformanceReporter::PerformanceReporter(int deviceIndex, + double l2ReadHits, + double l2WriteHits, + double l2ReadBwMultiplier, + double readEff) + { + hipGetDeviceProperties(&m_props, deviceIndex); +#if HIP_VERSION >= 50220730 + int hip_version; + hipRuntimeGetVersion(&hip_version); + if(hip_version >= 50220730) + { + hipDeviceGetAttribute(&m_props.multiProcessorCount, + hipDeviceAttributePhysicalMultiProcessorCount, + deviceIndex); + } +#endif + setNumCUs(); + setMemoryBusWidth(); + setPerfModel(l2ReadHits, l2WriteHits, l2ReadBwMultiplier, readEff); + m_deviceProps = true; + + perf.l2ReadHitRate = getL2ReadHits(); + perf.l2WriteHitRate = getL2WriteHits(); + perf.l2ReadBwMul = getL2ReadBwMultiplier(); + perf.readEff = getReadEff(); + perf.CUs = getNumCUs(); + } + + void PerformanceReporter::reportValue_uint(std::string const& key, uint64_t value) + { + reportValue_numeric(key, value); + } + + void PerformanceReporter::reportValue_double(std::string const& key, double value) + { + reportValue_numeric(key, value); + } + + template + void PerformanceReporter::reportValue_numeric(std::string const& key, T value) + { + if(key == ResultKey::ClockRateSys && m_deviceProps) + { + setClockMhz(value); + } + if(key == ResultKey::ClockRateMem && m_deviceProps) + { + setMemClockMhz(value); + } + } + + void PerformanceReporter::setClockMhz(double value) + { + m_clockMhz = value; + perf.clock = getClockMhz(); + } + + void PerformanceReporter::setMemClockMhz(double value) + { + m_memClockMhz = value; + perf.memClock = getMemClockMhz(); + setMemBandwidthMBps(); + } + + void PerformanceReporter::setMemBandwidthMBps() + { + m_memBandwidthMBps = m_memoryBusWidth * m_memClockMhz; + perf.memBandwidthMBps = getMemBandwidthMBps(); + } + + void PerformanceReporter::postSolution() + { + m_clockMhz = std::numeric_limits::quiet_NaN(); + m_memClockMhz = std::numeric_limits::quiet_NaN(); + m_gFlops = std::numeric_limits::quiet_NaN(); + m_memBandwidthMBps = std::numeric_limits::quiet_NaN(); + } + + void PerformanceReporter::setPerfModel(double l2ReadHits, + double l2WriteHits, + double l2ReadBwMultiplier, + double readEff) + { + m_l2ReadHits = l2ReadHits; + m_l2WriteHits = l2WriteHits; + m_l2ReadBwMul = l2ReadBwMultiplier; + m_readEff = readEff; + } + + void PerformanceReporter::setNumCUs() + { + m_numCUs = m_props.multiProcessorCount; + } + + void PerformanceReporter::setMemoryBusWidth() + { + m_memoryBusWidth = m_props.memoryBusWidth / 1024; + } + + int PerformanceReporter::getNumCUs() + { + return m_numCUs; + } + double PerformanceReporter::getMemClockMhz() + { + return m_memClockMhz; + } + double PerformanceReporter::getClockMhz() + { + return m_clockMhz; + } + double PerformanceReporter::getL2ReadBwMultiplier() + { + return m_l2ReadBwMul; + } + double PerformanceReporter::getL2ReadHits() + { + return m_l2ReadHits; + } + double PerformanceReporter::getL2WriteHits() + { + return m_l2WriteHits; + } + double PerformanceReporter::getReadEff() + { + return m_readEff; + } + double PerformanceReporter::getMemBandwidthMBps() + { + return m_memBandwidthMBps; + } + + void PerformanceReporter::reportValue_int(std::string const& key, int64_t value) {} + void PerformanceReporter::reportValue_string(std::string const& key, + std::string const& value) + { + } + void PerformanceReporter::reportValue_sizes(std::string const& key, + std::vector const& value) + { + } + void PerformanceReporter::preProblem(ContractionProblem const& problem) {} + void PerformanceReporter::preSolution(ContractionSolution const& solution) {} + void PerformanceReporter::finalizeReport() {} + + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/ProgressListener.cpp b/tensilelite/Tensile/Source/client/source/ProgressListener.cpp new file mode 100644 index 0000000000..0df1e60626 --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/ProgressListener.cpp @@ -0,0 +1,170 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include + +#include +#include + +#include + +namespace Tensile +{ + namespace Client + { + ProgressListener::ProgressListener(po::variables_map const& args) + : m_runOnce(args["selection-only"].as()) + { + } + + bool ProgressListener::needMoreBenchmarkRuns() const + { + return m_runOnce; + } + + void ProgressListener::preBenchmarkRun() + { + m_runOnce = false; + m_reporter->report(ResultKey::BenchmarkRunNumber, m_benchmarkRun); + } + + void ProgressListener::postBenchmarkRun() + { + m_benchmarkRun++; + } + + void ProgressListener::preProblem(ContractionProblem const& problem) + { + + m_reporter->report(ResultKey::OperationIdentifier, problem.operationIdentifier()); + + m_reporter->report(ResultKey::TotalFlops, problem.flopCount()); + + m_reporter->report(ResultKey::ASizes, problem.a().sizes()); + m_reporter->report(ResultKey::BSizes, problem.b().sizes()); + m_reporter->report(ResultKey::CSizes, problem.c().sizes()); + m_reporter->report(ResultKey::DSizes, problem.d().sizes()); + + m_reporter->report(ResultKey::AStrides, problem.a().strides()); + m_reporter->report(ResultKey::BStrides, problem.b().strides()); + m_reporter->report(ResultKey::CStrides, problem.c().strides()); + m_reporter->report(ResultKey::DStrides, problem.d().strides()); + + m_reporter->report(ResultKey::LDA, problem.a().strides()[1]); + m_reporter->report(ResultKey::LDB, problem.b().strides()[1]); + m_reporter->report(ResultKey::LDC, problem.c().strides()[1]); + m_reporter->report(ResultKey::LDD, problem.d().strides()[1]); + + m_reporter->report(ResultKey::ProblemSizes, problem.problemSizes()); + m_reporter->report(ResultKey::ActivationType, ToString(problem.activationEnumArg())); + } + + void ProgressListener::postProblem() {} + + void ProgressListener::preSolution(ContractionSolution const& solution) + { + m_reporter->report(ResultKey::SolutionName, solution.name()); + m_reporter->report(ResultKey::SolutionIndex, solution.index); + } + + void ProgressListener::postSolution() {} + + bool ProgressListener::needMoreRunsInSolution() const + { + return false; + } + + size_t ProgressListener::numWarmupRuns() + { + return 0; + } + + void ProgressListener::setNumWarmupRuns(size_t count) {} + + void ProgressListener::preWarmup() {} + + void ProgressListener::postWarmup() {} + + void ProgressListener::validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + } + + size_t ProgressListener::numSyncs() + { + return 0; + } + + void ProgressListener::setNumSyncs(size_t count) {} + + void ProgressListener::preSyncs() {} + + void ProgressListener::postSyncs() {} + + size_t ProgressListener::numEnqueuesPerSync() + { + return 0; + } + + void ProgressListener::setNumEnqueuesPerSync(size_t count) {} + + void ProgressListener::preEnqueues() {} + + void ProgressListener::postEnqueues(TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + } + + void ProgressListener::validateEnqueues(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + struct timeval tmnow; + struct tm* tm; + gettimeofday(&tmnow, NULL); // microsecond resolution + tm = localtime(&tmnow.tv_sec); + std::cout.fill('0'); + + std::ostringstream msg; + msg.fill('0'); + msg << (tm->tm_year + 1900) << "-" << std::setw(2) << (tm->tm_mon + 1) << "-" + << std::setw(2) << tm->tm_mday << " " << std::setw(2) << tm->tm_hour << ":" + << std::setw(2) << tm->tm_min << ":" << std::setw(2) << tm->tm_sec << "." + << std::setw(6) << static_cast(tmnow.tv_usec); + + m_reporter->report(ResultKey::EnqueueTime, msg.str()); + } + + void ProgressListener::finalizeReport() {} + + int ProgressListener::error() const + { + return 0; + } + + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/Reference.cpp b/tensilelite/Tensile/Source/client/source/Reference.cpp new file mode 100644 index 0000000000..6f1b05b40f --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/Reference.cpp @@ -0,0 +1,795 @@ +/** + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "Reference.hpp" +#include "Tensile/Debug.hpp" +#include "Tensile/Utils.hpp" + +#include + +namespace Tensile +{ + namespace Client + { + template + struct Transform + { + inline static T Input(T const& val, bool conj) + { + return val; + } + }; + + template + struct Transform> + { + inline static std::complex Input(std::complex const& val, bool conj) + { + if(conj) + return std::conj(val); + + return val; + } + }; + + // coord is vector with coordinates for dimensions in the anchor tensor + // tensor is tensor descriptor for a or b + // sumCoord is the coordinate in the sum dimension associated with the zero-pad + bool inZeroPad(ContractionProblem const& problem, + ContractionProblem::ZeroPad const& zp, + const TensorDescriptor& tensor, + const std::vector& anchorCoord, + int64_t sumCoord) + { + if(zp.valid()) + { + // Check to see if the element coordinate is below or above the zero-pad + // range The comparison is done in the element domain. + assert(zp.anchorPos != -1); // ensure initialized. + int64_t anchorRelCoord = anchorCoord[zp.anchorPos] * tensor.strides()[zp.anchorPos] + + sumCoord * tensor.strides()[zp.boundPos]; + // elementEdge calculation: + // size of anchor dim is in the output space, so add filter size-1 to get + // input spatial dim, then subtract padEnd anchorStride is typically spatial + // stride (W,H) * convolution stride boundPos stride is typically spatial + // stride (W,H) * dilation padStart, padEnd are pre-scaled by spatial stride + int64_t elementEdge + = tensor.sizes().at(zp.anchorPos) * tensor.strides()[zp.anchorPos] + + (tensor.sizes().at(zp.boundPos) - 1) * tensor.strides()[zp.boundPos] + - zp.padEnd; + + bool rv = anchorRelCoord < zp.padStart || anchorRelCoord >= elementEdge; + return rv; + } + else + { + return false; + } + } + + void throwException(const std::string& msg) + { + throw std::runtime_error(msg.c_str()); + } + + template + inline Accumulator multiply(TypeL l, TypeR r) + { + /* Transform the data type from TypeL/TypeR to Accumulator if TypeL!=ACC or TypeR!=ACC, but filter out cases, I8/I32/I32 and I8x4/I32/I32 + * + * There are three cases of doing multiplication and their conditions to do transform or not are as below. + * 1. AxB : (A!=ACC or B!=ACC) and A!=I8 and A!=I8x4 + * 2. Alpha x rC : (Alpha!=ACC or rC!=ACC) + * 3. Beta x C : (Beta!=ACC or C!=ACC) + */ + constexpr bool needAccumCast + = !(std::is_same() && std::is_same()) + && !std::is_same() //case I8/I32/I32, I8 be implicitly cast to int. + && !std::is_same(); //case I8x4/I32/I32, I8x4 overloading the op*. + + using LMultT = std::conditional_t; + using RMultT = std::conditional_t; + return static_cast(static_cast(l) * static_cast(r)); + } + + template + typename std::enable_if::value, T>::type + SaturateCast(Accumulator val) + { + if(val > static_cast(127)) + val = static_cast(127); + else if(val < static_cast(-128)) + val = static_cast(-128); + return static_cast(val); + } + + template + typename std::enable_if::value, T>::type + SaturateCast(Accumulator val) + { + return static_cast(val); + } + + template + typename std::enable_if::value || std::is_same::value + || std::is_same::value, + T>::type + Activation(ActivationType activationType, + T val, + ActivationType activationType2, + std::vector args) + { + // Only cast to float in BFloat16 + constexpr bool needCast = std::is_same(); + using castT = std::conditional_t; + auto new_type + = activationType == ActivationType::All ? activationType2 : activationType; + if(new_type == ActivationType::Abs) + { + return static_cast( + std::max(static_cast(val), -static_cast(val))); + } + else if(new_type == ActivationType::Clippedrelu) + { + if(val >= args[0]) + return static_cast( + std::min(static_cast(val), static_cast(args[1]))); + return static_cast(0.0); + } + else if(new_type == ActivationType::Exp) + { + return static_cast(exp(static_cast(val))); + } + else if(new_type == ActivationType::Gelu) + { + auto castedVal = static_cast(val); + auto k0 = static_cast(0.7978845608028654); + auto k1 = static_cast(0.044715); + // float(0.5 * x * (1 + tanh(k0 * x * (1 + k1 * x * x)))); + auto tmp = (static_cast(1) + + multiply(k1, multiply(castedVal, castedVal))); + tmp = multiply(k0, multiply(castedVal, tmp)); + tmp = static_cast(1) + static_cast(tanh(tmp)); + tmp = multiply(static_cast(0.5f), + multiply(castedVal, tmp)); + return static_cast(tmp); + } + else if(new_type == ActivationType::Leakyrelu) + { + assert((args.size() == getAdditionalArgNum(activationType))); + auto tmp = static_cast(val); + tmp = tmp > static_cast(0.f) ? tmp : multiply(tmp, args[0]); + return (T)(tmp); + } + else if(new_type == ActivationType::Relu) + { + return (T)(std::max(0.f, static_cast(val))); + } + else if(new_type == ActivationType::Sigmoid) + { + return static_cast( + 1.f / (1.f + static_cast(exp(-static_cast(val))))); + } + else if(new_type == ActivationType::Tanh) + { + return multiply( + tanh(multiply(static_cast(val), static_cast(args[0]))), + static_cast(args[1])); + } + return val; + } + + template + typename std::enable_if::value + || std::is_same::value, + T>::type + Activation(ActivationType activationType, + T val, + ActivationType activationType2, + std::vector args) + { + auto new_type + = activationType == ActivationType::All ? activationType2 : activationType; + if(new_type == ActivationType::Abs) + { + return static_cast(std::abs(val)); + } + else if(new_type == ActivationType::Clippedrelu) + { + if(val >= args[0]) + return static_cast(std::min(val, args[1])); + return static_cast(0); + } + else if(new_type == ActivationType::Relu) + { + return static_cast(std::max(static_cast(0.0), val)); + } + else if(new_type == ActivationType::Leakyrelu) + { + assert((args.size() == getAdditionalArgNum(activationType))); + val = val > 0 ? val : val * args[0]; + return val; + } + return val; + } + + template + typename std::enable_if::value && !std::is_same::value + && !std::is_same::value + && !std::is_same::value + && !std::is_same::value, + T>::type + Activation(ActivationType activationType, + T val, + ActivationType activationType2, + std::vector args) + { + return val; + } + + template + void ReferenceSolution::SolveCPU( + ContractionProblem const& problem, Inputs const& inputs, size_t validationStride) + { + auto const& freeIndicesA = problem.freeIndicesA(); + auto const& freeIndicesB = problem.freeIndicesB(); + auto const& batchIndices = problem.batchIndices(); + auto const& boundIndices = problem.boundIndices(); + + auto const& a = problem.a(); + auto const& b = problem.b(); + auto const& c = problem.c(); + auto const& d = problem.d(); + + bool aConjugate = false; + bool bConjugate = false; + + for(auto const& op : problem.aOps()) + if(op.type == TensorOp::Type::ComplexConjugate) + aConjugate = true; + + for(auto const& op : problem.bOps()) + if(op.type == TensorOp::Type::ComplexConjugate) + bConjugate = true; + + std::vector freeASize(freeIndicesA.size()); + std::vector freeBSize(freeIndicesB.size()); + std::vector batchSize(batchIndices.size()); + std::vector boundSize(boundIndices.size()); + + for(int i = 0; i < freeASize.size(); i++) + freeASize[i] = problem.freeSizeA(i); + for(int i = 0; i < freeBSize.size(); i++) + freeBSize[i] = problem.freeSizeB(i); + for(int i = 0; i < batchSize.size(); i++) + batchSize[i] = problem.batchSize(i); + for(int i = 0; i < boundSize.size(); i++) + boundSize[i] = problem.boundSize(i); + + auto boundCount = CoordCount(boundSize.begin() + 1, boundSize.end()); + + if(inputs.alpha != static_cast(0)) + { + if(inputs.a == nullptr || inputs.b == nullptr) + { + std::ostringstream msg; + msg << "Unsupported nullptr for"; + if(!inputs.a) + msg << " A"; + if(!inputs.b) + msg << " B"; + msg << " when Alpha !=0"; + + throw std::runtime_error(msg.str()); + } + } + +#pragma omp parallel for + for(size_t dNum = 0; dNum < d.totalLogicalElements(); dNum += validationStride) + { + std::vector aCoord(a.dimensions()); + std::vector bCoord(b.dimensions()); + std::vector cCoord(c.dimensions()); + std::vector dCoord(d.dimensions()); + + CoordNumbered( + dNum, dCoord.begin(), dCoord.end(), d.sizes().begin(), d.sizes().end()); + + for(size_t i = 0; i < problem.batchIndices().size(); i++) + { + auto const& idx = problem.batchIndices()[i]; + size_t coord = dCoord[idx.d]; + + aCoord[idx.a] = coord; + bCoord[idx.b] = coord; + cCoord[idx.c] = coord; + } + + for(size_t i = 0; i < problem.freeIndices().size(); i++) + { + auto const& idx = problem.freeIndices()[i]; + size_t coord = dCoord[idx.d]; + + cCoord[idx.c] = coord; + + if(idx.isA) + aCoord[idx.i] = coord; + else + bCoord[idx.i] = coord; + } + + Accumulator value(0); + + // Check short-circuit for alpha = 0 + if(inputs.alpha != static_cast(0)) + { + for(size_t boundNum = 0; boundNum < boundCount; boundNum++) + { + std::vector bound(problem.boundIndices().size()); + CoordNumbered(boundNum, + bound.begin() + 1, + bound.end(), + boundSize.begin() + 1, + boundSize.end()); + bool aInZeroPad = false; + bool bInZeroPad = false; + + for(int i = 1; i < bound.size(); i++) + { + auto const& zpA = problem.boundIndices()[i].aZeroPad; + auto const& zpB = problem.boundIndices()[i].bZeroPad; + aCoord[boundIndices[i].a] = bound[i]; + bCoord[boundIndices[i].b] = bound[i]; + + if(problem.boundIndices()[i].aMirror) + aCoord[boundIndices[i].a] + = boundSize[i] - aCoord[boundIndices[i].a] - 1; + if(problem.boundIndices()[i].bMirror) + bCoord[boundIndices[i].b] + = boundSize[i] - bCoord[boundIndices[i].b] - 1; + + if(zpA.valid()) + { + auto sumCoord = bound.at(problem.toBoundsPos(zpA.boundIndex)); + if(problem.boundIndices()[i].aMirror) + sumCoord = boundSize[i] - sumCoord - 1; + + if(inZeroPad(problem, zpA, a, aCoord, sumCoord)) + aInZeroPad = true; + } + if(zpB.valid()) + { + auto sumCoord = bound.at(problem.toBoundsPos(zpB.boundIndex)); + if(problem.boundIndices()[i].bMirror) + sumCoord = boundSize[i] - sumCoord - 1; + if(inZeroPad(problem, zpB, b, bCoord, sumCoord)) + bInZeroPad = true; + } + } + + size_t aIndex = a.index(aCoord); + size_t bIndex = b.index(bCoord); + for(int i = 1; i < bound.size(); i++) + { + auto const& zpA = problem.boundIndices()[i].aZeroPad; + auto const& zpB = problem.boundIndices()[i].bZeroPad; + + aIndex -= zpA.padStart; + bIndex -= zpB.padStart; + } + + auto aStride = problem.a().strides()[boundIndices[0].a]; + auto bStride = problem.b().strides()[boundIndices[0].b]; + + // innermost bound calculation: + for(size_t i = 0; i < boundSize[0]; i++) + { + auto const& zpA = problem.boundIndices()[0].aZeroPad; + auto const& zpB = problem.boundIndices()[0].bZeroPad; + size_t aI = problem.boundIndices()[0].aMirror + ? (boundSize[0] - i - 1) + : i; + size_t bI = problem.boundIndices()[0].bMirror + ? (boundSize[0] - i - 1) + : i; + + typename Inputs::AType aVal(0); + typename Inputs::BType bVal(0); + if(!aInZeroPad && !inZeroPad(problem, zpA, a, aCoord, aI)) + aVal = Transform::Input( + inputs.a[aIndex + (aI * aStride) - zpA.padStart], + aConjugate); + if(!bInZeroPad && !inZeroPad(problem, zpB, b, bCoord, bI)) + bVal = Transform::Input( + inputs.b[bIndex + (bI * bStride) - zpB.padStart], + bConjugate); + + value += multiply(aVal, bVal); + + if(0) + { + std::cout << " bound=" << bound[0] << "," << bound[1] + << " dNum=" << dNum << " value=" << value + << " aInZeroPad=" << aInZeroPad + << " aindex=" << aIndex << " +offset=" + << (int64_t)(i * aStride) - zpA.padStart + //<< " aVal=" << aVal // disable int8 + << "\n"; + } + } + } + } + + auto cIndex = c.index(cCoord); + auto dIndex = d.index(dCoord); + + // Ensure zero*nan returns zero + auto beta = inputs.beta; + auto zero = static_cast(0); + + auto resultD = multiply(inputs.alpha, value) + + ((beta == zero) ? static_cast(zero) + : multiply(beta, inputs.c[cIndex])); + // bias + if (problem.useBias()) + { + int pos = int(dNum % problem.d().sizes()[0]); + auto bias = multiply(Transform::Input(inputs.bias[pos], aConjugate), 1); + resultD += bias; + } + // Activation adds here + std::vector actArgs; + for(int i = 0; i < inputs.activationArgs.size(); i++) + actArgs.push_back(static_cast(inputs.activationArgs[i])); + resultD = Activation( + problem.activationType(), resultD, problem.activationEnumArg(), actArgs); + inputs.d[dIndex] = SaturateCast(resultD); + } + } + + void SolveCPU(ContractionProblem const& problem, + ContractionInputs const& inputs, + size_t validationStride) + { + // retreive alpha/beta type set via setAlpha/BetaType() + auto alphaType = problem.alphaType(); + auto betaType = problem.betaType(); + + // Backward-compatible: when setAlpha/BetaType() wasn't called, use the old way + // Could remove after rocBLAS is updated + if(alphaType == DataType::None) + { + alphaType = problem.a().dataType() == DataType::BFloat16 + ? DataType::Float + : problem.d().dataType(); + } + if(betaType == DataType::None) + { + betaType = alphaType; + } + + auto contractionInputsTypeId = ContractionInputs::TypeId(problem.a().dataType(), + problem.b().dataType(), + problem.c().dataType(), + problem.d().dataType(), + alphaType, + betaType); + + switch(contractionInputsTypeId) + { + case ContractionInputs_S_S_S::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } + case ContractionInputs_D_D_D::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } + case ContractionInputs_C_C_C::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } + case ContractionInputs_Z_Z_Z::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } +#ifdef TENSILE_USE_HALF + case ContractionInputs_H_H_H::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + + if(problem.highPrecisionAccumulate()) + { + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } + else + { + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } + } + case ContractionInputs_H_S_S::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } + case ContractionInputs_H_H_S::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } +#endif // TENSILE_USE_HALF + case ContractionInputs_I8x4_I32_I32::TypeId(): + { + auto const& typedInputs + = dynamic_cast(inputs); + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } + case ContractionInputs_I32_I32_I32::TypeId(): + { + auto const& typedInputs + = dynamic_cast(inputs); + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } + case ContractionInputs_I8_I8_I32::TypeId(): + { + auto const& typedInputs + = dynamic_cast(inputs); + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } + case ContractionInputs_I8_I32_I32::TypeId(): + { + auto const& typedInputs + = dynamic_cast(inputs); + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } +#ifdef TENSILE_USE_BF16 + case ContractionInputs_B_B_S::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + + if(problem.highPrecisionAccumulate()) + { + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } + else + { + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } + } + case ContractionInputs_B_S_S::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return ReferenceSolution::SolveCPU( + problem, typedInputs, validationStride); + } +#endif // TENSILE_USE_BF16 + + default:; + } + + throw std::runtime_error("Data type not implemented."); + } + + // A is activation, B is weights + // Assume packed. + template + void ReferenceSolution::SolveCPUConvolution( + ConvolutionProblem const& convProblem, + ContractionProblem const& problem, + Inputs const& inputs) + { + const bool db1 = Debug::Instance().printConvolutionReference1(); + const bool db2 = Debug::Instance().printConvolutionReference2(); + + if(static_cast(inputs.beta) + != static_cast(0.0)) + throw std::runtime_error("convolution requires beta==0"); + + ConvolutionProblem::LoopCounts counts; + counts.setupForData(convProblem, problem); + + convProblem.validate(problem, counts); + + TensorDescriptor activationTensor + = convProblem.setupDataActivation(counts, problem); + TensorDescriptor weightTensor = convProblem.setupForwardWeights(counts, problem); + TensorDescriptor outputTensor = convProblem.setupDataOutput(counts, problem); + + auto formatA = counts.formatA(); + auto formatB = counts.formatB(); + auto formatD = counts.formatD(); + + size_t padShift + = std::accumulate(problem.aZeroPad().begin(), + problem.aZeroPad().end(), + 0, + [](size_t sum, const ContractionProblem::ZeroPad& zp) { + return sum + zp.padStart; + }); + if(db1) + { + std::cout << "SolveCPUConvolution:\n"; + std::cout << " activationTensor=" << activationTensor << "\n"; + std::cout << "counts:" << std::endl << counts.description() << "\n"; + } + + // Loops always traverse in same order but addressing in memory can be flexible to support different activation + // and filter formats + size_t spatialCoordCount = CoordCount(counts.scount.begin(), counts.scount.end()); +#pragma omp parallel for collapse(3) + for(size_t cout = 0; cout < counts.coutCount; cout++) + for(size_t spatialIndex = 0; spatialIndex < spatialCoordCount; spatialIndex++) + for(size_t n = 0; n < counts.batchCount; n++) + { + std::vector spatialCoord(ConvolutionProblem::MaxNumSpatialDims, + 0); + + CoordNumbered(spatialIndex, + spatialCoord.begin(), + spatialCoord.end(), + counts.scount.begin(), + counts.scount.end()); + + Accumulator value(0); + size_t filterCoordCount + = CoordCount(counts.filterCount.begin(), counts.filterCount.end()); + for(size_t cin = 0; cin < counts.cinCount; cin++) + for(size_t filterIndex = 0; filterIndex < filterCoordCount; + filterIndex++) + { + + std::vector filterCoord(counts.filterCount.size(), 0); + CoordNumbered(filterIndex, + filterCoord.begin(), + filterCoord.end(), + counts.filterCount.begin(), + counts.filterCount.end()); + + // Save coordinates from the looop and compute memeory index + // Each component stores in appropriate memory order + std::vector aCoord(activationTensor.dimensions(), 0); + std::vector bCoord(weightTensor.dimensions(), 0); + + aCoord[formatA.batchPosition()] = n; + aCoord[formatA.channelPosition()] = cin; + for(auto i = 0; i < formatA.spatialPositions().size(); i++) + aCoord[formatA.spatialPositions()[i]] = spatialCoord[i]; + + // add filters to address calc, if they have non-unit strides: + for(int fi = 0; fi < counts.filterCount.size(); fi++) + { + auto fp = formatA.filterPositions()[fi]; + if(fp != ConvolutionProblem::InvalidPos) + aCoord[fp] = filterCoord[fi]; + } + + bCoord[formatB.weights().coutPosition()] = cout; + bCoord[formatB.weights().cinPosition()] = cin; + for(int fi = 0; fi < counts.filterCount.size(); fi++) + { + auto fp = formatB.weights().filterPositions()[fi]; + if(fp != ConvolutionProblem::InvalidPos) + bCoord[fp] = filterCoord[fi]; + } + + auto aIndex = activationTensor.index(aCoord) - padShift; + bool inZeroPads = std::accumulate( + problem.aZeroPad().begin(), + problem.aZeroPad().end(), + false, + [&](bool ret, const ContractionProblem::ZeroPad& zp) { + return ret + || inZeroPad(problem, + zp, + activationTensor, + aCoord, + aCoord[zp.boundPos]); + }); + + auto aVal = inZeroPads + ? static_cast(0.0) + : Transform::Input( + inputs.a[aIndex], false); + + auto bIndex = weightTensor.index(bCoord); + auto bVal = Transform::Input( + inputs.b[bIndex], false); + + if(db2) + { + std::cout << " n,cin,spatialCoord,cout=" << n << "," << cin + << "," + << "," << cout << "," + << " spatialCoord[2,1,0]=" << spatialCoord[2] + << "," << spatialCoord[1] << "," + << spatialCoord[0] + << " filterCoord[2,1,0]=" << filterCoord[2] << "," + << filterCoord[1] << "," << filterCoord[0] + << " aIndex=" << aIndex << " bIndex=" << bIndex + << " aVal=" << aVal << " bVal=" << bVal << "\n"; + } + value += multiply(aVal, bVal); + } + std::vector dCoord(outputTensor.dimensions(), 0); + dCoord[formatD.activation().batchPosition()] = n; + dCoord[formatD.activation().channelPosition()] = cout; + for(auto i = 0; i < formatD.activation().spatialPositions().size(); i++) + dCoord[formatD.activation().spatialPositions()[i]] + = spatialCoord[i]; + + auto dIndex = outputTensor.index(dCoord); + if(db1) + { + std::cout << "output: [n,spatialCoord,cout=" << n << "," + << "," << cout << "]" + << " spatialCoord[2,1,0]=" << spatialCoord[2] << "," + << spatialCoord[1] << "," << spatialCoord[0] + << " dIndex=" << dIndex << " value=" << value << "\n"; + } + inputs.d[dIndex] = static_cast( + multiply(inputs.alpha, value)); + } + } + + void SolveCPUConvolution(ConvolutionProblem const& convProblem, + ContractionProblem const& problem, + ContractionInputs& inputs) + { + // std::cout << "SolveCPUConvolution:" << convProblem << " (vs " << problem << + // ")\n"; + + if(problem.a().dataType() == DataType::Float + && problem.b().dataType() == DataType::Float + && problem.c().dataType() == DataType::Float + && problem.d().dataType() == DataType::Float) + { + auto& typedInputs = dynamic_cast&>(inputs); + return ReferenceSolution>::SolveCPUConvolution( + convProblem, problem, typedInputs); + } + else + { + throw std::runtime_error("Data type not implemented for conv-vs-contract."); + } + } + } // namespace Client + } // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/ReferenceValidator.cpp b/tensilelite/Tensile/Source/client/source/ReferenceValidator.cpp new file mode 100644 index 0000000000..c95718e598 --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/ReferenceValidator.cpp @@ -0,0 +1,595 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "ReferenceValidator.hpp" +#include "DataInitializationTyped.hpp" +#include "ResultComparison.hpp" +#include "ResultReporter.hpp" + +#include "Reference.hpp" + +#include +#include + +#include + +namespace Tensile +{ + namespace Client + { + ReferenceValidator::ReferenceValidator(po::variables_map const& args, + std::shared_ptr dataInit) + : m_dataInit(dataInit) + { + m_elementsToValidate = args["num-elements-to-validate"].as(); + m_printValids = args["print-valids"].as(); + m_printMax = args["print-max"].as(); + + m_printTensorA = args["print-tensor-a"].as(); + m_printTensorB = args["print-tensor-b"].as(); + m_printTensorC = args["print-tensor-c"].as(); + m_printTensorD = args["print-tensor-d"].as(); + m_printTensorRef = args["print-tensor-ref"].as(); + + m_printAny = m_printTensorA || m_printTensorB || m_printTensorC || m_printTensorD + || m_printTensorRef; + + m_convolutionVsContraction = args["convolution-vs-contraction"].as(); + if(args.count("convolution-identifier")) + m_convolutionProblem.FromIdentifier( + args["convolution-identifier"].as()); + + m_enabled = m_elementsToValidate != 0 || m_printAny; + } + + bool ReferenceValidator::needMoreBenchmarkRuns() const + { + if(m_enabled && m_numBenchmarkRuns == 0) + return true; + + return false; + } + + void ReferenceValidator::preBenchmarkRun() {} + + void ReferenceValidator::postBenchmarkRun() + { + m_numBenchmarkRuns++; + } + + void ReferenceValidator::preProblem(ContractionProblem const& problem) + { + if(m_enabled) + { + m_problem = problem; + m_referenceInputs = m_dataInit->prepareCPUInputs(problem); + m_validationStride = 1; + if(m_elementsToValidate > 0 + && m_elementsToValidate < problem.d().totalLogicalElements()) + m_validationStride + = NextPrime(problem.d().totalAllocatedElements() / m_elementsToValidate); + + SolveCPU(problem, *m_referenceInputs, m_validationStride); + + if(m_convolutionVsContraction) + { + + SolveCPUConvolution( + m_convolutionProblem, problem, *(m_dataInit->cpuConvInputs())); + // std::cout << "ValidateConv--Start\n"; + m_errorInConvolutionVsContraction = validateSolution( + m_dataInit->cpuConvInputs()); // validate conv against reference + // TODO - print problem dimensions?? + std::cout << m_convolutionProblem << " vs " << problem.operationIdentifier() + << " : "; + if(m_errorInConvolutionVsContraction) + { + std::cout << "FAILED_CONV"; + } + else + { + std::cout << "PASSED_CONV"; + } + std::cout << "\n"; + } + } + } + + void ReferenceValidator::preSolution(ContractionSolution const& solution) + { + m_validatedSolution = false; + m_errorInSolution = false; + } + + bool ReferenceValidator::needMoreRunsInSolution() const + { + if(m_enabled && !m_validatedSolution) + return true; + + return false; + } + + size_t ReferenceValidator::numWarmupRuns() + { + if(m_enabled && !m_validatedSolution) + return 1; + + return 0; + } + + void ReferenceValidator::setNumWarmupRuns(size_t count) {} + + void ReferenceValidator::preWarmup() {} + + void ReferenceValidator::postWarmup() {} + + template + bool ReferenceValidator::validateSolutionCast(std::shared_ptr inputs) + { + auto const& typedReference = dynamic_cast(*m_referenceInputs); + auto const& typedResult = dynamic_cast(*inputs); + + auto rv = validateTyped(typedReference, typedResult); + + if(0 and inputs == m_dataInit->cpuConvInputs()) + { + m_reporter->logTensor( + LogLevel::Verbose, "Aval-conv", typedResult.a, m_problem.a(), nullptr); + m_reporter->logTensor( + LogLevel::Verbose, "Bval-conv", typedResult.b, m_problem.b(), nullptr); + m_reporter->logTensor( + LogLevel::Verbose, "Dval-conv", typedResult.d, m_problem.d(), nullptr); + m_reporter->logTensor(LogLevel::Verbose, + "Bval-contraction", + typedReference.b, + m_problem.b(), + nullptr); + m_reporter->logTensor(LogLevel::Verbose, + "Dval-contraction", + typedReference.d, + m_problem.d(), + nullptr); + } + + return rv; + } + + bool ReferenceValidator::validateSolution(std::shared_ptr inputs) + { + // retreive alpha/beta type set via setAlpha/BetaType() + auto alphaType = m_problem.alphaType(); + auto betaType = m_problem.betaType(); + + // Backward-compatible: when setAlpha/BetaType() wasn't called, use the old way + // Could remove after rocBLAS is updated + if(alphaType == DataType::None) + { + alphaType = m_problem.a().dataType() == DataType::BFloat16 + ? DataType::Float + : m_problem.d().dataType(); + } + if(betaType == DataType::None) + { + betaType = alphaType; + } + + auto contractionInputsTypeId = ContractionInputs::TypeId(m_problem.a().dataType(), + m_problem.b().dataType(), + m_problem.c().dataType(), + m_problem.d().dataType(), + alphaType, + betaType); + + switch(contractionInputsTypeId) + { + case ManagedContractionInputs_S_S_S::TypeId(): + { + return validateSolutionCast(inputs); + } + case ManagedContractionInputs_D_D_D::TypeId(): + { + return validateSolutionCast(inputs); + } + case ManagedContractionInputs_C_C_C::TypeId(): + { + return validateSolutionCast(inputs); + } + case ManagedContractionInputs_Z_Z_Z::TypeId(): + { + return validateSolutionCast(inputs); + } +#ifdef TENSILE_USE_HALF + case ManagedContractionInputs_H_H_H::TypeId(): + { + return validateSolutionCast(inputs); + } + case ManagedContractionInputs_H_H_S::TypeId(): + { + return validateSolutionCast(inputs); + } + case ManagedContractionInputs_H_S_S::TypeId(): + { + return validateSolutionCast(inputs); + } +#endif // TENSILE_USE_HALF + case ManagedContractionInputs_I8x4_I32_I32::TypeId(): + { + return validateSolutionCast(inputs); + } + case ManagedContractionInputs_I32_I32_I32::TypeId(): + { + return validateSolutionCast(inputs); + } + case ManagedContractionInputs_I8_I8_I32::TypeId(): + { + return validateSolutionCast(inputs); + } + case ManagedContractionInputs_I8_I32_I32::TypeId(): + { + return validateSolutionCast(inputs); + } +#ifdef TENSILE_USE_BF16 + case ManagedContractionInputs_B_B_S::TypeId(): + { + return validateSolutionCast(inputs); + } + case ManagedContractionInputs_B_S_S::TypeId(): + { + return validateSolutionCast(inputs); + } +#endif // TENSILE_USE_BF16 + default:; + } + + throw std::runtime_error("Data type not implemented."); + } + + void ReferenceValidator::validateWarmups(std::shared_ptr inputs, + TimingEvents const& startEvents, + TimingEvents const& stopEvents) + { + if(m_enabled && !m_validatedSolution) + { + validateSolution(inputs); + m_validatedSolution = true; + } + } + + template + bool ReferenceValidator::validateTyped(ManagedInputs const& reference, + ManagedInputs const& result) + { + bool rv = false; + if(!m_enabled) + return rv; + + if(m_printAny) + printTensorsTyped(reference, result); + + if(m_elementsToValidate != 0) + { + // RMSComparison compareValid(1e-7, + // m_printMax > 0); + PointwiseComparison compareValid( + m_printValids, m_printMax, m_printMax > 0); + InvalidComparison compareInvalid(m_printMax, + m_printMax > 0); + rv = checkResultsTyped(reference, result, compareValid, compareInvalid); + } + + return rv; + } + + void ReferenceValidator::allocateResultBuffer(size_t bytes) + { + if(m_cpuResultBufferSize == bytes) + return; + m_cpuResultBuffer.reset(); + + uint8_t* buffer; + HIP_CHECK_EXC(hipHostMalloc(&buffer, bytes, 0)); + m_cpuResultBuffer.reset(buffer, hipFree); + m_cpuResultBufferSize = bytes; + } + + template + void ReferenceValidator::printTensorsTyped(ManagedInputs const& reference, + ManagedInputs const& result) + { + size_t requiredBufferSize = 0; + + std::cout << "reference alpha: " << reference.alpha << ", beta: " << reference.beta + << std::endl; + std::cout << "result alpha: " << result.alpha << ", beta: " << result.beta + << std::endl; + + if(m_printTensorA) + requiredBufferSize + = std::max(requiredBufferSize, m_problem.a().totalAllocatedBytes()); + if(m_printTensorB) + requiredBufferSize + = std::max(requiredBufferSize, m_problem.b().totalAllocatedBytes()); + if(m_printTensorC) + requiredBufferSize + = std::max(requiredBufferSize, m_problem.c().totalAllocatedBytes()); + if(m_printTensorD) + requiredBufferSize + = std::max(requiredBufferSize, m_problem.d().totalAllocatedBytes()); + if(m_printTensorRef) + requiredBufferSize + = std::max(requiredBufferSize, m_problem.d().totalAllocatedBytes()); + + if(m_cpuResultBufferSize < requiredBufferSize) + allocateResultBuffer(requiredBufferSize); + + if(m_printTensorA) + { + HIP_CHECK_EXC(hipMemcpy(m_cpuResultBuffer.get(), + result.a, + m_problem.a().totalAllocatedBytes(), + hipMemcpyDeviceToHost)); + auto const* buffer = reinterpret_cast( + m_cpuResultBuffer.get()); + + m_reporter->logTensor(LogLevel::Verbose, "A", buffer, m_problem.a(), result.a); + } + + if(m_printTensorB) + { + HIP_CHECK_EXC(hipMemcpy(m_cpuResultBuffer.get(), + result.b, + m_problem.b().totalAllocatedBytes(), + hipMemcpyDeviceToHost)); + auto const* buffer = reinterpret_cast( + m_cpuResultBuffer.get()); + + m_reporter->logTensor(LogLevel::Verbose, "B", buffer, m_problem.b(), result.b); + } + + if(result.c == result.d && (m_printTensorC || m_printTensorD)) + { + // If the pointers are the same, only print the buffer once. + HIP_CHECK_EXC(hipMemcpy(m_cpuResultBuffer.get(), + result.c, + m_problem.c().totalAllocatedBytes(), + hipMemcpyDeviceToHost)); + auto const* buffer = reinterpret_cast( + m_cpuResultBuffer.get()); + + m_reporter->logTensor(LogLevel::Verbose, "C_D", buffer, m_problem.c(), result.c); + } + else + { + if(m_printTensorC) + { + HIP_CHECK_EXC(hipMemcpy(m_cpuResultBuffer.get(), + result.c, + m_problem.c().totalAllocatedBytes(), + hipMemcpyDeviceToHost)); + auto const* buffer = reinterpret_cast( + m_cpuResultBuffer.get()); + + m_reporter->logTensor(LogLevel::Verbose, "C", buffer, m_problem.c(), result.c); + } + + if(m_printTensorD) + { + HIP_CHECK_EXC(hipMemcpy(m_cpuResultBuffer.get(), + result.d, + m_problem.d().totalAllocatedBytes(), + hipMemcpyDeviceToHost)); + auto const* buffer = reinterpret_cast( + m_cpuResultBuffer.get()); + + m_reporter->logTensor(LogLevel::Verbose, "D", buffer, m_problem.d(), result.d); + } + } + + if(m_printTensorRef) + { + m_reporter->logTensor( + LogLevel::Verbose, "Ref", reference.d, m_problem.d(), reference.d); + } + } + + template + bool ReferenceValidator::checkResultsTyped(ManagedInputs const& reference, + ManagedInputs const& result, + CompareValid& compareValid, + CompareInvalid& compareInvalid) + { + using Type = typename ManagedInputs::DType; + auto const& tensor = m_problem.d(); + + size_t elementsToCopy = tensor.totalAllocatedElements(); + size_t elementsOffsetToCopy = 0; + size_t elementsBeforeData = 0; + size_t elementsAfterData = 0; + + BoundsCheckMode boundsCheck = m_dataInit->getCurBoundsCheck(); + if(boundsCheck == BoundsCheckMode::NaN) + elementsToCopy = result.dElements; + size_t bytesToCopy = elementsToCopy * sizeof(Type); + + if(m_cpuResultBufferSize < bytesToCopy) + allocateResultBuffer(bytesToCopy); + + if(boundsCheck == BoundsCheckMode::GuardPageBack) + elementsOffsetToCopy = result.dElements - tensor.totalAllocatedElements(); + + auto copykind = result.gpu ? hipMemcpyDeviceToHost : hipMemcpyHostToHost; + + HIP_CHECK_EXC(hipMemcpy(m_cpuResultBuffer.get(), + result.managedD.get() + elementsOffsetToCopy, + bytesToCopy, + copykind)); + + if(boundsCheck == BoundsCheckMode::NaN) + { + elementsBeforeData = result.d - result.managedD.get(); + elementsAfterData + = elementsToCopy - (tensor.totalAllocatedElements() + elementsBeforeData); + } + // If there was extra data allocated before the tensor to do bounds + // checking, resultBuffer is the whole allocation, while resultData + // points directly to the result. + Type const* resultBuffer = reinterpret_cast(m_cpuResultBuffer.get()); + Type const* resultData = resultBuffer + elementsBeforeData; + Type const* resultAfterData = resultData + tensor.totalAllocatedElements(); + + size_t boundsCheckElements = 0; + + for(ptrdiff_t i = 0; i < elementsBeforeData; i++) + { + boundsCheckElements++; + compareInvalid.before(resultBuffer[i], i, elementsBeforeData); + } + + if(m_validationStride == 1) + { + std::vector coord(tensor.dimensions()); + size_t outerCount = CoordCount(tensor.sizes().begin() + 1, tensor.sizes().end()); + + size_t prevBaseIndex = 0; + const size_t innerDimSize = tensor.sizes()[0]; + const size_t initialStride = tensor.strides()[0]; + + for(size_t i = 0; i < outerCount; i++) + { + CoordNumbered(i, + coord.begin() + 1, + coord.end(), + tensor.sizes().begin() + 1, + tensor.sizes().end()); + size_t baseElemIndex = tensor.index(coord); + + if(boundsCheck == BoundsCheckMode::NaN && baseElemIndex != 0 + && baseElemIndex != prevBaseIndex + innerDimSize) + { + for(auto innerIndex = prevBaseIndex + innerDimSize; + innerIndex < baseElemIndex; + innerIndex++) + { + compareInvalid.inside( + resultData[innerIndex], innerIndex, baseElemIndex); + } + } + + prevBaseIndex = baseElemIndex; + + for(size_t j = 0; j < innerDimSize; j++) + { + size_t elemIndex = baseElemIndex + (j * initialStride); + + Type referenceValue = reference.d[elemIndex]; + Type resultValue = resultData[elemIndex]; + + compareValid( + referenceValue, resultValue, elemIndex, (i * tensor.sizes()[0]) + j); + } + } + } + else + { + std::vector coord(tensor.dimensions()); + for(size_t elemNumber = 0; elemNumber < tensor.totalLogicalElements(); + elemNumber += m_validationStride) + { + CoordNumbered(elemNumber, + coord.begin(), + coord.end(), + tensor.sizes().begin(), + tensor.sizes().end()); + size_t elemIndex = tensor.index(coord); + + Type referenceValue = reference.d[elemIndex]; + Type resultValue = resultData[elemIndex]; + + compareValid(referenceValue, resultValue, elemIndex, elemNumber); + } + } + + for(ptrdiff_t i = 0; i < elementsAfterData; i++) + { + compareInvalid.after(resultAfterData[i], i, elementsAfterData); + } + + if(boundsCheckElements > 0) + std::cout << "Performed bounds check on " << boundsCheckElements << " elements (" + << elementsBeforeData << " before data)" << std::endl; + + compareValid.report(); + compareInvalid.report(); + + if(compareValid.error() || compareInvalid.error()) + { + m_errorInSolution = true; + m_error = true; + + return true; + } + + return false; + } + + void ReferenceValidator::postSolution() + { + if(m_enabled && !m_validatedSolution) + return; + + if(m_elementsToValidate != 0) + { + if(m_errorInConvolutionVsContraction) + { + m_errorsReported++; + m_reporter->report(ResultKey::Validation, "FAILED_CONV"); + } + else if(m_errorInSolution) + { + m_errorsReported++; + m_reporter->report(ResultKey::Validation, "FAILED"); + } + else + m_reporter->report(ResultKey::Validation, "PASSED"); + } + else + { + m_reporter->report(ResultKey::Validation, "NO_CHECK"); + } + + m_errorInSolution = false; + } + + void ReferenceValidator::postProblem() {} + + void ReferenceValidator::finalizeReport() {} + + int ReferenceValidator::error() const + { + return m_errorsReported; + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/ResultFileReporter.cpp b/tensilelite/Tensile/Source/client/source/ResultFileReporter.cpp new file mode 100644 index 0000000000..a4c87c6af6 --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/ResultFileReporter.cpp @@ -0,0 +1,276 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include + +#include + +namespace Tensile +{ + namespace Client + { + std::shared_ptr + ResultFileReporter::Default(po::variables_map const& args) + { + return std::make_shared( + args["results-file"].as(), + args["csv-export-extra-cols"].as(), + args["csv-merge-same-problems"].as(), + args["performance-metric"].as()); + } + + ResultFileReporter::ResultFileReporter(std::string const& filename, + bool exportExtraCols, + bool mergeSameProblems, + PerformanceMetric performanceMetric) + : m_output(filename) + , m_extraCol(exportExtraCols) + , m_mergeSameProblems(mergeSameProblems) + , m_performanceMetric(performanceMetric) + { + if(m_performanceMetric == PerformanceMetric::CUEfficiency) + m_output.setHeaderForKey(ResultKey::ProblemIndex, "GFlopsPerCU"); + else // Default to 'DeviceEfficiency' benchmarking if CUEfficiency not specified + m_output.setHeaderForKey(ResultKey::ProblemIndex, "GFlops"); + } + + template + void ResultFileReporter::reportValue(std::string const& key, T const& value) + { + std::string valueStr = boost::lexical_cast(value); + + if(key == ResultKey::Validation) + { + if(valueStr != "PASSED" && valueStr != "NO_CHECK") + { + m_output.setValueForKey(m_solutionName, -1.0); + m_invalidSolution = true; + } + } + else if(key == ResultKey::SolutionName) + { + m_solutionName = valueStr; + m_output.setHeaderForKey(valueStr, valueStr); + } + else if(key == ResultKey::TimeUS) + { + // cascade from BenchmarkTimer, Time-US first + ++m_currSolutionIdx; + if(!m_invalidSolution) + { + double timeUS = std::stod(valueStr); + if(m_fasterTimeUS < 0 || m_fasterTimeUS > timeUS) + { + m_fasterTimeUS = timeUS; + } + } + } + else if((key == ResultKey::SpeedGFlops + && m_performanceMetric == PerformanceMetric::DeviceEfficiency) + || (key == ResultKey::SpeedGFlopsPerCu + && m_performanceMetric == PerformanceMetric::CUEfficiency)) + { + // cascade from BenchmarkTimer, SpeedGFlops or SpeedGFlopsPerCU second + if(!m_invalidSolution) + { + m_output.setValueForKey(m_solutionName, value); + + double gflops = std::stod(valueStr); + if(m_fastestGflops < gflops) + { + m_winnerSolution = m_solutionName; + m_winnerSolutionIdx = m_currSolutionIdx; + m_fastestGflops = gflops; + } + } + } + else + { + m_output.setValueForKey(key, value); + } + } + + void ResultFileReporter::reportValue_string(std::string const& key, + std::string const& value) + { + reportValue(key, value); + } + + void ResultFileReporter::reportValue_uint(std::string const& key, uint64_t value) + { + reportValue(key, value); + } + + void ResultFileReporter::reportValue_int(std::string const& key, int64_t value) + { + reportValue(key, value); + } + + void ResultFileReporter::reportValue_double(std::string const& key, double value) + { + reportValue(key, value); + } + + void ResultFileReporter::reportValue_sizes(std::string const& key, + std::vector const& value) + { + if(key == ResultKey::ProblemSizes) + { + for(size_t i = 0; i < value.size(); i++) + { + std::string key = concatenate("Size", static_cast('I' + i)); + m_output.setHeaderForKey(key, key); + m_output.setValueForKey(key, value[i]); + } + + // Values for these come separately. + m_output.setHeaderForKey(ResultKey::LDD, "LDD"); + m_output.setHeaderForKey(ResultKey::LDC, "LDC"); + m_output.setHeaderForKey(ResultKey::LDA, "LDA"); + m_output.setHeaderForKey(ResultKey::LDB, "LDB"); + m_output.setHeaderForKey(ResultKey::TotalFlops, "TotalFlops"); + if(m_extraCol) + { + m_output.setHeaderForKey(ResultKey::FastestGFlops, "WinnerGFlops"); + m_output.setHeaderForKey(ResultKey::TimeUS, "WinnerTimeUS"); + m_output.setHeaderForKey(ResultKey::SolutionWinnerIdx, "WinnerIdx"); + m_output.setHeaderForKey(ResultKey::SolutionWinner, "WinnerName"); + } + } + } + + void ResultFileReporter::mergeRow(std::unordered_map& newRow) + { + m_currProbID = std::stoull(newRow[ResultKey::ProblemIndex]); + if(m_probMap.count(m_currProbID) == 0) + { + m_probMap[m_currProbID] = newRow; + return; + } + + auto& oldRow = m_probMap[m_currProbID]; + for(auto& oldRowIter : oldRow) + { + const std::string& key = oldRowIter.first; + if(key.compare(ResultKey::ProblemIndex) == 0 + || key.find("Size") != std::string::npos || key.compare(ResultKey::LDD) == 0 + || key.compare(ResultKey::LDC) == 0 || key.compare(ResultKey::LDA) == 0 + || key.compare(ResultKey::LDB) == 0 || key.compare(ResultKey::TotalFlops) == 0) + { + // these data should be the same for same problem + assert(oldRowIter.second == newRow[key]); + } + else if(key.compare(ResultKey::FastestGFlops) == 0) + { + // if new row is better, update, dummy guard for -1 and empty str + int64_t oldFastest + = (oldRowIter.second.empty()) ? 0 : std::stoll(oldRowIter.second); + int64_t newFastest = (newRow[key].empty()) ? 0 : std::stoll(newRow[key]); + if(newFastest > oldFastest) + { + oldRow[ResultKey::FastestGFlops] = newRow[ResultKey::FastestGFlops]; + oldRow[ResultKey::TimeUS] = newRow[ResultKey::TimeUS]; + oldRow[ResultKey::SolutionWinnerIdx] = newRow[ResultKey::SolutionWinnerIdx]; + oldRow[ResultKey::SolutionWinner] = newRow[ResultKey::SolutionWinner]; + } + } + else if(key.compare(ResultKey::TimeUS) == 0 + || key.compare(ResultKey::SolutionWinnerIdx) == 0 + || key.compare(ResultKey::SolutionWinner) == 0) + { + // skip, we update these together with FastestGFlops + continue; + } + else + { + // these are gflops for each solution + // if new row is better, update. Dummy guard for -1 and empty str + int64_t oldFastest + = (oldRowIter.second.empty()) ? 0 : std::stoll(oldRowIter.second); + int64_t newFastest = (newRow[key].empty()) ? 0 : std::stoll(newRow[key]); + if(newFastest > oldFastest) + { + oldRow[key] = newRow[key]; + } + } + } + } + + void ResultFileReporter::postProblem() + { + if(m_extraCol) + { + // update winner + m_output.setValueForKey(ResultKey::FastestGFlops, m_fastestGflops); + m_output.setValueForKey(ResultKey::TimeUS, m_fasterTimeUS); + m_output.setValueForKey(ResultKey::SolutionWinnerIdx, m_winnerSolutionIdx); + m_output.setValueForKey(ResultKey::SolutionWinner, m_winnerSolution); + } + // reset + m_winnerSolution = ""; + m_currSolutionIdx = -1; + m_winnerSolutionIdx = -1; + m_fastestGflops = -1.0; + m_fasterTimeUS = -1.0; + + if(!m_mergeSameProblems) + { + m_output.writeCurrentRow(); + } + else + { + std::unordered_map curRow; + m_output.readCurrentRow(curRow); + m_output.clearCurrentRow(); + // for (auto & field : curRow ) + // std::cout << "key:" << field.first << ", value:" << field.second << std::endl; + this->mergeRow(curRow); + } + } + + void ResultFileReporter::postSolution() + { + m_solutionName = ""; + m_invalidSolution = false; + } + + void ResultFileReporter::finalizeReport() + { + if(m_mergeSameProblems) + { + for(auto& probIter : m_probMap) + { + auto& single_row = probIter.second; + for(auto& field : single_row) + { + m_output.setValueForKey(field.first, field.second); + } + m_output.writeCurrentRow(); + } + } + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/ResultReporter.cpp b/tensilelite/Tensile/Source/client/source/ResultReporter.cpp new file mode 100644 index 0000000000..e793f63767 --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/ResultReporter.cpp @@ -0,0 +1,78 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "ResultReporter.hpp" + +namespace Tensile +{ + namespace Client + { + std::string ToString(LogLevel level) + { + switch(level) + { + case LogLevel::Error: + return "Error"; + case LogLevel::Terse: + return "Terse"; + case LogLevel::Normal: + return "Normal"; + case LogLevel::Verbose: + return "Verbose"; + case LogLevel::Debug: + return "Debug"; + + case LogLevel::Count:; + } + + return "Invalid"; + } + + std::ostream& operator<<(std::ostream& stream, LogLevel level) + { + return stream << ToString(level); + } + + std::istream& operator>>(std::istream& stream, LogLevel& level) + { + std::string str; + stream >> str; + + for(int i = 0; i < (int)LogLevel::Count; i++) + { + auto myVal = (LogLevel)(i); + auto myStr = ToString(myVal); + if(str == myStr) + { + level = myVal; + break; + } + } + + return stream; + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/SolutionIterator.cpp b/tensilelite/Tensile/Source/client/source/SolutionIterator.cpp new file mode 100644 index 0000000000..bbc2f570be --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/SolutionIterator.cpp @@ -0,0 +1,264 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "SolutionIterator.hpp" + +#include "ResultReporter.hpp" + +namespace Tensile +{ + namespace Client + { + std::shared_ptr SolutionIterator::Default( + std::shared_ptr> library, + std::shared_ptr hardware, + po::variables_map const& args) + { + bool bestSolution = args["best-solution"].as(); + + if(bestSolution) + { + return std::make_shared(library, hardware); + } + else + { + int firstSolutionIdx = args["solution-start-idx"].as(); + int numSolutions = args["num-solutions"].as(); + + return std::make_shared( + library, + hardware, + firstSolutionIdx, + numSolutions, + AllSolutionsIterator::CreateCriteria(library, hardware, args)); + } + } + + SolutionIterator::SolutionIterator( + std::shared_ptr> library, + std::shared_ptr hardware) + : m_library(library) + , m_hardware(hardware) + { + } + + void SolutionIterator::preProblem(ContractionProblem const& problem) + { + m_problem = problem; + } + + bool SolutionIterator::checkSolution(ContractionSolution const& solution) + { + if(!(*solution.hardwarePredicate)(*m_hardware)) + { + m_reporter->report(ResultKey::Validation, "WRONG_HARDWARE"); + if(m_reporter->logAtLevel(LogLevel::Verbose)) + { + std::ostringstream msg; + solution.hardwarePredicate->debugEval(*m_hardware, msg); + msg << std::endl; + m_reporter->log(LogLevel::Verbose, msg.str()); + } + + return false; + } + + // Test if the persistent kernel is eligible for the current hw and solution + m_problem.checkPersistentKernelEligibility(solution, *m_hardware); + if(!(*solution.problemPredicate)(m_problem)) + { + m_reporter->report(ResultKey::Validation, "DID_NOT_SATISFY_ASSERTS"); + if(m_reporter->logAtLevel(LogLevel::Verbose)) + { + std::ostringstream msg; + solution.problemPredicate->debugEval(m_problem, msg); + msg << std::endl; + m_reporter->log(LogLevel::Verbose, msg.str()); + } + + return false; + } + + return true; + } + + bool SolutionIterator::runCurrentSolution() + { + auto solution = getSolution(); + return checkSolution(*solution); + } + + AllSolutionsIterator::RunCriteria AllSolutionsIterator::CreateCriteria( + std::shared_ptr> library, + std::shared_ptr hardware, + po::variables_map const& args) + { + RunCriteria criteria; + + double granThresh = args["granularity-threshold"].as(); + if(granThresh > 0.0) + { + criteria.push_back([granThresh](ContractionProblem const& problem, + Hardware const& hardware, + ContractionSolution const& solution) { + auto projPerf = solution.projectedPerformance(problem, hardware); + return projPerf.granularities.totalGranularity >= granThresh; + }); + } + return criteria; + } + + AllSolutionsIterator::AllSolutionsIterator( + std::shared_ptr> library, + std::shared_ptr hardware, + int firstSolutionIdx, + int numSolutions, + RunCriteria runCriteria) + : SolutionIterator(library, hardware) + , m_runCriteria(runCriteria) + { + m_firstSolutionIdx = firstSolutionIdx; + + if(m_firstSolutionIdx < 0) + m_firstSolutionIdx = library->solutions.begin()->first; + + if(numSolutions < 0) + { + auto iter = library->solutions.rbegin(); + m_lastSolutionIdx = iter->first; + } + else + { + m_lastSolutionIdx = m_firstSolutionIdx + numSolutions - 1; + } + + m_currentSolutionIdx = m_firstSolutionIdx; + } + + void AllSolutionsIterator::preProblem(ContractionProblem const& problem) + { + SolutionIterator::preProblem(problem); + + m_currentSolutionIdx = m_firstSolutionIdx; + } + + void AllSolutionsIterator::postProblem() {} + + void AllSolutionsIterator::preSolution(ContractionSolution const& solution) + { + { + std::string idx = "-1"; + auto iter = solution.info.find("SolutionIndex"); + if(iter != solution.info.end()) + idx = iter->second; + m_reporter->report(ResultKey::SolutionLibraryIndex, idx); + } + m_reporter->report(ResultKey::SolutionIndex, m_currentSolutionIdx); + m_reporter->report(ResultKey::SolutionProgress, + concatenate(m_currentSolutionIdx, "/", m_lastSolutionIdx)); + } + + void AllSolutionsIterator::postSolution() + { + m_currentSolutionIdx++; + } + + bool AllSolutionsIterator::moreSolutionsInProblem() const + { + return m_currentSolutionIdx <= m_lastSolutionIdx; + } + + std::shared_ptr AllSolutionsIterator::getSolution() + { + auto iter = m_library->solutions.find(m_currentSolutionIdx); + if(iter == m_library->solutions.end()) + return std::shared_ptr(); + + return iter->second; + } + + bool AllSolutionsIterator::runCurrentSolution() + { + auto solution = getSolution(); + + if(!checkSolution(*solution)) + return false; + + for(auto const& criterion : m_runCriteria) + { + if(!criterion(m_problem, *m_hardware, *solution)) + return false; + } + return true; + } + + BestSolutionIterator::BestSolutionIterator( + std::shared_ptr> library, + std::shared_ptr hardware) + : SolutionIterator(library, hardware) + { + } + + void BestSolutionIterator::preProblem(ContractionProblem const& problem) + { + SolutionIterator::preProblem(problem); + + m_currentSolution = m_library->findBestSolution(m_problem, *m_hardware); + m_usedCurrentSolution = false; + } + + void BestSolutionIterator::postProblem() {} + + void BestSolutionIterator::preSolution(ContractionSolution const& solution) + { + { + std::string idx = "-1"; + auto iter = solution.info.find("SolutionIndex"); + if(iter != solution.info.end()) + idx = iter->second; + m_reporter->report(ResultKey::SolutionLibraryIndex, idx); + } + + m_reporter->report(ResultKey::SolutionIndex, 0); + m_reporter->report(ResultKey::SolutionProgress, "1/1"); + } + + void BestSolutionIterator::postSolution() + { + m_usedCurrentSolution = true; + } + + bool BestSolutionIterator::moreSolutionsInProblem() const + { + return !m_usedCurrentSolution; + } + + std::shared_ptr BestSolutionIterator::getSolution() + { + return m_currentSolution; + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/client/source/TimingEvents.cpp b/tensilelite/Tensile/Source/client/source/TimingEvents.cpp new file mode 100644 index 0000000000..b0f8437264 --- /dev/null +++ b/tensilelite/Tensile/Source/client/source/TimingEvents.cpp @@ -0,0 +1,79 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "TimingEvents.hpp" + +#include + +#include + +namespace Tensile +{ + namespace Client + { + TimingEvents::TimingEvents(size_t numInvocations, size_t numKernels) + : m_events(numInvocations) + { + for(auto& vec : m_events) + { + vec.resize(numKernels, nullptr); + + for(auto& event : vec) + HIP_CHECK_EXC(hipEventCreateWithFlags(&event, hipEventDefault)); + } + } + + TimingEvents::~TimingEvents() + { + for(auto& vec : m_events) + { + for(auto& event : vec) + { + if(event) + { + hipEventDestroy(event); + event = nullptr; + } + } + } + } + + std::vector const& TimingEvents::operator[](size_t index) const + { + return m_events[index]; + } + + std::vector> const& TimingEvents::operator*() const + { + return m_events; + } + + std::vector> const* TimingEvents::operator->() const + { + return &m_events; + } + } // namespace Client +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/cmake/FindROCmSMI.cmake b/tensilelite/Tensile/Source/cmake/FindROCmSMI.cmake new file mode 100644 index 0000000000..04d09c3124 --- /dev/null +++ b/tensilelite/Tensile/Source/cmake/FindROCmSMI.cmake @@ -0,0 +1,53 @@ +################################################################################ +# Copyright 2019-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +if(NOT ROCM_ROOT) + if(NOT ROCM_DIR) + set(ROCM_ROOT "/opt/rocm") + else() + set(ROCM_DIR "${ROCM_DIR}/../../..") + endif() +endif() + + +# For some reason the *_DIR variables have inconsistent values between Tensile and rocBLAS. Search various paths. +find_path(ROCM_SMI_ROOT "include/rocm_smi/rocm_smi.h" + PATHS "${ROCM_ROOT}" "${HIP_DIR}/../../../.." "${HIP_DIR}/../../.." + PATH_SUFFIXES "rocm_smi" + ) +mark_as_advanced(ROCM_SMI_ROOT) + +find_library(ROCM_SMI_LIBRARY rocm_smi64 + PATHS "${ROCM_SMI_ROOT}/lib") +mark_as_advanced(ROCM_SMI_LIBRARY) + +include( FindPackageHandleStandardArgs ) +find_package_handle_standard_args( ROCmSMI DEFAULT_MSG ROCM_SMI_LIBRARY ROCM_SMI_ROOT ) + +add_library(rocm_smi SHARED IMPORTED) + +set_target_properties(rocm_smi PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${ROCM_SMI_ROOT}/include" + IMPORTED_LOCATION "${ROCM_SMI_LIBRARY}" + INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${ROCM_SMI_ROOT}/include") + + +#set(rocm_smi_root "${hip_LIB_INSTALL_DIR}/../../rocm_smi") diff --git a/tensilelite/Tensile/Source/lib/CMakeLists.txt b/tensilelite/Tensile/Source/lib/CMakeLists.txt new file mode 100644 index 0000000000..2bf7aa2a24 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/CMakeLists.txt @@ -0,0 +1,166 @@ +################################################################################ +# Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +cmake_minimum_required(VERSION 3.13) + +if(NOT DEFINED TENSILE_ROOT) + get_filename_component(TENSILE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}" DIRECTORY) +endif() +if(NOT DEFINED TENSILE_SCRIPT_ROOT) + set(TENSILE_SCRIPT_ROOT "${TENSILE_ROOT}/Tensile") +endif() + +set(tensile_sources ${tensile_sources} + source/AMDGPU.cpp + source/ArithmeticUnitTypes.cpp + source/ContractionProblem.cpp + source/ContractionSolution.cpp + source/DataTypes.cpp + source/Debug.cpp + source/EmbeddedLibrary.cpp + source/Activation.cpp + source/KernelArguments.cpp + source/KernelLanguageTypes.cpp + source/PerformanceMetricTypes.cpp + source/ScalarValueTypes.cpp + source/TensorDescriptor.cpp + source/TensorOps.cpp + source/Tensile.cpp + source/Utils.cpp + ) + +if(TENSILE_USE_LLVM) + find_package(LLVM 13.0 QUIET CONFIG) + if(NOT LLVM_FOUND) + find_package(LLVM 12.0 QUIET CONFIG) + if(NOT LLVM_FOUND) + find_package(LLVM REQUIRED CONFIG) + endif() + endif() + + set(tensile_sources ${tensile_sources} + source/llvm/YAML.cpp + source/llvm/Loading.cpp + ) +endif() + +if(TENSILE_USE_MSGPACK) + set(tensile_sources ${tensile_sources} + source/msgpack/MessagePack.cpp + ) +endif() + +if(TENSILE_USE_HIP) + set(tensile_sources ${tensile_sources} + source/hip/HipSolutionAdapter.cpp + source/hip/HipHardware.cpp + ) +endif() + +if(TENSILE_USE_OPENCL) + set(tensile_sources ${tensile_sources} + source/ocl/OclHardware.cpp + source/ocl/OclSolutionAdapter.cpp + source/ocl/OclUtils.cpp + ) +endif() + +include_directories ("${CMAKE_CURRENT_SOURCE_DIR}/include") + +add_library (TensileHost STATIC ${tensile_sources}) + +set_target_properties(TensileHost + PROPERTIES + CXX_STANDARD 14 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF) + +target_include_directories(TensileHost INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include") + +if(TENSILE_USE_LLVM OR TENSILE_USE_MSGPACK) + target_compile_definitions(TensileHost PUBLIC TENSILE_DEFAULT_SERIALIZATION) +endif() + +if(TENSILE_USE_MSGPACK) + find_package(msgpack REQUIRED) + target_compile_definitions(TensileHost PUBLIC -DTENSILE_MSGPACK=1) + + if(TARGET msgpackc-cxx) + get_target_property(msgpack_inc msgpackc-cxx INTERFACE_INCLUDE_DIRECTORIES) + elseif(TARGET msgpackc) + get_target_property(msgpack_inc msgpackc INTERFACE_INCLUDE_DIRECTORIES) + endif() + + if(DEFINED msgpack_inc) + # include C++ headers manually + # External header includes included as system files + target_include_directories(TensileHost + SYSTEM PRIVATE $ + ) + endif() +endif() + +if(TENSILE_USE_LLVM) + target_compile_definitions(TensileHost PUBLIC -DTENSILE_YAML=1) + find_library(LLVMObjectYAML_LIBRARY + NAMES LLVMObjectYAML + PATHS ${LLVM_LIBRARY_DIR}) + message("LLVMObjectYAML_LIBRARY: ${LLVMObjectYAML_LIBRARY}") + #Use LLVM shared libs if LLVMObjectYAML static lib cannot be found + if(LLVMObjectYAML_LIBRARY) + target_link_libraries(TensileHost PRIVATE LLVMObjectYAML ) + else() + target_link_libraries(TensileHost PRIVATE LLVM ) + endif() + target_include_directories(TensileHost PRIVATE ${LLVM_INCLUDE_DIRS}) +endif() + +if(TENSILE_STATIC_ONLY) + target_compile_definitions(TensileHost PUBLIC TENSILE_STATIC_ONLY) +endif() + +if(TENSILE_USE_HIP) + target_compile_definitions(TensileHost PUBLIC TENSILE_USE_HIP) + + #add_subdirectory(client) + target_link_libraries( TensileHost PUBLIC ${HSA_LIBRARIES} hip::device ) + + if(WIN32) + target_compile_options( TensileHost PUBLIC -Wno-deprecated-declarations -Wno-ignored-attributes -Wdll-attribute-on-redeclaration -fdelayed-template-parsing ) + endif() +endif() + +if(TENSILE_USE_OPENCL) + find_package(OpenCL REQUIRED) + target_compile_definitions(TensileHost PUBLIC CL_TARGET_OPENCL_VERSION=200) + target_compile_definitions(TensileHost PUBLIC CL_HPP_TARGET_OPENCL_VERSION=200) + target_compile_definitions(TensileHost PUBLIC CL_HPP_ENABLE_EXCEPTIONS) + target_link_libraries(TensileHost PUBLIC ${OPENCL_LIBRARIES}) + target_include_directories(TensileHost PUBLIC ${OPENCL_INCLUDE_DIRS}) +endif() + +if(false) +get_cmake_property(_variableNames VARIABLES) +list (SORT _variableNames) +foreach (_variableName ${_variableNames}) + message(STATUS "${_variableName}=${${_variableName}}") +endforeach() +endif() diff --git a/tensilelite/Tensile/Source/lib/configs/SolutionLibraries/KernelsLiteNavi.yaml b/tensilelite/Tensile/Source/lib/configs/SolutionLibraries/KernelsLiteNavi.yaml new file mode 100644 index 0000000000..d1f60c6b90 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/configs/SolutionLibraries/KernelsLiteNavi.yaml @@ -0,0 +1,14881 @@ +--- +library: + rows: + - library: + map: + Contraction_l_Ailk_Bjlk_Cijk_Dijk: + rows: + - library: + distance: Euclidean + properties: + - {index: 0, type: FreeSizeA} + - {index: 0, type: FreeSizeB} + - {index: 0, type: BoundSize} + table: + - key: [4, 4, 256] + speed: 0.234298 + value: {index: 27, type: Single} + - key: [4, 4, 768] + speed: 0.333103 + value: {index: 27, type: Single} + - key: [4, 4, 1792] + speed: 0.369687 + value: {index: 27, type: Single} + - key: [4, 4, 3328] + speed: 0.393663 + value: {index: 27, type: Single} + - key: [4, 64, 256] + speed: 5.26605 + value: {index: 27, type: Single} + - key: [4, 64, 768] + speed: 5.52945 + value: {index: 27, type: Single} + - key: [4, 64, 1792] + speed: 6.14994 + value: {index: 27, type: Single} + - key: [4, 64, 3328] + speed: 6.58345 + value: {index: 27, type: Single} + - key: [4, 128, 256] + speed: 10.2874 + value: {index: 27, type: Single} + - key: [4, 128, 768] + speed: 13.7519 + value: {index: 27, type: Single} + - key: [4, 128, 1792] + speed: 14.8691 + value: {index: 27, type: Single} + - key: [4, 128, 3328] + speed: 13.1069 + value: {index: 27, type: Single} + - key: [4, 256, 256] + speed: 20.3386 + value: {index: 27, type: Single} + - key: [4, 256, 768] + speed: 27.2216 + value: {index: 27, type: Single} + - key: [4, 256, 1792] + speed: 29.3855 + value: {index: 27, type: Single} + - key: [4, 256, 3328] + speed: 26.0357 + value: {index: 27, type: Single} + - key: [4, 448, 256] + speed: 27.5246 + value: {index: 27, type: Single} + - key: [4, 448, 768] + speed: 38.6258 + value: {index: 27, type: Single} + - key: [4, 448, 1792] + speed: 43.2214 + value: {index: 27, type: Single} + - key: [4, 448, 3328] + speed: 39.6791 + value: {index: 27, type: Single} + - key: [4, 704, 256] + speed: 40.7194 + value: {index: 27, type: Single} + - key: [4, 704, 768] + speed: 58.9811 + value: {index: 27, type: Single} + - key: [4, 704, 1792] + speed: 66.0745 + value: {index: 27, type: Single} + - key: [4, 704, 3328] + speed: 59.9876 + value: {index: 27, type: Single} + - key: [4, 1024, 256] + speed: 57.0793 + value: {index: 27, type: Single} + - key: [4, 1024, 768] + speed: 81.1968 + value: {index: 27, type: Single} + - key: [4, 1024, 1792] + speed: 82.1621 + value: {index: 27, type: Single} + - key: [4, 1024, 3328] + speed: 82.8174 + value: {index: 27, type: Single} + - key: [4, 1408, 256] + speed: 81.0975 + value: {index: 27, type: Single} + - key: [4, 1408, 768] + speed: 115.625 + value: {index: 27, type: Single} + - key: [4, 1408, 1792] + speed: 113.823 + value: {index: 27, type: Single} + - key: [4, 1408, 3328] + speed: 119.296 + value: {index: 27, type: Single} + - key: [4, 1856, 256] + speed: 105.151 + value: {index: 27, type: Single} + - key: [4, 1856, 768] + speed: 150.334 + value: {index: 27, type: Single} + - key: [4, 1856, 1792] + speed: 150.039 + value: {index: 27, type: Single} + - key: [4, 1856, 3328] + speed: 155.64 + value: {index: 27, type: Single} + - key: [4, 2368, 256] + speed: 131.463 + value: {index: 27, type: Single} + - key: [4, 2368, 768] + speed: 186.698 + value: {index: 27, type: Single} + - key: [4, 2368, 1792] + speed: 189.844 + value: {index: 27, type: Single} + - key: [4, 2368, 3328] + speed: 197.195 + value: {index: 27, type: Single} + - key: [4, 2944, 256] + speed: 164.762 + value: {index: 27, type: Single} + - key: [4, 2944, 768] + speed: 219.195 + value: {index: 27, type: Single} + - key: [4, 2944, 1792] + speed: 230.111 + value: {index: 27, type: Single} + - key: [4, 2944, 3328] + speed: 238.101 + value: {index: 27, type: Single} + - key: [4, 3584, 256] + speed: 198.176 + value: {index: 27, type: Single} + - key: [4, 3584, 768] + speed: 261.677 + value: {index: 27, type: Single} + - key: [4, 3584, 1792] + speed: 279.233 + value: {index: 27, type: Single} + - key: [4, 3584, 3328] + speed: 287.789 + value: {index: 27, type: Single} + - key: [4, 4288, 256] + speed: 228.866 + value: {index: 27, type: Single} + - key: [4, 4288, 768] + speed: 292.48 + value: {index: 27, type: Single} + - key: [4, 4288, 1792] + speed: 317.712 + value: {index: 27, type: Single} + - key: [4, 4288, 3328] + speed: 326.382 + value: {index: 27, type: Single} + - key: [4, 5056, 256] + speed: 252.32 + value: {index: 27, type: Single} + - key: [4, 5056, 768] + speed: 332.82 + value: {index: 27, type: Single} + - key: [4, 5056, 1792] + speed: 357.636 + value: {index: 27, type: Single} + - key: [4, 5056, 3328] + speed: 367.706 + value: {index: 27, type: Single} + - key: [4, 5888, 256] + speed: 286.598 + value: {index: 27, type: Single} + - key: [4, 5888, 768] + speed: 362.825 + value: {index: 27, type: Single} + - key: [4, 5888, 1792] + speed: 367.347 + value: {index: 27, type: Single} + - key: [4, 5888, 3328] + speed: 369.067 + value: {index: 27, type: Single} + - key: [4, 6784, 256] + speed: 296.771 + value: {index: 27, type: Single} + - key: [4, 6784, 768] + speed: 346.478 + value: {index: 27, type: Single} + - key: [4, 6784, 1792] + speed: 341.905 + value: {index: 27, type: Single} + - key: [4, 6784, 3328] + speed: 348.523 + value: {index: 27, type: Single} + - key: [64, 4, 256] + speed: 4.09587 + value: {index: 27, type: Single} + - key: [64, 4, 768] + speed: 5.32957 + value: {index: 27, type: Single} + - key: [64, 4, 1792] + speed: 5.9663 + value: {index: 27, type: Single} + - key: [64, 4, 3328] + speed: 6.48688 + value: {index: 27, type: Single} + - key: [64, 64, 256] + speed: 53.2151 + value: {index: 22, type: Single} + - key: [64, 64, 768] + speed: 85.1036 + value: {index: 23, type: Single} + - key: [64, 64, 1792] + speed: 99.9875 + value: {index: 23, type: Single} + - key: [64, 64, 3328] + speed: 102.977 + value: {index: 22, type: Single} + - key: [64, 128, 256] + speed: 136.769 + value: {index: 22, type: Single} + - key: [64, 128, 768] + speed: 179.561 + value: {index: 22, type: Single} + - key: [64, 128, 1792] + speed: 197.189 + value: {index: 22, type: Single} + - key: [64, 128, 3328] + speed: 195.974 + value: {index: 22, type: Single} + - key: [64, 256, 256] + speed: 274.856 + value: {index: 22, type: Single} + - key: [64, 256, 768] + speed: 363.736 + value: {index: 23, type: Single} + - key: [64, 256, 1792] + speed: 386.684 + value: {index: 23, type: Single} + - key: [64, 256, 3328] + speed: 365.664 + value: {index: 23, type: Single} + - key: [64, 448, 256] + speed: 460.869 + value: {index: 26, type: Single} + - key: [64, 448, 768] + speed: 637.903 + value: {index: 22, type: Single} + - key: [64, 448, 1792] + speed: 628.843 + value: {index: 22, type: Single} + - key: [64, 448, 3328] + speed: 645.684 + value: {index: 22, type: Single} + - key: [64, 704, 256] + speed: 707.758 + value: {index: 22, type: Single} + - key: [64, 704, 768] + speed: 945.605 + value: {index: 23, type: Single} + - key: [64, 704, 1792] + speed: 971.447 + value: {index: 22, type: Single} + - key: [64, 704, 3328] + speed: 1013.12 + value: {index: 22, type: Single} + - key: [64, 1024, 256] + speed: 972.029 + value: {index: 26, type: Single} + - key: [64, 1024, 768] + speed: 1255.92 + value: {index: 22, type: Single} + - key: [64, 1024, 1792] + speed: 1370.27 + value: {index: 22, type: Single} + - key: [64, 1024, 3328] + speed: 1411.47 + value: {index: 22, type: Single} + - key: [64, 1408, 256] + speed: 1319.57 + value: {index: 22, type: Single} + - key: [64, 1408, 768] + speed: 1743.01 + value: {index: 23, type: Single} + - key: [64, 1408, 1792] + speed: 1932.57 + value: {index: 22, type: Single} + - key: [64, 1408, 3328] + speed: 2020.18 + value: {index: 22, type: Single} + - key: [64, 1856, 256] + speed: 1732.1 + value: {index: 22, type: Single} + - key: [64, 1856, 768] + speed: 2268.01 + value: {index: 23, type: Single} + - key: [64, 1856, 1792] + speed: 2514.04 + value: {index: 23, type: Single} + - key: [64, 1856, 3328] + speed: 2595.61 + value: {index: 23, type: Single} + - key: [64, 2368, 256] + speed: 1775.42 + value: {index: 26, type: Single} + - key: [64, 2368, 768] + speed: 2270.57 + value: {index: 23, type: Single} + - key: [64, 2368, 1792] + speed: 2472.19 + value: {index: 23, type: Single} + - key: [64, 2368, 3328] + speed: 2540.57 + value: {index: 23, type: Single} + - key: [64, 2944, 256] + speed: 2170.48 + value: {index: 22, type: Single} + - key: [64, 2944, 768] + speed: 2778.72 + value: {index: 22, type: Single} + - key: [64, 2944, 1792] + speed: 3073.52 + value: {index: 22, type: Single} + - key: [64, 2944, 3328] + speed: 3135.16 + value: {index: 23, type: Single} + - key: [64, 3584, 256] + speed: 2557.11 + value: {index: 24, type: Single} + - key: [64, 3584, 768] + speed: 3344.74 + value: {index: 24, type: Single} + - key: [64, 3584, 1792] + speed: 3686.99 + value: {index: 22, type: Single} + - key: [64, 3584, 3328] + speed: 3785.86 + value: {index: 22, type: Single} + - key: [64, 4288, 256] + speed: 2285.3 + value: {index: 24, type: Single} + - key: [64, 4288, 768] + speed: 2743.71 + value: {index: 24, type: Single} + - key: [64, 4288, 1792] + speed: 2895.28 + value: {index: 24, type: Single} + - key: [64, 4288, 3328] + speed: 2946.08 + value: {index: 24, type: Single} + - key: [64, 5056, 256] + speed: 2707.68 + value: {index: 24, type: Single} + - key: [64, 5056, 768] + speed: 3219.62 + value: {index: 24, type: Single} + - key: [64, 5056, 1792] + speed: 3410.86 + value: {index: 24, type: Single} + - key: [64, 5056, 3328] + speed: 3464.63 + value: {index: 24, type: Single} + - key: [64, 5888, 256] + speed: 3191.91 + value: {index: 24, type: Single} + - key: [64, 5888, 768] + speed: 3735.08 + value: {index: 24, type: Single} + - key: [64, 5888, 1792] + speed: 3958.34 + value: {index: 24, type: Single} + - key: [64, 5888, 3328] + speed: 4038.61 + value: {index: 24, type: Single} + - key: [64, 6784, 256] + speed: 3624.33 + value: {index: 24, type: Single} + - key: [64, 6784, 768] + speed: 4315.83 + value: {index: 24, type: Single} + - key: [64, 6784, 1792] + speed: 4558.73 + value: {index: 24, type: Single} + - key: [64, 6784, 3328] + speed: 4645.42 + value: {index: 24, type: Single} + - key: [128, 4, 256] + speed: 10.2874 + value: {index: 27, type: Single} + - key: [128, 4, 768] + speed: 13.507 + value: {index: 27, type: Single} + - key: [128, 4, 1792] + speed: 14.5718 + value: {index: 27, type: Single} + - key: [128, 4, 3328] + speed: 12.8938 + value: {index: 27, type: Single} + - key: [128, 64, 256] + speed: 137.428 + value: {index: 26, type: Single} + - key: [128, 64, 768] + speed: 183.438 + value: {index: 23, type: Single} + - key: [128, 64, 1792] + speed: 200.582 + value: {index: 23, type: Single} + - key: [128, 64, 3328] + speed: 195.35 + value: {index: 23, type: Single} + - key: [128, 128, 256] + speed: 274.865 + value: {index: 26, type: Single} + - key: [128, 128, 768] + speed: 357.611 + value: {index: 23, type: Single} + - key: [128, 128, 1792] + speed: 388.961 + value: {index: 23, type: Single} + - key: [128, 128, 3328] + speed: 358.016 + value: {index: 23, type: Single} + - key: [128, 256, 256] + speed: 539.252 + value: {index: 23, type: Single} + - key: [128, 256, 768] + speed: 712.217 + value: {index: 23, type: Single} + - key: [128, 256, 1792] + speed: 747.129 + value: {index: 23, type: Single} + - key: [128, 256, 3328] + speed: 742.016 + value: {index: 23, type: Single} + - key: [128, 448, 256] + speed: 865.391 + value: {index: 22, type: Single} + - key: [128, 448, 768] + speed: 1218.3 + value: {index: 23, type: Single} + - key: [128, 448, 1792] + speed: 1221.15 + value: {index: 22, type: Single} + - key: [128, 448, 3328] + speed: 1267.23 + value: {index: 23, type: Single} + - key: [128, 704, 256] + speed: 1390.27 + value: {index: 26, type: Single} + - key: [128, 704, 768] + speed: 1850.01 + value: {index: 23, type: Single} + - key: [128, 704, 1792] + speed: 1948.1 + value: {index: 23, type: Single} + - key: [128, 704, 3328] + speed: 2011.14 + value: {index: 23, type: Single} + - key: [128, 1024, 256] + speed: 1797.53 + value: {index: 26, type: Single} + - key: [128, 1024, 768] + speed: 2257.35 + value: {index: 22, type: Single} + - key: [128, 1024, 1792] + speed: 2536.65 + value: {index: 22, type: Single} + - key: [128, 1024, 3328] + speed: 2587.27 + value: {index: 22, type: Single} + - key: [128, 1408, 256] + speed: 2264.85 + value: {index: 26, type: Single} + - key: [128, 1408, 768] + speed: 2680.79 + value: {index: 24, type: Single} + - key: [128, 1408, 1792] + speed: 2898.84 + value: {index: 24, type: Single} + - key: [128, 1408, 3328] + speed: 2996.63 + value: {index: 24, type: Single} + - key: [128, 1856, 256] + speed: 2727.6 + value: {index: 24, type: Single} + - key: [128, 1856, 768] + speed: 3483.77 + value: {index: 24, type: Single} + - key: [128, 1856, 1792] + speed: 3828.85 + value: {index: 24, type: Single} + - key: [128, 1856, 3328] + speed: 3948.64 + value: {index: 22, type: Single} + - key: [128, 2368, 256] + speed: 2561.13 + value: {index: 24, type: Single} + - key: [128, 2368, 768] + speed: 3030.37 + value: {index: 24, type: Single} + - key: [128, 2368, 1792] + speed: 3185.27 + value: {index: 24, type: Single} + - key: [128, 2368, 3328] + speed: 3251.55 + value: {index: 24, type: Single} + - key: [128, 2944, 256] + speed: 3199.74 + value: {index: 24, type: Single} + - key: [128, 2944, 768] + speed: 3763.86 + value: {index: 24, type: Single} + - key: [128, 2944, 1792] + speed: 3961.78 + value: {index: 24, type: Single} + - key: [128, 2944, 3328] + speed: 4041.5 + value: {index: 24, type: Single} + - key: [128, 3584, 256] + speed: 3774.77 + value: {index: 24, type: Single} + - key: [128, 3584, 768] + speed: 4465.93 + value: {index: 24, type: Single} + - key: [128, 3584, 1792] + speed: 4795.95 + value: {index: 24, type: Single} + - key: [128, 3584, 3328] + speed: 4888.58 + value: {index: 24, type: Single} + - key: [128, 4288, 256] + speed: 3430.07 + value: {index: 24, type: Single} + - key: [128, 4288, 768] + speed: 3878.99 + value: {index: 24, type: Single} + - key: [128, 4288, 1792] + speed: 4032.12 + value: {index: 24, type: Single} + - key: [128, 4288, 3328] + speed: 4085.27 + value: {index: 24, type: Single} + - key: [128, 5056, 256] + speed: 3965.56 + value: {index: 24, type: Single} + - key: [128, 5056, 768] + speed: 4512.19 + value: {index: 24, type: Single} + - key: [128, 5056, 1792] + speed: 4734.16 + value: {index: 24, type: Single} + - key: [128, 5056, 3328] + speed: 4810.58 + value: {index: 24, type: Single} + - key: [128, 5888, 256] + speed: 4482.92 + value: {index: 24, type: Single} + - key: [128, 5888, 768] + speed: 5268.91 + value: {index: 24, type: Single} + - key: [128, 5888, 1792] + speed: 5516.53 + value: {index: 24, type: Single} + - key: [128, 5888, 3328] + speed: 5600.35 + value: {index: 24, type: Single} + - key: [128, 6784, 256] + speed: 4256.63 + value: {index: 24, type: Single} + - key: [128, 6784, 768] + speed: 4745.84 + value: {index: 24, type: Single} + - key: [128, 6784, 1792] + speed: 4904.64 + value: {index: 24, type: Single} + - key: [128, 6784, 3328] + speed: 4966.55 + value: {index: 24, type: Single} + - key: [256, 4, 256] + speed: 19.9934 + value: {index: 27, type: Single} + - key: [256, 4, 768] + speed: 26.021 + value: {index: 27, type: Single} + - key: [256, 4, 1792] + speed: 28.7045 + value: {index: 27, type: Single} + - key: [256, 4, 3328] + speed: 25.4312 + value: {index: 27, type: Single} + - key: [256, 64, 256] + speed: 255.05 + value: {index: 26, type: Single} + - key: [256, 64, 768] + speed: 364.516 + value: {index: 22, type: Single} + - key: [256, 64, 1792] + speed: 393.987 + value: {index: 23, type: Single} + - key: [256, 64, 3328] + speed: 364.397 + value: {index: 23, type: Single} + - key: [256, 128, 256] + speed: 514.733 + value: {index: 26, type: Single} + - key: [256, 128, 768] + speed: 721.29 + value: {index: 22, type: Single} + - key: [256, 128, 1792] + speed: 747.833 + value: {index: 23, type: Single} + - key: [256, 128, 3328] + speed: 730.238 + value: {index: 22, type: Single} + - key: [256, 256, 256] + speed: 976.243 + value: {index: 22, type: Single} + - key: [256, 256, 768] + speed: 1395.19 + value: {index: 22, type: Single} + - key: [256, 256, 1792] + speed: 1436.05 + value: {index: 23, type: Single} + - key: [256, 256, 3328] + speed: 1518.47 + value: {index: 23, type: Single} + - key: [256, 448, 256] + speed: 1701.1 + value: {index: 26, type: Single} + - key: [256, 448, 768] + speed: 2299.93 + value: {index: 22, type: Single} + - key: [256, 448, 1792] + speed: 2385.6 + value: {index: 22, type: Single} + - key: [256, 448, 3328] + speed: 2492.77 + value: {index: 22, type: Single} + - key: [256, 704, 256] + speed: 2240.38 + value: {index: 26, type: Single} + - key: [256, 704, 768] + speed: 2735.74 + value: {index: 26, type: Single} + - key: [256, 704, 1792] + speed: 2912.4 + value: {index: 23, type: Single} + - key: [256, 704, 3328] + speed: 3017.84 + value: {index: 22, type: Single} + - key: [256, 1024, 256] + speed: 2950.94 + value: {index: 24, type: Single} + - key: [256, 1024, 768] + speed: 3801.16 + value: {index: 24, type: Single} + - key: [256, 1024, 1792] + speed: 4161.16 + value: {index: 24, type: Single} + - key: [256, 1024, 3328] + speed: 4326.71 + value: {index: 24, type: Single} + - key: [256, 1408, 256] + speed: 3016.15 + value: {index: 24, type: Single} + - key: [256, 1408, 768] + speed: 3579.52 + value: {index: 24, type: Single} + - key: [256, 1408, 1792] + speed: 3784.6 + value: {index: 24, type: Single} + - key: [256, 1408, 3328] + speed: 3865.78 + value: {index: 24, type: Single} + - key: [256, 1856, 256] + speed: 3689.93 + value: {index: 24, type: Single} + - key: [256, 1856, 768] + speed: 4599.51 + value: {index: 24, type: Single} + - key: [256, 1856, 1792] + speed: 4924.67 + value: {index: 24, type: Single} + - key: [256, 1856, 3328] + speed: 5069.18 + value: {index: 24, type: Single} + - key: [256, 2368, 256] + speed: 3754.47 + value: {index: 24, type: Single} + - key: [256, 2368, 768] + speed: 4255.26 + value: {index: 24, type: Single} + - key: [256, 2368, 1792] + speed: 4430.51 + value: {index: 24, type: Single} + - key: [256, 2368, 3328] + speed: 4503.13 + value: {index: 24, type: Single} + - key: [256, 2944, 256] + speed: 4537.64 + value: {index: 24, type: Single} + - key: [256, 2944, 768] + speed: 5268.91 + value: {index: 24, type: Single} + - key: [256, 2944, 1792] + speed: 5493.26 + value: {index: 24, type: Single} + - key: [256, 2944, 3328] + speed: 5592.95 + value: {index: 24, type: Single} + - key: [256, 3584, 256] + speed: 4428.5 + value: {index: 24, type: Single} + - key: [256, 3584, 768] + speed: 4998.66 + value: {index: 24, type: Single} + - key: [256, 3584, 1792] + speed: 5165.38 + value: {index: 24, type: Single} + - key: [256, 3584, 3328] + speed: 5229.04 + value: {index: 24, type: Single} + - key: [256, 4288, 256] + speed: 4370.56 + value: {index: 24, type: Single} + - key: [256, 4288, 768] + speed: 4863.65 + value: {index: 24, type: Single} + - key: [256, 4288, 1792] + speed: 5014.26 + value: {index: 24, type: Single} + - key: [256, 4288, 3328] + speed: 5067.03 + value: {index: 24, type: Single} + - key: [256, 5056, 256] + speed: 5083.07 + value: {index: 24, type: Single} + - key: [256, 5056, 768] + speed: 5717.65 + value: {index: 24, type: Single} + - key: [256, 5056, 1792] + speed: 5897.86 + value: {index: 24, type: Single} + - key: [256, 5056, 3328] + speed: 5959.86 + value: {index: 24, type: Single} + - key: [256, 5888, 256] + speed: 5018.48 + value: {index: 24, type: Single} + - key: [256, 5888, 768] + speed: 5643.76 + value: {index: 24, type: Single} + - key: [256, 5888, 1792] + speed: 5819.39 + value: {index: 24, type: Single} + - key: [256, 5888, 3328] + speed: 5871.79 + value: {index: 24, type: Single} + - key: [256, 6784, 256] + speed: 5182.95 + value: {index: 24, type: Single} + - key: [256, 6784, 768] + speed: 5642.63 + value: {index: 24, type: Single} + - key: [256, 6784, 1792] + speed: 5788.54 + value: {index: 24, type: Single} + - key: [256, 6784, 3328] + speed: 5837.54 + value: {index: 24, type: Single} + - key: [448, 4, 256] + speed: 27.6473 + value: {index: 27, type: Single} + - key: [448, 4, 768] + speed: 37.2323 + value: {index: 27, type: Single} + - key: [448, 4, 1792] + speed: 42.2524 + value: {index: 27, type: Single} + - key: [448, 4, 3328] + speed: 39.5231 + value: {index: 27, type: Single} + - key: [448, 64, 256] + speed: 454.533 + value: {index: 26, type: Single} + - key: [448, 64, 768] + speed: 629.793 + value: {index: 23, type: Single} + - key: [448, 64, 1792] + speed: 638.101 + value: {index: 23, type: Single} + - key: [448, 64, 3328] + speed: 646.009 + value: {index: 23, type: Single} + - key: [448, 128, 256] + speed: 857.906 + value: {index: 22, type: Single} + - key: [448, 128, 768] + speed: 1208.38 + value: {index: 23, type: Single} + - key: [448, 128, 1792] + speed: 1229.81 + value: {index: 22, type: Single} + - key: [448, 128, 3328] + speed: 1267.23 + value: {index: 22, type: Single} + - key: [448, 256, 256] + speed: 1644.59 + value: {index: 26, type: Single} + - key: [448, 256, 768] + speed: 2286.64 + value: {index: 23, type: Single} + - key: [448, 256, 1792] + speed: 2408.39 + value: {index: 23, type: Single} + - key: [448, 256, 3328] + speed: 2525.77 + value: {index: 23, type: Single} + - key: [448, 448, 256] + speed: 2259.37 + value: {index: 26, type: Single} + - key: [448, 448, 768] + speed: 3055.56 + value: {index: 26, type: Single} + - key: [448, 448, 1792] + speed: 3245.53 + value: {index: 23, type: Single} + - key: [448, 448, 3328] + speed: 3338.39 + value: {index: 22, type: Single} + - key: [448, 704, 256] + speed: 2658.47 + value: {index: 24, type: Single} + - key: [448, 704, 768] + speed: 3162.38 + value: {index: 24, type: Single} + - key: [448, 704, 1792] + speed: 3324.51 + value: {index: 24, type: Single} + - key: [448, 704, 3328] + speed: 3382.56 + value: {index: 24, type: Single} + - key: [448, 1024, 256] + speed: 3820.26 + value: {index: 24, type: Single} + - key: [448, 1024, 768] + speed: 4547.05 + value: {index: 24, type: Single} + - key: [448, 1024, 1792] + speed: 4771.21 + value: {index: 24, type: Single} + - key: [448, 1024, 3328] + speed: 4910.71 + value: {index: 24, type: Single} + - key: [448, 1408, 256] + speed: 3942.02 + value: {index: 24, type: Single} + - key: [448, 1408, 768] + speed: 4430.76 + value: {index: 24, type: Single} + - key: [448, 1408, 1792] + speed: 4624.1 + value: {index: 24, type: Single} + - key: [448, 1408, 3328] + speed: 4695.79 + value: {index: 24, type: Single} + - key: [448, 1856, 256] + speed: 4087.54 + value: {index: 24, type: Single} + - key: [448, 1856, 768] + speed: 4527.64 + value: {index: 24, type: Single} + - key: [448, 1856, 1792] + speed: 4700.82 + value: {index: 24, type: Single} + - key: [448, 1856, 3328] + speed: 4752.68 + value: {index: 24, type: Single} + - key: [448, 2368, 256] + speed: 4323.41 + value: {index: 24, type: Single} + - key: [448, 2368, 768] + speed: 4702.32 + value: {index: 24, type: Single} + - key: [448, 2368, 1792] + speed: 4862.39 + value: {index: 24, type: Single} + - key: [448, 2368, 3328] + speed: 4904.93 + value: {index: 24, type: Single} + - key: [448, 2944, 256] + speed: 4641.6 + value: {index: 24, type: Single} + - key: [448, 2944, 768] + speed: 4974.22 + value: {index: 24, type: Single} + - key: [448, 2944, 1792] + speed: 5099.29 + value: {index: 24, type: Single} + - key: [448, 2944, 3328] + speed: 5141.83 + value: {index: 24, type: Single} + - key: [448, 3584, 256] + speed: 4893.21 + value: {index: 24, type: Single} + - key: [448, 3584, 768] + speed: 5231.54 + value: {index: 24, type: Single} + - key: [448, 3584, 1792] + speed: 5357.57 + value: {index: 24, type: Single} + - key: [448, 3584, 3328] + speed: 5397.77 + value: {index: 24, type: Single} + - key: [448, 4288, 256] + speed: 5211.05 + value: {index: 24, type: Single} + - key: [448, 4288, 768] + speed: 5535.47 + value: {index: 24, type: Single} + - key: [448, 4288, 1792] + speed: 5647.36 + value: {index: 24, type: Single} + - key: [448, 4288, 3328] + speed: 5685.09 + value: {index: 24, type: Single} + - key: [448, 5056, 256] + speed: 5481.73 + value: {index: 24, type: Single} + - key: [448, 5056, 768] + speed: 5827.24 + value: {index: 24, type: Single} + - key: [448, 5056, 1792] + speed: 5940.54 + value: {index: 24, type: Single} + - key: [448, 5056, 3328] + speed: 5979.73 + value: {index: 24, type: Single} + - key: [448, 5888, 256] + speed: 5356.07 + value: {index: 24, type: Single} + - key: [448, 5888, 768] + speed: 5619.12 + value: {index: 24, type: Single} + - key: [448, 5888, 1792] + speed: 5705.19 + value: {index: 24, type: Single} + - key: [448, 5888, 3328] + speed: 5735.32 + value: {index: 24, type: Single} + - key: [448, 6784, 256] + speed: 5659.11 + value: {index: 24, type: Single} + - key: [448, 6784, 768] + speed: 5946.39 + value: {index: 24, type: Single} + - key: [448, 6784, 1792] + speed: 6053.27 + value: {index: 21, type: Single} + - key: [448, 6784, 3328] + speed: 6121.63 + value: {index: 21, type: Single} + - key: [704, 4, 256] + speed: 42.4969 + value: {index: 27, type: Single} + - key: [704, 4, 768] + speed: 58.3911 + value: {index: 27, type: Single} + - key: [704, 4, 1792] + speed: 63.3703 + value: {index: 27, type: Single} + - key: [704, 4, 3328] + speed: 59.7891 + value: {index: 27, type: Single} + - key: [704, 64, 256] + speed: 707.758 + value: {index: 26, type: Single} + - key: [704, 64, 768] + speed: 973.184 + value: {index: 23, type: Single} + - key: [704, 64, 1792] + speed: 981.067 + value: {index: 22, type: Single} + - key: [704, 64, 3328] + speed: 1011.1 + value: {index: 23, type: Single} + - key: [704, 128, 256] + speed: 1314.0 + value: {index: 23, type: Single} + - key: [704, 128, 768] + speed: 1831.89 + value: {index: 23, type: Single} + - key: [704, 128, 1792] + speed: 1963.91 + value: {index: 22, type: Single} + - key: [704, 128, 3328] + speed: 2022.2 + value: {index: 22, type: Single} + - key: [704, 256, 256] + speed: 2332.7 + value: {index: 26, type: Single} + - key: [704, 256, 768] + speed: 2924.09 + value: {index: 26, type: Single} + - key: [704, 256, 1792] + speed: 2953.83 + value: {index: 23, type: Single} + - key: [704, 256, 3328] + speed: 3047.38 + value: {index: 23, type: Single} + - key: [704, 448, 256] + speed: 2658.47 + value: {index: 24, type: Single} + - key: [704, 448, 768] + speed: 3159.31 + value: {index: 24, type: Single} + - key: [704, 448, 1792] + speed: 3324.51 + value: {index: 24, type: Single} + - key: [704, 448, 3328] + speed: 3393.9 + value: {index: 24, type: Single} + - key: [704, 704, 256] + speed: 4187.8 + value: {index: 24, type: Single} + - key: [704, 704, 768] + speed: 4917.14 + value: {index: 24, type: Single} + - key: [704, 704, 1792] + speed: 5188.06 + value: {index: 24, type: Single} + - key: [704, 704, 3328] + speed: 5305.32 + value: {index: 24, type: Single} + - key: [704, 1024, 256] + speed: 4448.8 + value: {index: 24, type: Single} + - key: [704, 1024, 768] + speed: 5056.89 + value: {index: 24, type: Single} + - key: [704, 1024, 1792] + speed: 5260.77 + value: {index: 24, type: Single} + - key: [704, 1024, 3328] + speed: 5353.31 + value: {index: 24, type: Single} + - key: [704, 1408, 256] + speed: 4865.92 + value: {index: 24, type: Single} + - key: [704, 1408, 768] + speed: 5408.83 + value: {index: 24, type: Single} + - key: [704, 1408, 1792] + speed: 5597.39 + value: {index: 24, type: Single} + - key: [704, 1408, 3328] + speed: 5662.16 + value: {index: 24, type: Single} + - key: [704, 1856, 256] + speed: 5256.76 + value: {index: 24, type: Single} + - key: [704, 1856, 768] + speed: 5806.56 + value: {index: 24, type: Single} + - key: [704, 1856, 1792] + speed: 5969.59 + value: {index: 24, type: Single} + - key: [704, 1856, 3328] + speed: 6024.47 + value: {index: 24, type: Single} + - key: [704, 2368, 256] + speed: 5062.61 + value: {index: 24, type: Single} + - key: [704, 2368, 768] + speed: 5452.27 + value: {index: 24, type: Single} + - key: [704, 2368, 1792] + speed: 5556.45 + value: {index: 24, type: Single} + - key: [704, 2368, 3328] + speed: 5600.97 + value: {index: 24, type: Single} + - key: [704, 2944, 256] + speed: 5595.83 + value: {index: 24, type: Single} + - key: [704, 2944, 768] + speed: 5958.92 + value: {index: 24, type: Single} + - key: [704, 2944, 1792] + speed: 6084.02 + value: {index: 24, type: Single} + - key: [704, 2944, 3328] + speed: 6131.18 + value: {index: 21, type: Single} + - key: [704, 3584, 256] + speed: 5561.04 + value: {index: 24, type: Single} + - key: [704, 3584, 768] + speed: 5875.82 + value: {index: 24, type: Single} + - key: [704, 3584, 1792] + speed: 5983.54 + value: {index: 21, type: Single} + - key: [704, 3584, 3328] + speed: 6042.77 + value: {index: 21, type: Single} + - key: [704, 4288, 256] + speed: 5636.17 + value: {index: 24, type: Single} + - key: [704, 4288, 768] + speed: 5914.13 + value: {index: 24, type: Single} + - key: [704, 4288, 1792] + speed: 6008.03 + value: {index: 21, type: Single} + - key: [704, 4288, 3328] + speed: 6069.76 + value: {index: 21, type: Single} + - key: [704, 5056, 256] + speed: 5758.91 + value: {index: 24, type: Single} + - key: [704, 5056, 768] + speed: 6007.34 + value: {index: 24, type: Single} + - key: [704, 5056, 1792] + speed: 6123.83 + value: {index: 21, type: Single} + - key: [704, 5056, 3328] + speed: 6177.6 + value: {index: 21, type: Single} + - key: [704, 5888, 256] + speed: 5900.01 + value: {index: 24, type: Single} + - key: [704, 5888, 768] + speed: 6145.54 + value: {index: 24, type: Single} + - key: [704, 5888, 1792] + speed: 6275.53 + value: {index: 21, type: Single} + - key: [704, 5888, 3328] + speed: 6330.85 + value: {index: 21, type: Single} + - key: [704, 6784, 256] + speed: 5785.21 + value: {index: 24, type: Single} + - key: [704, 6784, 768] + speed: 5998.98 + value: {index: 24, type: Single} + - key: [704, 6784, 1792] + speed: 6062.67 + value: {index: 24, type: Single} + - key: [704, 6784, 3328] + speed: 6081.84 + value: {index: 24, type: Single} + - key: [1024, 4, 256] + speed: 59.4768 + value: {index: 27, type: Single} + - key: [1024, 4, 768] + speed: 77.0709 + value: {index: 27, type: Single} + - key: [1024, 4, 1792] + speed: 82.2989 + value: {index: 27, type: Single} + - key: [1024, 4, 3328] + speed: 81.8959 + value: {index: 27, type: Single} + - key: [1024, 64, 256] + speed: 967.879 + value: {index: 26, type: Single} + - key: [1024, 64, 768] + speed: 1237.62 + value: {index: 23, type: Single} + - key: [1024, 64, 1792] + speed: 1370.27 + value: {index: 23, type: Single} + - key: [1024, 64, 3328] + speed: 1414.18 + value: {index: 23, type: Single} + - key: [1024, 128, 256] + speed: 1735.56 + value: {index: 26, type: Single} + - key: [1024, 128, 768] + speed: 2409.42 + value: {index: 23, type: Single} + - key: [1024, 128, 1792] + speed: 2607.56 + value: {index: 23, type: Single} + - key: [1024, 128, 3328] + speed: 2644.2 + value: {index: 23, type: Single} + - key: [1024, 256, 256] + speed: 2885.16 + value: {index: 24, type: Single} + - key: [1024, 256, 768] + speed: 3871.55 + value: {index: 23, type: Single} + - key: [1024, 256, 1792] + speed: 4191.42 + value: {index: 23, type: Single} + - key: [1024, 256, 3328] + speed: 4326.71 + value: {index: 24, type: Single} + - key: [1024, 448, 256] + speed: 3530.93 + value: {index: 24, type: Single} + - key: [1024, 448, 768] + speed: 4355.51 + value: {index: 24, type: Single} + - key: [1024, 448, 1792] + speed: 4750.79 + value: {index: 24, type: Single} + - key: [1024, 448, 3328] + speed: 4864.35 + value: {index: 24, type: Single} + - key: [1024, 704, 256] + speed: 4159.19 + value: {index: 24, type: Single} + - key: [1024, 704, 768] + speed: 4959.56 + value: {index: 24, type: Single} + - key: [1024, 704, 1792] + speed: 5221.38 + value: {index: 24, type: Single} + - key: [1024, 704, 3328] + speed: 5304.21 + value: {index: 24, type: Single} + - key: [1024, 1024, 256] + speed: 4622.18 + value: {index: 21, type: Single} + - key: [1024, 1024, 768] + speed: 5331.71 + value: {index: 21, type: Single} + - key: [1024, 1024, 1792] + speed: 5728.65 + value: {index: 21, type: Single} + - key: [1024, 1024, 3328] + speed: 5832.51 + value: {index: 21, type: Single} + - key: [1024, 1408, 256] + speed: 4870.66 + value: {index: 24, type: Single} + - key: [1024, 1408, 768] + speed: 5377.01 + value: {index: 24, type: Single} + - key: [1024, 1408, 1792] + speed: 5538.96 + value: {index: 24, type: Single} + - key: [1024, 1408, 3328] + speed: 5605.32 + value: {index: 24, type: Single} + - key: [1024, 1856, 256] + speed: 5052.38 + value: {index: 24, type: Single} + - key: [1024, 1856, 768] + speed: 5458.26 + value: {index: 24, type: Single} + - key: [1024, 1856, 1792] + speed: 5563.49 + value: {index: 24, type: Single} + - key: [1024, 1856, 3328] + speed: 5613.76 + value: {index: 24, type: Single} + - key: [1024, 2368, 256] + speed: 5273.75 + value: {index: 24, type: Single} + - key: [1024, 2368, 768] + speed: 5629.19 + value: {index: 24, type: Single} + - key: [1024, 2368, 1792] + speed: 5729.63 + value: {index: 24, type: Single} + - key: [1024, 2368, 3328] + speed: 5786.67 + value: {index: 21, type: Single} + - key: [1024, 2944, 256] + speed: 5506.54 + value: {index: 24, type: Single} + - key: [1024, 2944, 768] + speed: 5865.1 + value: {index: 24, type: Single} + - key: [1024, 2944, 1792] + speed: 6036.13 + value: {index: 25, type: Single} + - key: [1024, 2944, 3328] + speed: 6100.29 + value: {index: 25, type: Single} + - key: [1024, 3584, 256] + speed: 5741.62 + value: {index: 24, type: Single} + - key: [1024, 3584, 768] + speed: 6080.17 + value: {index: 24, type: Single} + - key: [1024, 3584, 1792] + speed: 6213.79 + value: {index: 24, type: Single} + - key: [1024, 3584, 3328] + speed: 6330.9 + value: {index: 21, type: Single} + - key: [1024, 4288, 256] + speed: 5739.26 + value: {index: 24, type: Single} + - key: [1024, 4288, 768] + speed: 6076.31 + value: {index: 24, type: Single} + - key: [1024, 4288, 1792] + speed: 6189.37 + value: {index: 24, type: Single} + - key: [1024, 4288, 3328] + speed: 6223.21 + value: {index: 24, type: Single} + - key: [1024, 5056, 256] + speed: 5858.68 + value: {index: 24, type: Single} + - key: [1024, 5056, 768] + speed: 6156.35 + value: {index: 25, type: Single} + - key: [1024, 5056, 1792] + speed: 6347.72 + value: {index: 25, type: Single} + - key: [1024, 5056, 3328] + speed: 6418.18 + value: {index: 25, type: Single} + - key: [1024, 5888, 256] + speed: 5869.51 + value: {index: 24, type: Single} + - key: [1024, 5888, 768] + speed: 6195.87 + value: {index: 24, type: Single} + - key: [1024, 5888, 1792] + speed: 6305.15 + value: {index: 24, type: Single} + - key: [1024, 5888, 3328] + speed: 6339.73 + value: {index: 24, type: Single} + - key: [1024, 6784, 256] + speed: 5907.37 + value: {index: 24, type: Single} + - key: [1024, 6784, 768] + speed: 6146.32 + value: {index: 24, type: Single} + - key: [1024, 6784, 1792] + speed: 6220.49 + value: {index: 24, type: Single} + - key: [1024, 6784, 3328] + speed: 6247.33 + value: {index: 24, type: Single} + - key: [1408, 4, 256] + speed: 82.8235 + value: {index: 27, type: Single} + - key: [1408, 4, 768] + speed: 106.943 + value: {index: 27, type: Single} + - key: [1408, 4, 1792] + speed: 116.45 + value: {index: 27, type: Single} + - key: [1408, 4, 3328] + speed: 117.251 + value: {index: 27, type: Single} + - key: [1408, 64, 256] + speed: 1336.54 + value: {index: 23, type: Single} + - key: [1408, 64, 768] + speed: 1762.76 + value: {index: 23, type: Single} + - key: [1408, 64, 1792] + speed: 1920.64 + value: {index: 23, type: Single} + - key: [1408, 64, 3328] + speed: 1989.4 + value: {index: 23, type: Single} + - key: [1408, 128, 256] + speed: 2248.52 + value: {index: 26, type: Single} + - key: [1408, 128, 768] + speed: 2743.77 + value: {index: 26, type: Single} + - key: [1408, 128, 1792] + speed: 2949.84 + value: {index: 22, type: Single} + - key: [1408, 128, 3328] + speed: 3030.27 + value: {index: 22, type: Single} + - key: [1408, 256, 256] + speed: 3091.02 + value: {index: 24, type: Single} + - key: [1408, 256, 768] + speed: 3610.64 + value: {index: 24, type: Single} + - key: [1408, 256, 1792] + speed: 3807.73 + value: {index: 24, type: Single} + - key: [1408, 256, 3328] + speed: 3875.96 + value: {index: 23, type: Single} + - key: [1408, 448, 256] + speed: 3913.73 + value: {index: 24, type: Single} + - key: [1408, 448, 768] + speed: 4445.83 + value: {index: 24, type: Single} + - key: [1408, 448, 1792] + speed: 4617.1 + value: {index: 24, type: Single} + - key: [1408, 448, 3328] + speed: 4696.57 + value: {index: 24, type: Single} + - key: [1408, 704, 256] + speed: 4859.03 + value: {index: 24, type: Single} + - key: [1408, 704, 768] + speed: 5408.83 + value: {index: 24, type: Single} + - key: [1408, 704, 1792] + speed: 5586.95 + value: {index: 24, type: Single} + - key: [1408, 704, 3328] + speed: 5649.94 + value: {index: 24, type: Single} + - key: [1408, 1024, 256] + speed: 4992.68 + value: {index: 24, type: Single} + - key: [1408, 1024, 768] + speed: 5377.01 + value: {index: 24, type: Single} + - key: [1408, 1024, 1792] + speed: 5448.12 + value: {index: 21, type: Single} + - key: [1408, 1024, 3328] + speed: 5606.29 + value: {index: 24, type: Single} + - key: [1408, 1408, 256] + speed: 5331.7 + value: {index: 24, type: Single} + - key: [1408, 1408, 768] + speed: 5710.93 + value: {index: 24, type: Single} + - key: [1408, 1408, 1792] + speed: 5823.74 + value: {index: 24, type: Single} + - key: [1408, 1408, 3328] + speed: 5876.99 + value: {index: 25, type: Single} + - key: [1408, 1856, 256] + speed: 5737.71 + value: {index: 24, type: Single} + - key: [1408, 1856, 768] + speed: 6084.3 + value: {index: 24, type: Single} + - key: [1408, 1856, 1792] + speed: 6193.59 + value: {index: 24, type: Single} + - key: [1408, 1856, 3328] + speed: 6228.7 + value: {index: 24, type: Single} + - key: [1408, 2368, 256] + speed: 5761.23 + value: {index: 24, type: Single} + - key: [1408, 2368, 768] + speed: 6043.27 + value: {index: 24, type: Single} + - key: [1408, 2368, 1792] + speed: 6127.59 + value: {index: 24, type: Single} + - key: [1408, 2368, 3328] + speed: 6157.45 + value: {index: 24, type: Single} + - key: [1408, 2944, 256] + speed: 5897.59 + value: {index: 24, type: Single} + - key: [1408, 2944, 768] + speed: 6164.93 + value: {index: 25, type: Single} + - key: [1408, 2944, 1792] + speed: 6319.43 + value: {index: 25, type: Single} + - key: [1408, 2944, 3328] + speed: 6380.96 + value: {index: 25, type: Single} + - key: [1408, 3584, 256] + speed: 5836.5 + value: {index: 24, type: Single} + - key: [1408, 3584, 768] + speed: 6072.23 + value: {index: 25, type: Single} + - key: [1408, 3584, 1792] + speed: 6215.69 + value: {index: 25, type: Single} + - key: [1408, 3584, 3328] + speed: 6273.18 + value: {index: 25, type: Single} + - key: [1408, 4288, 256] + speed: 5851.11 + value: {index: 24, type: Single} + - key: [1408, 4288, 768] + speed: 6071.3 + value: {index: 25, type: Single} + - key: [1408, 4288, 1792] + speed: 6222.27 + value: {index: 25, type: Single} + - key: [1408, 4288, 3328] + speed: 6284.8 + value: {index: 25, type: Single} + - key: [1408, 5056, 256] + speed: 5939.65 + value: {index: 24, type: Single} + - key: [1408, 5056, 768] + speed: 6180.91 + value: {index: 25, type: Single} + - key: [1408, 5056, 1792] + speed: 6315.63 + value: {index: 25, type: Single} + - key: [1408, 5056, 3328] + speed: 6373.08 + value: {index: 25, type: Single} + - key: [1408, 5888, 256] + speed: 6066.17 + value: {index: 24, type: Single} + - key: [1408, 5888, 768] + speed: 6371.97 + value: {index: 25, type: Single} + - key: [1408, 5888, 1792] + speed: 6489.99 + value: {index: 25, type: Single} + - key: [1408, 5888, 3328] + speed: 6531.23 + value: {index: 25, type: Single} + - key: [1408, 6784, 256] + speed: 6075.9 + value: {index: 24, type: Single} + - key: [1408, 6784, 768] + speed: 6232.29 + value: {index: 24, type: Single} + - key: [1408, 6784, 1792] + speed: 6278.63 + value: {index: 24, type: Single} + - key: [1408, 6784, 3328] + speed: 6294.98 + value: {index: 24, type: Single} + - key: [1856, 4, 256] + speed: 108.256 + value: {index: 27, type: Single} + - key: [1856, 4, 768] + speed: 140.971 + value: {index: 27, type: Single} + - key: [1856, 4, 1792] + speed: 149.664 + value: {index: 27, type: Single} + - key: [1856, 4, 3328] + speed: 152.3 + value: {index: 27, type: Single} + - key: [1856, 64, 256] + speed: 1754.28 + value: {index: 26, type: Single} + - key: [1856, 64, 768] + speed: 2289.07 + value: {index: 23, type: Single} + - key: [1856, 64, 1792] + speed: 2558.81 + value: {index: 23, type: Single} + - key: [1856, 64, 3328] + speed: 2615.98 + value: {index: 22, type: Single} + - key: [1856, 128, 256] + speed: 2614.68 + value: {index: 24, type: Single} + - key: [1856, 128, 768] + speed: 3478.89 + value: {index: 23, type: Single} + - key: [1856, 128, 1792] + speed: 3833.95 + value: {index: 22, type: Single} + - key: [1856, 128, 3328] + speed: 3975.11 + value: {index: 22, type: Single} + - key: [1856, 256, 256] + speed: 4004.9 + value: {index: 24, type: Single} + - key: [1856, 256, 768] + speed: 4709.45 + value: {index: 24, type: Single} + - key: [1856, 256, 1792] + speed: 4990.97 + value: {index: 24, type: Single} + - key: [1856, 256, 3328] + speed: 5095.8 + value: {index: 24, type: Single} + - key: [1856, 448, 256] + speed: 4128.65 + value: {index: 24, type: Single} + - key: [1856, 448, 768] + speed: 4546.74 + value: {index: 24, type: Single} + - key: [1856, 448, 1792] + speed: 4693.14 + value: {index: 24, type: Single} + - key: [1856, 448, 3328] + speed: 4755.71 + value: {index: 24, type: Single} + - key: [1856, 704, 256] + speed: 5299.95 + value: {index: 24, type: Single} + - key: [1856, 704, 768] + speed: 5801.59 + value: {index: 24, type: Single} + - key: [1856, 704, 1792] + speed: 5970.72 + value: {index: 24, type: Single} + - key: [1856, 704, 3328] + speed: 6027.56 + value: {index: 24, type: Single} + - key: [1856, 1024, 256] + speed: 5002.36 + value: {index: 21, type: Single} + - key: [1856, 1024, 768] + speed: 5399.91 + value: {index: 21, type: Single} + - key: [1856, 1024, 1792] + speed: 5564.84 + value: {index: 21, type: Single} + - key: [1856, 1024, 3328] + speed: 5625.23 + value: {index: 25, type: Single} + - key: [1856, 1408, 256] + speed: 5763.34 + value: {index: 24, type: Single} + - key: [1856, 1408, 768] + speed: 6095.25 + value: {index: 24, type: Single} + - key: [1856, 1408, 1792] + speed: 6196.62 + value: {index: 24, type: Single} + - key: [1856, 1408, 3328] + speed: 6265.94 + value: {index: 21, type: Single} + - key: [1856, 1856, 256] + speed: 5589.06 + value: {index: 24, type: Single} + - key: [1856, 1856, 768] + speed: 5829.92 + value: {index: 24, type: Single} + - key: [1856, 1856, 1792] + speed: 5895.91 + value: {index: 24, type: Single} + - key: [1856, 1856, 3328] + speed: 5971.17 + value: {index: 21, type: Single} + - key: [1856, 2368, 256] + speed: 5910.01 + value: {index: 24, type: Single} + - key: [1856, 2368, 768] + speed: 6146.8 + value: {index: 24, type: Single} + - key: [1856, 2368, 1792] + speed: 6217.25 + value: {index: 24, type: Single} + - key: [1856, 2368, 3328] + speed: 6241.62 + value: {index: 24, type: Single} + - key: [1856, 2944, 256] + speed: 6004.23 + value: {index: 24, type: Single} + - key: [1856, 2944, 768] + speed: 6219.11 + value: {index: 24, type: Single} + - key: [1856, 2944, 1792] + speed: 6282.16 + value: {index: 24, type: Single} + - key: [1856, 2944, 3328] + speed: 6303.32 + value: {index: 24, type: Single} + - key: [1856, 3584, 256] + speed: 5952.46 + value: {index: 24, type: Single} + - key: [1856, 3584, 768] + speed: 6196.32 + value: {index: 21, type: Single} + - key: [1856, 3584, 1792] + speed: 6298.67 + value: {index: 21, type: Single} + - key: [1856, 3584, 3328] + speed: 6342.83 + value: {index: 21, type: Single} + - key: [1856, 4288, 256] + speed: 6010.48 + value: {index: 24, type: Single} + - key: [1856, 4288, 768] + speed: 6183.44 + value: {index: 24, type: Single} + - key: [1856, 4288, 1792] + speed: 6232.68 + value: {index: 24, type: Single} + - key: [1856, 4288, 3328] + speed: 6259.76 + value: {index: 25, type: Single} + - key: [1856, 5056, 256] + speed: 6125.8 + value: {index: 24, type: Single} + - key: [1856, 5056, 768] + speed: 6289.35 + value: {index: 24, type: Single} + - key: [1856, 5056, 1792] + speed: 6337.87 + value: {index: 24, type: Single} + - key: [1856, 5056, 3328] + speed: 6354.32 + value: {index: 24, type: Single} + - key: [1856, 5888, 256] + speed: 6137.92 + value: {index: 24, type: Single} + - key: [1856, 5888, 768] + speed: 6373.39 + value: {index: 21, type: Single} + - key: [1856, 5888, 1792] + speed: 6466.09 + value: {index: 21, type: Single} + - key: [1856, 5888, 3328] + speed: 6503.71 + value: {index: 21, type: Single} + - key: [1856, 6784, 256] + speed: 6096.06 + value: {index: 24, type: Single} + - key: [1856, 6784, 768] + speed: 6226.02 + value: {index: 24, type: Single} + - key: [1856, 6784, 1792] + speed: 6294.34 + value: {index: 21, type: Single} + - key: [1856, 6784, 3328] + speed: 6324.24 + value: {index: 21, type: Single} + - key: [2368, 4, 256] + speed: 139.893 + value: {index: 27, type: Single} + - key: [2368, 4, 768] + speed: 173.504 + value: {index: 27, type: Single} + - key: [2368, 4, 1792] + speed: 187.975 + value: {index: 27, type: Single} + - key: [2368, 4, 3328] + speed: 192.817 + value: {index: 27, type: Single} + - key: [2368, 64, 256] + speed: 1883.96 + value: {index: 26, type: Single} + - key: [2368, 64, 768] + speed: 2314.07 + value: {index: 22, type: Single} + - key: [2368, 64, 1792] + speed: 2477.2 + value: {index: 22, type: Single} + - key: [2368, 64, 3328] + speed: 2546.28 + value: {index: 23, type: Single} + - key: [2368, 128, 256] + speed: 2592.8 + value: {index: 24, type: Single} + - key: [2368, 128, 768] + speed: 3004.28 + value: {index: 24, type: Single} + - key: [2368, 128, 1792] + speed: 3189.43 + value: {index: 24, type: Single} + - key: [2368, 128, 3328] + speed: 3259.33 + value: {index: 24, type: Single} + - key: [2368, 256, 256] + speed: 3795.29 + value: {index: 24, type: Single} + - key: [2368, 256, 768] + speed: 4278.43 + value: {index: 24, type: Single} + - key: [2368, 256, 1792] + speed: 4438.55 + value: {index: 24, type: Single} + - key: [2368, 256, 3328] + speed: 4514.33 + value: {index: 24, type: Single} + - key: [2368, 448, 256] + speed: 4369.77 + value: {index: 24, type: Single} + - key: [2368, 448, 768] + speed: 4730.64 + value: {index: 24, type: Single} + - key: [2368, 448, 1792] + speed: 4856.87 + value: {index: 24, type: Single} + - key: [2368, 448, 3328] + speed: 4903.92 + value: {index: 24, type: Single} + - key: [2368, 704, 256] + speed: 5121.11 + value: {index: 24, type: Single} + - key: [2368, 704, 768] + speed: 5450.57 + value: {index: 24, type: Single} + - key: [2368, 704, 1792] + speed: 5564.88 + value: {index: 24, type: Single} + - key: [2368, 704, 3328] + speed: 5602.23 + value: {index: 24, type: Single} + - key: [2368, 1024, 256] + speed: 5208.2 + value: {index: 21, type: Single} + - key: [2368, 1024, 768] + speed: 5610.35 + value: {index: 21, type: Single} + - key: [2368, 1024, 1792] + speed: 5748.16 + value: {index: 21, type: Single} + - key: [2368, 1024, 3328] + speed: 5811.99 + value: {index: 21, type: Single} + - key: [2368, 1408, 256] + speed: 5775.69 + value: {index: 24, type: Single} + - key: [2368, 1408, 768] + speed: 6047.49 + value: {index: 24, type: Single} + - key: [2368, 1408, 1792] + speed: 6125.26 + value: {index: 24, type: Single} + - key: [2368, 1408, 3328] + speed: 6157.71 + value: {index: 24, type: Single} + - key: [2368, 1856, 256] + speed: 5905.41 + value: {index: 24, type: Single} + - key: [2368, 1856, 768] + speed: 6145.14 + value: {index: 24, type: Single} + - key: [2368, 1856, 1792] + speed: 6213.97 + value: {index: 24, type: Single} + - key: [2368, 1856, 3328] + speed: 6240.83 + value: {index: 24, type: Single} + - key: [2368, 2368, 256] + speed: 5899.16 + value: {index: 24, type: Single} + - key: [2368, 2368, 768] + speed: 6103.54 + value: {index: 24, type: Single} + - key: [2368, 2368, 1792] + speed: 6221.38 + value: {index: 21, type: Single} + - key: [2368, 2368, 3328] + speed: 6276.9 + value: {index: 21, type: Single} + - key: [2368, 2944, 256] + speed: 6020.11 + value: {index: 24, type: Single} + - key: [2368, 2944, 768] + speed: 6198.22 + value: {index: 24, type: Single} + - key: [2368, 2944, 1792] + speed: 6257.78 + value: {index: 24, type: Single} + - key: [2368, 2944, 3328] + speed: 6276.97 + value: {index: 24, type: Single} + - key: [2368, 3584, 256] + speed: 6036.22 + value: {index: 24, type: Single} + - key: [2368, 3584, 768] + speed: 6202.15 + value: {index: 24, type: Single} + - key: [2368, 3584, 1792] + speed: 6245.35 + value: {index: 24, type: Single} + - key: [2368, 3584, 3328] + speed: 6264.92 + value: {index: 24, type: Single} + - key: [2368, 4288, 256] + speed: 6130.54 + value: {index: 24, type: Single} + - key: [2368, 4288, 768] + speed: 6288.0 + value: {index: 24, type: Single} + - key: [2368, 4288, 1792] + speed: 6334.15 + value: {index: 24, type: Single} + - key: [2368, 4288, 3328] + speed: 6351.62 + value: {index: 24, type: Single} + - key: [2368, 5056, 256] + speed: 6152.59 + value: {index: 24, type: Single} + - key: [2368, 5056, 768] + speed: 6298.71 + value: {index: 24, type: Single} + - key: [2368, 5056, 1792] + speed: 6341.87 + value: {index: 24, type: Single} + - key: [2368, 5056, 3328] + speed: 6357.52 + value: {index: 24, type: Single} + - key: [2368, 5888, 256] + speed: 6127.29 + value: {index: 24, type: Single} + - key: [2368, 5888, 768] + speed: 6362.73 + value: {index: 21, type: Single} + - key: [2368, 5888, 1792] + speed: 6443.07 + value: {index: 21, type: Single} + - key: [2368, 5888, 3328] + speed: 6475.2 + value: {index: 21, type: Single} + - key: [2368, 6784, 256] + speed: 6160.39 + value: {index: 24, type: Single} + - key: [2368, 6784, 768] + speed: 6400.44 + value: {index: 21, type: Single} + - key: [2368, 6784, 1792] + speed: 6478.01 + value: {index: 21, type: Single} + - key: [2368, 6784, 3328] + speed: 6506.15 + value: {index: 21, type: Single} + - key: [2944, 4, 256] + speed: 164.099 + value: {index: 27, type: Single} + - key: [2944, 4, 768] + speed: 209.776 + value: {index: 27, type: Single} + - key: [2944, 4, 1792] + speed: 229.371 + value: {index: 27, type: Single} + - key: [2944, 4, 3328] + speed: 235.556 + value: {index: 27, type: Single} + - key: [2944, 64, 256] + speed: 2253.11 + value: {index: 22, type: Single} + - key: [2944, 64, 768] + speed: 2831.08 + value: {index: 22, type: Single} + - key: [2944, 64, 1792] + speed: 3059.08 + value: {index: 22, type: Single} + - key: [2944, 64, 3328] + speed: 3194.31 + value: {index: 26, type: Single} + - key: [2944, 128, 256] + speed: 3145.64 + value: {index: 24, type: Single} + - key: [2944, 128, 768] + speed: 3738.65 + value: {index: 24, type: Single} + - key: [2944, 128, 1792] + speed: 3958.34 + value: {index: 24, type: Single} + - key: [2944, 128, 3328] + speed: 4044.4 + value: {index: 24, type: Single} + - key: [2944, 256, 256] + speed: 4651.07 + value: {index: 24, type: Single} + - key: [2944, 256, 768] + speed: 5268.89 + value: {index: 24, type: Single} + - key: [2944, 256, 1792] + speed: 5501.56 + value: {index: 24, type: Single} + - key: [2944, 256, 3328] + speed: 5599.42 + value: {index: 24, type: Single} + - key: [2944, 448, 256] + speed: 4674.89 + value: {index: 24, type: Single} + - key: [2944, 448, 768] + speed: 5005.17 + value: {index: 24, type: Single} + - key: [2944, 448, 1792] + speed: 5101.73 + value: {index: 24, type: Single} + - key: [2944, 448, 3328] + speed: 5148.09 + value: {index: 24, type: Single} + - key: [2944, 704, 256] + speed: 5578.38 + value: {index: 24, type: Single} + - key: [2944, 704, 768] + speed: 5965.53 + value: {index: 24, type: Single} + - key: [2944, 704, 1792] + speed: 6087.71 + value: {index: 24, type: Single} + - key: [2944, 704, 3328] + speed: 6123.92 + value: {index: 24, type: Single} + - key: [2944, 1024, 256] + speed: 5409.33 + value: {index: 21, type: Single} + - key: [2944, 1024, 768] + speed: 5855.21 + value: {index: 25, type: Single} + - key: [2944, 1024, 1792] + speed: 6037.63 + value: {index: 25, type: Single} + - key: [2944, 1024, 3328] + speed: 6094.81 + value: {index: 25, type: Single} + - key: [2944, 1408, 256] + speed: 5900.01 + value: {index: 24, type: Single} + - key: [2944, 1408, 768] + speed: 6171.14 + value: {index: 25, type: Single} + - key: [2944, 1408, 1792] + speed: 6327.01 + value: {index: 25, type: Single} + - key: [2944, 1408, 3328] + speed: 6378.99 + value: {index: 25, type: Single} + - key: [2944, 1856, 256] + speed: 5998.5 + value: {index: 24, type: Single} + - key: [2944, 1856, 768] + speed: 6218.42 + value: {index: 24, type: Single} + - key: [2944, 1856, 1792] + speed: 6280.97 + value: {index: 24, type: Single} + - key: [2944, 1856, 3328] + speed: 6301.54 + value: {index: 24, type: Single} + - key: [2944, 2368, 256] + speed: 6020.11 + value: {index: 24, type: Single} + - key: [2944, 2368, 768] + speed: 6199.28 + value: {index: 24, type: Single} + - key: [2944, 2368, 1792] + speed: 6256.86 + value: {index: 24, type: Single} + - key: [2944, 2368, 3328] + speed: 6275.71 + value: {index: 24, type: Single} + - key: [2944, 2944, 256] + speed: 5991.76 + value: {index: 24, type: Single} + - key: [2944, 2944, 768] + speed: 6231.51 + value: {index: 21, type: Single} + - key: [2944, 2944, 1792] + speed: 6314.21 + value: {index: 21, type: Single} + - key: [2944, 2944, 3328] + speed: 6352.97 + value: {index: 21, type: Single} + - key: [2944, 3584, 256] + speed: 6064.24 + value: {index: 24, type: Single} + - key: [2944, 3584, 768] + speed: 6219.39 + value: {index: 24, type: Single} + - key: [2944, 3584, 1792] + speed: 6263.5 + value: {index: 24, type: Single} + - key: [2944, 3584, 3328] + speed: 6284.61 + value: {index: 21, type: Single} + - key: [2944, 4288, 256] + speed: 6099.96 + value: {index: 24, type: Single} + - key: [2944, 4288, 768] + speed: 6242.52 + value: {index: 24, type: Single} + - key: [2944, 4288, 1792] + speed: 6290.7 + value: {index: 21, type: Single} + - key: [2944, 4288, 3328] + speed: 6324.57 + value: {index: 21, type: Single} + - key: [2944, 5056, 256] + speed: 6187.24 + value: {index: 24, type: Single} + - key: [2944, 5056, 768] + speed: 6329.09 + value: {index: 24, type: Single} + - key: [2944, 5056, 1792] + speed: 6387.31 + value: {index: 21, type: Single} + - key: [2944, 5056, 3328] + speed: 6434.54 + value: {index: 21, type: Single} + - key: [2944, 5888, 256] + speed: 6158.06 + value: {index: 24, type: Single} + - key: [2944, 5888, 768] + speed: 6399.49 + value: {index: 25, type: Single} + - key: [2944, 5888, 1792] + speed: 6483.99 + value: {index: 25, type: Single} + - key: [2944, 5888, 3328] + speed: 6509.19 + value: {index: 25, type: Single} + - key: [2944, 6784, 256] + speed: 6180.3 + value: {index: 24, type: Single} + - key: [2944, 6784, 768] + speed: 6346.82 + value: {index: 21, type: Single} + - key: [2944, 6784, 1792] + speed: 6414.65 + value: {index: 21, type: Single} + - key: [2944, 6784, 3328] + speed: 6451.09 + value: {index: 21, type: Single} + - key: [3584, 4, 256] + speed: 192.032 + value: {index: 27, type: Single} + - key: [3584, 4, 768] + speed: 251.494 + value: {index: 27, type: Single} + - key: [3584, 4, 1792] + speed: 273.724 + value: {index: 27, type: Single} + - key: [3584, 4, 3328] + speed: 283.607 + value: {index: 27, type: Single} + - key: [3584, 64, 256] + speed: 2607.59 + value: {index: 23, type: Single} + - key: [3584, 64, 768] + speed: 3378.0 + value: {index: 23, type: Single} + - key: [3584, 64, 1792] + speed: 3674.77 + value: {index: 23, type: Single} + - key: [3584, 64, 3328] + speed: 3869.74 + value: {index: 23, type: Single} + - key: [3584, 128, 256] + speed: 3765.81 + value: {index: 24, type: Single} + - key: [3584, 128, 768] + speed: 4367.52 + value: {index: 24, type: Single} + - key: [3584, 128, 1792] + speed: 4708.45 + value: {index: 24, type: Single} + - key: [3584, 128, 3328] + speed: 4902.53 + value: {index: 24, type: Single} + - key: [3584, 256, 256] + speed: 4523.25 + value: {index: 24, type: Single} + - key: [3584, 256, 768] + speed: 4977.72 + value: {index: 24, type: Single} + - key: [3584, 256, 1792] + speed: 5084.92 + value: {index: 21, type: Single} + - key: [3584, 256, 3328] + speed: 5146.77 + value: {index: 21, type: Single} + - key: [3584, 448, 256] + speed: 4880.31 + value: {index: 24, type: Single} + - key: [3584, 448, 768] + speed: 5244.72 + value: {index: 24, type: Single} + - key: [3584, 448, 1792] + speed: 5352.4 + value: {index: 24, type: Single} + - key: [3584, 448, 3328] + speed: 5397.77 + value: {index: 24, type: Single} + - key: [3584, 704, 256] + speed: 5543.38 + value: {index: 24, type: Single} + - key: [3584, 704, 768] + speed: 5885.08 + value: {index: 24, type: Single} + - key: [3584, 704, 1792] + speed: 5975.92 + value: {index: 24, type: Single} + - key: [3584, 704, 3328] + speed: 6012.64 + value: {index: 24, type: Single} + - key: [3584, 1024, 256] + speed: 5592.26 + value: {index: 21, type: Single} + - key: [3584, 1024, 768] + speed: 6120.26 + value: {index: 21, type: Single} + - key: [3584, 1024, 1792] + speed: 6269.96 + value: {index: 21, type: Single} + - key: [3584, 1024, 3328] + speed: 6356.53 + value: {index: 21, type: Single} + - key: [3584, 1408, 256] + speed: 5822.85 + value: {index: 24, type: Single} + - key: [3584, 1408, 768] + speed: 6078.58 + value: {index: 25, type: Single} + - key: [3584, 1408, 1792] + speed: 6213.47 + value: {index: 25, type: Single} + - key: [3584, 1408, 3328] + speed: 6265.21 + value: {index: 25, type: Single} + - key: [3584, 1856, 256] + speed: 5946.29 + value: {index: 24, type: Single} + - key: [3584, 1856, 768] + speed: 6138.96 + value: {index: 24, type: Single} + - key: [3584, 1856, 1792] + speed: 6194.89 + value: {index: 24, type: Single} + - key: [3584, 1856, 3328] + speed: 6215.14 + value: {index: 24, type: Single} + - key: [3584, 2368, 256] + speed: 6018.87 + value: {index: 24, type: Single} + - key: [3584, 2368, 768] + speed: 6194.73 + value: {index: 24, type: Single} + - key: [3584, 2368, 1792] + speed: 6246.68 + value: {index: 24, type: Single} + - key: [3584, 2368, 3328] + speed: 6262.87 + value: {index: 24, type: Single} + - key: [3584, 2944, 256] + speed: 6057.19 + value: {index: 24, type: Single} + - key: [3584, 2944, 768] + speed: 6216.92 + value: {index: 24, type: Single} + - key: [3584, 2944, 1792] + speed: 6263.65 + value: {index: 24, type: Single} + - key: [3584, 2944, 3328] + speed: 6279.28 + value: {index: 24, type: Single} + - key: [3584, 3584, 256] + speed: 6168.02 + value: {index: 24, type: Single} + - key: [3584, 3584, 768] + speed: 6271.44 + value: {index: 21, type: Single} + - key: [3584, 3584, 1792] + speed: 6387.63 + value: {index: 24, type: Single} + - key: [3584, 3584, 3328] + speed: 6425.64 + value: {index: 21, type: Single} + - key: [3584, 4288, 256] + speed: 6165.66 + value: {index: 24, type: Single} + - key: [3584, 4288, 768] + speed: 6356.03 + value: {index: 25, type: Single} + - key: [3584, 4288, 1792] + speed: 6476.39 + value: {index: 25, type: Single} + - key: [3584, 4288, 3328] + speed: 6518.29 + value: {index: 25, type: Single} + - key: [3584, 5056, 256] + speed: 6151.01 + value: {index: 24, type: Single} + - key: [3584, 5056, 768] + speed: 6288.34 + value: {index: 21, type: Single} + - key: [3584, 5056, 1792] + speed: 6424.99 + value: {index: 21, type: Single} + - key: [3584, 5056, 3328] + speed: 6484.15 + value: {index: 21, type: Single} + - key: [3584, 5888, 256] + speed: 6204.05 + value: {index: 24, type: Single} + - key: [3584, 5888, 768] + speed: 6339.41 + value: {index: 21, type: Single} + - key: [3584, 5888, 1792] + speed: 6430.77 + value: {index: 21, type: Single} + - key: [3584, 5888, 3328] + speed: 6468.08 + value: {index: 21, type: Single} + - key: [3584, 6784, 256] + speed: 6224.64 + value: {index: 24, type: Single} + - key: [3584, 6784, 768] + speed: 6379.01 + value: {index: 25, type: Single} + - key: [3584, 6784, 1792] + speed: 6468.9 + value: {index: 21, type: Single} + - key: [3584, 6784, 3328] + speed: 6506.07 + value: {index: 21, type: Single} + - key: [4288, 4, 256] + speed: 225.383 + value: {index: 27, type: Single} + - key: [4288, 4, 768] + speed: 289.152 + value: {index: 27, type: Single} + - key: [4288, 4, 1792] + speed: 311.275 + value: {index: 27, type: Single} + - key: [4288, 4, 3328] + speed: 321.211 + value: {index: 27, type: Single} + - key: [4288, 64, 256] + speed: 2318.86 + value: {index: 24, type: Single} + - key: [4288, 64, 768] + speed: 2741.07 + value: {index: 24, type: Single} + - key: [4288, 64, 1792] + speed: 2894.02 + value: {index: 24, type: Single} + - key: [4288, 64, 3328] + speed: 2941.86 + value: {index: 24, type: Single} + - key: [4288, 128, 256] + speed: 3375.11 + value: {index: 24, type: Single} + - key: [4288, 128, 768] + speed: 3868.43 + value: {index: 24, type: Single} + - key: [4288, 128, 1792] + speed: 4026.01 + value: {index: 24, type: Single} + - key: [4288, 128, 3328] + speed: 4085.95 + value: {index: 24, type: Single} + - key: [4288, 256, 256] + speed: 4473.64 + value: {index: 24, type: Single} + - key: [4288, 256, 768] + speed: 4888.72 + value: {index: 24, type: Single} + - key: [4288, 256, 1792] + speed: 5019.95 + value: {index: 24, type: Single} + - key: [4288, 256, 3328] + speed: 5073.28 + value: {index: 24, type: Single} + - key: [4288, 448, 256] + speed: 5244.0 + value: {index: 24, type: Single} + - key: [4288, 448, 768] + speed: 5550.89 + value: {index: 24, type: Single} + - key: [4288, 448, 1792] + speed: 5646.67 + value: {index: 24, type: Single} + - key: [4288, 448, 3328] + speed: 5679.86 + value: {index: 24, type: Single} + - key: [4288, 704, 256] + speed: 5627.04 + value: {index: 24, type: Single} + - key: [4288, 704, 768] + speed: 5914.13 + value: {index: 24, type: Single} + - key: [4288, 704, 1792] + speed: 5998.17 + value: {index: 24, type: Single} + - key: [4288, 704, 3328] + speed: 6022.59 + value: {index: 24, type: Single} + - key: [4288, 1024, 256] + speed: 5765.41 + value: {index: 24, type: Single} + - key: [4288, 1024, 768] + speed: 5933.75 + value: {index: 24, type: Single} + - key: [4288, 1024, 1792] + speed: 5942.16 + value: {index: 21, type: Single} + - key: [4288, 1024, 3328] + speed: 5987.66 + value: {index: 21, type: Single} + - key: [4288, 1408, 256] + speed: 5860.97 + value: {index: 24, type: Single} + - key: [4288, 1408, 768] + speed: 6089.61 + value: {index: 25, type: Single} + - key: [4288, 1408, 1792] + speed: 6226.78 + value: {index: 25, type: Single} + - key: [4288, 1408, 3328] + speed: 6282.76 + value: {index: 25, type: Single} + - key: [4288, 1856, 256] + speed: 6005.22 + value: {index: 24, type: Single} + - key: [4288, 1856, 768] + speed: 6182.97 + value: {index: 24, type: Single} + - key: [4288, 1856, 1792] + speed: 6231.27 + value: {index: 24, type: Single} + - key: [4288, 1856, 3328] + speed: 6251.77 + value: {index: 25, type: Single} + - key: [4288, 2368, 256] + speed: 6125.19 + value: {index: 24, type: Single} + - key: [4288, 2368, 768] + speed: 6288.0 + value: {index: 24, type: Single} + - key: [4288, 2368, 1792] + speed: 6334.48 + value: {index: 24, type: Single} + - key: [4288, 2368, 3328] + speed: 6350.47 + value: {index: 24, type: Single} + - key: [4288, 2944, 256] + speed: 6105.08 + value: {index: 24, type: Single} + - key: [4288, 2944, 768] + speed: 6242.22 + value: {index: 24, type: Single} + - key: [4288, 2944, 1792] + speed: 6303.3 + value: {index: 21, type: Single} + - key: [4288, 2944, 3328] + speed: 6332.2 + value: {index: 21, type: Single} + - key: [4288, 3584, 256] + speed: 6179.29 + value: {index: 24, type: Single} + - key: [4288, 3584, 768] + speed: 6393.53 + value: {index: 25, type: Single} + - key: [4288, 3584, 1792] + speed: 6492.79 + value: {index: 25, type: Single} + - key: [4288, 3584, 3328] + speed: 6525.99 + value: {index: 25, type: Single} + - key: [4288, 4288, 256] + speed: 6167.48 + value: {index: 24, type: Single} + - key: [4288, 4288, 768] + speed: 6299.55 + value: {index: 21, type: Single} + - key: [4288, 4288, 1792] + speed: 6386.65 + value: {index: 21, type: Single} + - key: [4288, 4288, 3328] + speed: 6415.65 + value: {index: 21, type: Single} + - key: [4288, 5056, 256] + speed: 6219.86 + value: {index: 24, type: Single} + - key: [4288, 5056, 768] + speed: 6356.53 + value: {index: 21, type: Single} + - key: [4288, 5056, 1792] + speed: 6451.94 + value: {index: 21, type: Single} + - key: [4288, 5056, 3328] + speed: 6489.99 + value: {index: 21, type: Single} + - key: [4288, 5888, 256] + speed: 6213.77 + value: {index: 24, type: Single} + - key: [4288, 5888, 768] + speed: 6398.63 + value: {index: 21, type: Single} + - key: [4288, 5888, 1792] + speed: 6467.08 + value: {index: 21, type: Single} + - key: [4288, 5888, 3328] + speed: 6494.14 + value: {index: 21, type: Single} + - key: [4288, 6784, 256] + speed: 6262.87 + value: {index: 24, type: Single} + - key: [4288, 6784, 768] + speed: 6455.23 + value: {index: 21, type: Single} + - key: [4288, 6784, 1792] + speed: 6525.85 + value: {index: 21, type: Single} + - key: [4288, 6784, 3328] + speed: 6550.96 + value: {index: 21, type: Single} + - key: [5056, 4, 256] + speed: 252.313 + value: {index: 27, type: Single} + - key: [5056, 4, 768] + speed: 325.585 + value: {index: 27, type: Single} + - key: [5056, 4, 1792] + speed: 354.783 + value: {index: 27, type: Single} + - key: [5056, 4, 3328] + speed: 359.984 + value: {index: 27, type: Single} + - key: [5056, 64, 256] + speed: 2714.25 + value: {index: 24, type: Single} + - key: [5056, 64, 768] + speed: 3219.6 + value: {index: 24, type: Single} + - key: [5056, 64, 1792] + speed: 3403.45 + value: {index: 24, type: Single} + - key: [5056, 64, 3328] + speed: 3467.11 + value: {index: 24, type: Single} + - key: [5056, 128, 256] + speed: 4000.99 + value: {index: 24, type: Single} + - key: [5056, 128, 768] + speed: 4518.29 + value: {index: 24, type: Single} + - key: [5056, 128, 1792] + speed: 4738.46 + value: {index: 24, type: Single} + - key: [5056, 128, 3328] + speed: 4818.55 + value: {index: 24, type: Single} + - key: [5056, 256, 256] + speed: 5256.31 + value: {index: 24, type: Single} + - key: [5056, 256, 768] + speed: 5761.82 + value: {index: 24, type: Single} + - key: [5056, 256, 1792] + speed: 5917.92 + value: {index: 24, type: Single} + - key: [5056, 256, 3328] + speed: 5956.2 + value: {index: 24, type: Single} + - key: [5056, 448, 256] + speed: 5489.43 + value: {index: 24, type: Single} + - key: [5056, 448, 768] + speed: 5833.03 + value: {index: 24, type: Single} + - key: [5056, 448, 1792] + speed: 5941.83 + value: {index: 24, type: Single} + - key: [5056, 448, 3328] + speed: 5979.03 + value: {index: 24, type: Single} + - key: [5056, 704, 256] + speed: 5756.22 + value: {index: 24, type: Single} + - key: [5056, 704, 768] + speed: 6010.28 + value: {index: 24, type: Single} + - key: [5056, 704, 1792] + speed: 6086.61 + value: {index: 24, type: Single} + - key: [5056, 704, 3328] + speed: 6115.47 + value: {index: 24, type: Single} + - key: [5056, 1024, 256] + speed: 5860.61 + value: {index: 21, type: Single} + - key: [5056, 1024, 768] + speed: 6211.21 + value: {index: 21, type: Single} + - key: [5056, 1024, 1792] + speed: 6370.97 + value: {index: 25, type: Single} + - key: [5056, 1024, 3328] + speed: 6423.15 + value: {index: 25, type: Single} + - key: [5056, 1408, 256] + speed: 5956.91 + value: {index: 24, type: Single} + - key: [5056, 1408, 768] + speed: 6204.81 + value: {index: 25, type: Single} + - key: [5056, 1408, 1792] + speed: 6327.92 + value: {index: 25, type: Single} + - key: [5056, 1408, 3328] + speed: 6380.46 + value: {index: 25, type: Single} + - key: [5056, 1856, 256] + speed: 6123.49 + value: {index: 24, type: Single} + - key: [5056, 1856, 768] + speed: 6288.13 + value: {index: 24, type: Single} + - key: [5056, 1856, 1792] + speed: 6336.28 + value: {index: 24, type: Single} + - key: [5056, 1856, 3328] + speed: 6354.51 + value: {index: 24, type: Single} + - key: [5056, 2368, 256] + speed: 6150.77 + value: {index: 24, type: Single} + - key: [5056, 2368, 768] + speed: 6298.71 + value: {index: 24, type: Single} + - key: [5056, 2368, 1792] + speed: 6342.29 + value: {index: 24, type: Single} + - key: [5056, 2368, 3328] + speed: 6356.32 + value: {index: 24, type: Single} + - key: [5056, 2944, 256] + speed: 6193.94 + value: {index: 24, type: Single} + - key: [5056, 2944, 768] + speed: 6334.54 + value: {index: 21, type: Single} + - key: [5056, 2944, 1792] + speed: 6410.17 + value: {index: 21, type: Single} + - key: [5056, 2944, 3328] + speed: 6442.6 + value: {index: 21, type: Single} + - key: [5056, 3584, 256] + speed: 6164.33 + value: {index: 21, type: Single} + - key: [5056, 3584, 768] + speed: 6399.29 + value: {index: 21, type: Single} + - key: [5056, 3584, 1792] + speed: 6474.52 + value: {index: 21, type: Single} + - key: [5056, 3584, 3328] + speed: 6507.68 + value: {index: 21, type: Single} + - key: [5056, 4288, 256] + speed: 6226.06 + value: {index: 24, type: Single} + - key: [5056, 4288, 768] + speed: 6375.29 + value: {index: 21, type: Single} + - key: [5056, 4288, 1792] + speed: 6461.23 + value: {index: 21, type: Single} + - key: [5056, 4288, 3328] + speed: 6490.9 + value: {index: 21, type: Single} + - key: [5056, 5056, 256] + speed: 6227.97 + value: {index: 24, type: Single} + - key: [5056, 5056, 768] + speed: 6376.77 + value: {index: 25, type: Single} + - key: [5056, 5056, 1792] + speed: 6500.37 + value: {index: 25, type: Single} + - key: [5056, 5056, 3328] + speed: 6533.0 + value: {index: 25, type: Single} + - key: [5056, 5888, 256] + speed: 6283.6 + value: {index: 25, type: Single} + - key: [5056, 5888, 768] + speed: 6488.48 + value: {index: 21, type: Single} + - key: [5056, 5888, 1792] + speed: 6560.0 + value: {index: 25, type: Single} + - key: [5056, 5888, 3328] + speed: 6585.02 + value: {index: 21, type: Single} + - key: [5056, 6784, 256] + speed: 6265.51 + value: {index: 24, type: Single} + - key: [5056, 6784, 768] + speed: 6464.96 + value: {index: 21, type: Single} + - key: [5056, 6784, 1792] + speed: 6532.78 + value: {index: 21, type: Single} + - key: [5056, 6784, 3328] + speed: 6560.53 + value: {index: 21, type: Single} + - key: [5888, 4, 256] + speed: 288.636 + value: {index: 27, type: Single} + - key: [5888, 4, 768] + speed: 366.086 + value: {index: 27, type: Single} + - key: [5888, 4, 1792] + speed: 373.121 + value: {index: 27, type: Single} + - key: [5888, 4, 3328] + speed: 368.682 + value: {index: 27, type: Single} + - key: [5888, 64, 256] + speed: 3007.61 + value: {index: 24, type: Single} + - key: [5888, 64, 768] + speed: 3727.93 + value: {index: 24, type: Single} + - key: [5888, 64, 1792] + speed: 3949.76 + value: {index: 24, type: Single} + - key: [5888, 64, 3328] + speed: 4034.76 + value: {index: 24, type: Single} + - key: [5888, 128, 256] + speed: 4593.65 + value: {index: 24, type: Single} + - key: [5888, 128, 768] + speed: 5268.89 + value: {index: 24, type: Single} + - key: [5888, 128, 1792] + speed: 5501.56 + value: {index: 24, type: Single} + - key: [5888, 128, 3328] + speed: 5598.49 + value: {index: 24, type: Single} + - key: [5888, 256, 256] + speed: 5062.36 + value: {index: 24, type: Single} + - key: [5888, 256, 768] + speed: 5666.27 + value: {index: 24, type: Single} + - key: [5888, 256, 1792] + speed: 5819.39 + value: {index: 24, type: Single} + - key: [5888, 256, 3328] + speed: 5862.13 + value: {index: 24, type: Single} + - key: [5888, 448, 256] + speed: 5365.54 + value: {index: 24, type: Single} + - key: [5888, 448, 768] + speed: 5627.21 + value: {index: 24, type: Single} + - key: [5888, 448, 1792] + speed: 5711.31 + value: {index: 24, type: Single} + - key: [5888, 448, 3328] + speed: 5735.04 + value: {index: 24, type: Single} + - key: [5888, 704, 256] + speed: 5890.32 + value: {index: 24, type: Single} + - key: [5888, 704, 768] + speed: 6142.91 + value: {index: 24, type: Single} + - key: [5888, 704, 1792] + speed: 6219.87 + value: {index: 24, type: Single} + - key: [5888, 704, 3328] + speed: 6244.45 + value: {index: 24, type: Single} + - key: [5888, 1024, 256] + speed: 5768.75 + value: {index: 24, type: Single} + - key: [5888, 1024, 768] + speed: 6115.86 + value: {index: 25, type: Single} + - key: [5888, 1024, 1792] + speed: 6241.75 + value: {index: 25, type: Single} + - key: [5888, 1024, 3328] + speed: 6283.11 + value: {index: 25, type: Single} + - key: [5888, 1408, 256] + speed: 6053.34 + value: {index: 24, type: Single} + - key: [5888, 1408, 768] + speed: 6367.25 + value: {index: 25, type: Single} + - key: [5888, 1408, 1792] + speed: 6485.79 + value: {index: 25, type: Single} + - key: [5888, 1408, 3328] + speed: 6527.23 + value: {index: 25, type: Single} + - key: [5888, 1856, 256] + speed: 6130.94 + value: {index: 24, type: Single} + - key: [5888, 1856, 768] + speed: 6288.14 + value: {index: 24, type: Single} + - key: [5888, 1856, 1792] + speed: 6334.4 + value: {index: 24, type: Single} + - key: [5888, 1856, 3328] + speed: 6350.85 + value: {index: 24, type: Single} + - key: [5888, 2368, 256] + speed: 6118.72 + value: {index: 24, type: Single} + - key: [5888, 2368, 768] + speed: 6255.89 + value: {index: 24, type: Single} + - key: [5888, 2368, 1792] + speed: 6297.51 + value: {index: 24, type: Single} + - key: [5888, 2368, 3328] + speed: 6333.97 + value: {index: 25, type: Single} + - key: [5888, 2944, 256] + speed: 6159.33 + value: {index: 25, type: Single} + - key: [5888, 2944, 768] + speed: 6396.3 + value: {index: 25, type: Single} + - key: [5888, 2944, 1792] + speed: 6482.19 + value: {index: 25, type: Single} + - key: [5888, 2944, 3328] + speed: 6508.11 + value: {index: 25, type: Single} + - key: [5888, 3584, 256] + speed: 6198.78 + value: {index: 24, type: Single} + - key: [5888, 3584, 768] + speed: 6360.6 + value: {index: 21, type: Single} + - key: [5888, 3584, 1792] + speed: 6440.34 + value: {index: 21, type: Single} + - key: [5888, 3584, 3328] + speed: 6474.48 + value: {index: 21, type: Single} + - key: [5888, 4288, 256] + speed: 6213.33 + value: {index: 24, type: Single} + - key: [5888, 4288, 768] + speed: 6351.58 + value: {index: 25, type: Single} + - key: [5888, 4288, 1792] + speed: 6443.95 + value: {index: 21, type: Single} + - key: [5888, 4288, 3328] + speed: 6483.94 + value: {index: 21, type: Single} + - key: [5888, 5056, 256] + speed: 6240.15 + value: {index: 24, type: Single} + - key: [5888, 5056, 768] + speed: 6434.11 + value: {index: 25, type: Single} + - key: [5888, 5056, 1792] + speed: 6542.77 + value: {index: 25, type: Single} + - key: [5888, 5056, 3328] + speed: 6574.83 + value: {index: 25, type: Single} + - key: [5888, 5888, 256] + speed: 6291.28 + value: {index: 25, type: Single} + - key: [5888, 5888, 768] + speed: 6456.16 + value: {index: 25, type: Single} + - key: [5888, 5888, 1792] + speed: 6529.83 + value: {index: 25, type: Single} + - key: [5888, 5888, 3328] + speed: 6549.7 + value: {index: 25, type: Single} + - key: [5888, 6784, 256] + speed: 6317.77 + value: {index: 25, type: Single} + - key: [5888, 6784, 768] + speed: 6497.28 + value: {index: 25, type: Single} + - key: [5888, 6784, 1792] + speed: 6561.44 + value: {index: 25, type: Single} + - key: [5888, 6784, 3328] + speed: 6583.91 + value: {index: 25, type: Single} + - key: [6784, 4, 256] + speed: 294.906 + value: {index: 27, type: Single} + - key: [6784, 4, 768] + speed: 350.796 + value: {index: 27, type: Single} + - key: [6784, 4, 1792] + speed: 354.267 + value: {index: 27, type: Single} + - key: [6784, 4, 3328] + speed: 348.225 + value: {index: 27, type: Single} + - key: [6784, 64, 256] + speed: 3606.92 + value: {index: 24, type: Single} + - key: [6784, 64, 768] + speed: 4303.46 + value: {index: 24, type: Single} + - key: [6784, 64, 1792] + speed: 4548.84 + value: {index: 24, type: Single} + - key: [6784, 64, 3328] + speed: 4643.21 + value: {index: 24, type: Single} + - key: [6784, 128, 256] + speed: 4250.65 + value: {index: 24, type: Single} + - key: [6784, 128, 768] + speed: 4723.42 + value: {index: 24, type: Single} + - key: [6784, 128, 1792] + speed: 4906.93 + value: {index: 24, type: Single} + - key: [6784, 128, 3328] + speed: 4955.2 + value: {index: 24, type: Single} + - key: [6784, 256, 256] + speed: 5283.35 + value: {index: 24, type: Single} + - key: [6784, 256, 768] + speed: 5672.86 + value: {index: 24, type: Single} + - key: [6784, 256, 1792] + speed: 5796.52 + value: {index: 24, type: Single} + - key: [6784, 256, 3328] + speed: 5834.92 + value: {index: 24, type: Single} + - key: [6784, 448, 256] + speed: 5665.21 + value: {index: 24, type: Single} + - key: [6784, 448, 768] + speed: 5948.63 + value: {index: 24, type: Single} + - key: [6784, 448, 1792] + speed: 6033.9 + value: {index: 24, type: Single} + - key: [6784, 448, 3328] + speed: 6070.73 + value: {index: 24, type: Single} + - key: [6784, 704, 256] + speed: 5799.43 + value: {index: 24, type: Single} + - key: [6784, 704, 768] + speed: 6001.16 + value: {index: 24, type: Single} + - key: [6784, 704, 1792] + speed: 6055.04 + value: {index: 24, type: Single} + - key: [6784, 704, 3328] + speed: 6079.43 + value: {index: 24, type: Single} + - key: [6784, 1024, 256] + speed: 5878.44 + value: {index: 24, type: Single} + - key: [6784, 1024, 768] + speed: 6074.78 + value: {index: 25, type: Single} + - key: [6784, 1024, 1792] + speed: 6193.9 + value: {index: 25, type: Single} + - key: [6784, 1024, 3328] + speed: 6232.61 + value: {index: 25, type: Single} + - key: [6784, 1408, 256] + speed: 6077.02 + value: {index: 24, type: Single} + - key: [6784, 1408, 768] + speed: 6229.15 + value: {index: 24, type: Single} + - key: [6784, 1408, 1792] + speed: 6277.78 + value: {index: 24, type: Single} + - key: [6784, 1408, 3328] + speed: 6294.51 + value: {index: 24, type: Single} + - key: [6784, 1856, 256] + speed: 6096.91 + value: {index: 24, type: Single} + - key: [6784, 1856, 768] + speed: 6226.91 + value: {index: 24, type: Single} + - key: [6784, 1856, 1792] + speed: 6266.89 + value: {index: 24, type: Single} + - key: [6784, 1856, 3328] + speed: 6305.56 + value: {index: 21, type: Single} + - key: [6784, 2368, 256] + speed: 6155.61 + value: {index: 24, type: Single} + - key: [6784, 2368, 768] + speed: 6285.94 + value: {index: 24, type: Single} + - key: [6784, 2368, 1792] + speed: 6365.22 + value: {index: 25, type: Single} + - key: [6784, 2368, 3328] + speed: 6395.68 + value: {index: 25, type: Single} + - key: [6784, 2944, 256] + speed: 6180.31 + value: {index: 24, type: Single} + - key: [6784, 2944, 768] + speed: 6345.46 + value: {index: 21, type: Single} + - key: [6784, 2944, 1792] + speed: 6423.69 + value: {index: 21, type: Single} + - key: [6784, 2944, 3328] + speed: 6445.99 + value: {index: 21, type: Single} + - key: [6784, 3584, 256] + speed: 6235.26 + value: {index: 24, type: Single} + - key: [6784, 3584, 768] + speed: 6409.17 + value: {index: 21, type: Single} + - key: [6784, 3584, 1792] + speed: 6487.31 + value: {index: 21, type: Single} + - key: [6784, 3584, 3328] + speed: 6515.93 + value: {index: 21, type: Single} + - key: [6784, 4288, 256] + speed: 6262.48 + value: {index: 24, type: Single} + - key: [6784, 4288, 768] + speed: 6379.45 + value: {index: 24, type: Single} + - key: [6784, 4288, 1792] + speed: 6414.73 + value: {index: 24, type: Single} + - key: [6784, 4288, 3328] + speed: 6435.89 + value: {index: 21, type: Single} + - key: [6784, 5056, 256] + speed: 6265.51 + value: {index: 24, type: Single} + - key: [6784, 5056, 768] + speed: 6377.66 + value: {index: 24, type: Single} + - key: [6784, 5056, 1792] + speed: 6446.5 + value: {index: 25, type: Single} + - key: [6784, 5056, 3328] + speed: 6474.48 + value: {index: 25, type: Single} + - key: [6784, 5888, 256] + speed: 6327.33 + value: {index: 25, type: Single} + - key: [6784, 5888, 768] + speed: 6494.22 + value: {index: 25, type: Single} + - key: [6784, 5888, 1792] + speed: 6562.02 + value: {index: 25, type: Single} + - key: [6784, 5888, 3328] + speed: 6583.77 + value: {index: 25, type: Single} + - key: [6784, 6784, 256] + speed: 6436.66 + value: {index: 25, type: Single} + - key: [6784, 6784, 768] + speed: 6627.17 + value: {index: 25, type: Single} + - key: [6784, 6784, 1792] + speed: 6699.74 + value: {index: 25, type: Single} + - key: [6784, 6784, 3328] + speed: 6723.28 + value: {index: 25, type: Single} + type: Matching + predicate: + type: And + value: + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + type: Problem + Contraction_l_Ailk_Bljk_Cijk_Dijk: + rows: + - library: + distance: Euclidean + properties: + - {index: 0, type: FreeSizeA} + - {index: 0, type: FreeSizeB} + - {index: 0, type: BoundSize} + table: + - key: [4, 4, 256] + speed: 0.239371 + value: {index: 6, type: Single} + - key: [4, 4, 768] + speed: 0.312398 + value: {index: 6, type: Single} + - key: [4, 4, 1792] + speed: 0.357397 + value: {index: 6, type: Single} + - key: [4, 4, 3328] + speed: 0.36882 + value: {index: 6, type: Single} + - key: [4, 64, 256] + speed: 4.7564 + value: {index: 6, type: Single} + - key: [4, 64, 768] + speed: 5.76986 + value: {index: 6, type: Single} + - key: [4, 64, 1792] + speed: 6.06558 + value: {index: 6, type: Single} + - key: [4, 64, 3328] + speed: 6.20683 + value: {index: 6, type: Single} + - key: [4, 128, 256] + speed: 10.2874 + value: {index: 6, type: Single} + - key: [4, 128, 768] + speed: 13.172 + value: {index: 6, type: Single} + - key: [4, 128, 1792] + speed: 12.2755 + value: {index: 6, type: Single} + - key: [4, 128, 3328] + speed: 12.515 + value: {index: 6, type: Single} + - key: [4, 256, 256] + speed: 20.1069 + value: {index: 6, type: Single} + - key: [4, 256, 768] + speed: 27.0145 + value: {index: 6, type: Single} + - key: [4, 256, 1792] + speed: 24.3339 + value: {index: 6, type: Single} + - key: [4, 256, 3328] + speed: 24.9621 + value: {index: 6, type: Single} + - key: [4, 448, 256] + speed: 31.1208 + value: {index: 6, type: Single} + - key: [4, 448, 768] + speed: 41.8442 + value: {index: 6, type: Single} + - key: [4, 448, 1792] + speed: 44.5082 + value: {index: 6, type: Single} + - key: [4, 448, 3328] + speed: 43.66 + value: {index: 6, type: Single} + - key: [4, 704, 256] + speed: 43.8368 + value: {index: 6, type: Single} + - key: [4, 704, 768] + speed: 59.8271 + value: {index: 6, type: Single} + - key: [4, 704, 1792] + speed: 60.8784 + value: {index: 6, type: Single} + - key: [4, 704, 3328] + speed: 58.2745 + value: {index: 6, type: Single} + - key: [4, 1024, 256] + speed: 62.3596 + value: {index: 6, type: Single} + - key: [4, 1024, 768] + speed: 79.8236 + value: {index: 6, type: Single} + - key: [4, 1024, 1792] + speed: 83.6886 + value: {index: 6, type: Single} + - key: [4, 1024, 3328] + speed: 83.7598 + value: {index: 6, type: Single} + - key: [4, 1408, 256] + speed: 82.8235 + value: {index: 6, type: Single} + - key: [4, 1408, 768] + speed: 105.973 + value: {index: 6, type: Single} + - key: [4, 1408, 1792] + speed: 110.41 + value: {index: 6, type: Single} + - key: [4, 1408, 3328] + speed: 113.161 + value: {index: 6, type: Single} + - key: [4, 1856, 256] + speed: 101.411 + value: {index: 6, type: Single} + - key: [4, 1856, 768] + speed: 134.563 + value: {index: 6, type: Single} + - key: [4, 1856, 1792] + speed: 140.419 + value: {index: 6, type: Single} + - key: [4, 1856, 3328] + speed: 145.776 + value: {index: 6, type: Single} + - key: [4, 2368, 256] + speed: 124.94 + value: {index: 6, type: Single} + - key: [4, 2368, 768] + speed: 166.728 + value: {index: 6, type: Single} + - key: [4, 2368, 1792] + speed: 176.125 + value: {index: 6, type: Single} + - key: [4, 2368, 3328] + speed: 172.636 + value: {index: 6, type: Single} + - key: [4, 2944, 256] + speed: 142.298 + value: {index: 6, type: Single} + - key: [4, 2944, 768] + speed: 199.823 + value: {index: 6, type: Single} + - key: [4, 2944, 1792] + speed: 198.382 + value: {index: 6, type: Single} + - key: [4, 2944, 3328] + speed: 200.022 + value: {index: 6, type: Single} + - key: [4, 3584, 256] + speed: 160.856 + value: {index: 6, type: Single} + - key: [4, 3584, 768] + speed: 228.664 + value: {index: 6, type: Single} + - key: [4, 3584, 1792] + speed: 184.766 + value: {index: 6, type: Single} + - key: [4, 3584, 3328] + speed: 178.265 + value: {index: 6, type: Single} + - key: [4, 4288, 256] + speed: 169.36 + value: {index: 6, type: Single} + - key: [4, 4288, 768] + speed: 241.613 + value: {index: 6, type: Single} + - key: [4, 4288, 1792] + speed: 158.31 + value: {index: 6, type: Single} + - key: [4, 4288, 3328] + speed: 167.847 + value: {index: 6, type: Single} + - key: [4, 5056, 256] + speed: 183.444 + value: {index: 6, type: Single} + - key: [4, 5056, 768] + speed: 262.754 + value: {index: 6, type: Single} + - key: [4, 5056, 1792] + speed: 166.807 + value: {index: 6, type: Single} + - key: [4, 5056, 3328] + speed: 171.272 + value: {index: 6, type: Single} + - key: [4, 5888, 256] + speed: 189.729 + value: {index: 6, type: Single} + - key: [4, 5888, 768] + speed: 240.808 + value: {index: 6, type: Single} + - key: [4, 5888, 1792] + speed: 168.966 + value: {index: 6, type: Single} + - key: [4, 5888, 3328] + speed: 170.142 + value: {index: 6, type: Single} + - key: [4, 6784, 256] + speed: 197.847 + value: {index: 6, type: Single} + - key: [4, 6784, 768] + speed: 239.437 + value: {index: 6, type: Single} + - key: [4, 6784, 1792] + speed: 165.772 + value: {index: 6, type: Single} + - key: [4, 6784, 3328] + speed: 170.009 + value: {index: 6, type: Single} + - key: [64, 4, 256] + speed: 4.02148 + value: {index: 6, type: Single} + - key: [64, 4, 768] + speed: 5.24533 + value: {index: 6, type: Single} + - key: [64, 4, 1792] + speed: 5.85348 + value: {index: 6, type: Single} + - key: [64, 4, 3328] + speed: 6.07247 + value: {index: 6, type: Single} + - key: [64, 64, 256] + speed: 55.9494 + value: {index: 3, type: Single} + - key: [64, 64, 768] + speed: 93.7442 + value: {index: 3, type: Single} + - key: [64, 64, 1792] + speed: 114.025 + value: {index: 3, type: Single} + - key: [64, 64, 3328] + speed: 117.061 + value: {index: 3, type: Single} + - key: [64, 128, 256] + speed: 148.224 + value: {index: 3, type: Single} + - key: [64, 128, 768] + speed: 203.673 + value: {index: 3, type: Single} + - key: [64, 128, 1792] + speed: 223.674 + value: {index: 3, type: Single} + - key: [64, 128, 3328] + speed: 219.724 + value: {index: 3, type: Single} + - key: [64, 256, 256] + speed: 301.174 + value: {index: 3, type: Single} + - key: [64, 256, 768] + speed: 408.324 + value: {index: 3, type: Single} + - key: [64, 256, 1792] + speed: 433.171 + value: {index: 3, type: Single} + - key: [64, 256, 3328] + speed: 407.124 + value: {index: 3, type: Single} + - key: [64, 448, 256] + speed: 483.358 + value: {index: 3, type: Single} + - key: [64, 448, 768] + speed: 704.418 + value: {index: 3, type: Single} + - key: [64, 448, 1792] + speed: 681.349 + value: {index: 3, type: Single} + - key: [64, 448, 3328] + speed: 714.441 + value: {index: 3, type: Single} + - key: [64, 704, 256] + speed: 745.016 + value: {index: 3, type: Single} + - key: [64, 704, 768] + speed: 1035.77 + value: {index: 3, type: Single} + - key: [64, 704, 1792] + speed: 1046.03 + value: {index: 3, type: Single} + - key: [64, 704, 3328] + speed: 1111.6 + value: {index: 3, type: Single} + - key: [64, 1024, 256] + speed: 989.019 + value: {index: 3, type: Single} + - key: [64, 1024, 768] + speed: 1412.6 + value: {index: 3, type: Single} + - key: [64, 1024, 1792] + speed: 1524.43 + value: {index: 3, type: Single} + - key: [64, 1024, 3328] + speed: 1593.24 + value: {index: 3, type: Single} + - key: [64, 1408, 256] + speed: 1359.9 + value: {index: 3, type: Single} + - key: [64, 1408, 768] + speed: 1835.46 + value: {index: 3, type: Single} + - key: [64, 1408, 1792] + speed: 2033.52 + value: {index: 3, type: Single} + - key: [64, 1408, 3328] + speed: 2142.03 + value: {index: 3, type: Single} + - key: [64, 1856, 256] + speed: 1769.44 + value: {index: 3, type: Single} + - key: [64, 1856, 768] + speed: 2267.98 + value: {index: 3, type: Single} + - key: [64, 1856, 1792] + speed: 2522.87 + value: {index: 3, type: Single} + - key: [64, 1856, 3328] + speed: 2725.53 + value: {index: 3, type: Single} + - key: [64, 2368, 256] + speed: 1837.69 + value: {index: 3, type: Single} + - key: [64, 2368, 768] + speed: 2331.22 + value: {index: 3, type: Single} + - key: [64, 2368, 1792] + speed: 2600.17 + value: {index: 3, type: Single} + - key: [64, 2368, 3328] + speed: 2741.03 + value: {index: 3, type: Single} + - key: [64, 2944, 256] + speed: 2268.84 + value: {index: 3, type: Single} + - key: [64, 2944, 768] + speed: 2743.61 + value: {index: 3, type: Single} + - key: [64, 2944, 1792] + speed: 3115.54 + value: {index: 3, type: Single} + - key: [64, 2944, 3328] + speed: 3315.67 + value: {index: 3, type: Single} + - key: [64, 3584, 256] + speed: 2508.56 + value: {index: 3, type: Single} + - key: [64, 3584, 768] + speed: 3068.52 + value: {index: 3, type: Single} + - key: [64, 3584, 1792] + speed: 3679.64 + value: {index: 3, type: Single} + - key: [64, 3584, 3328] + speed: 3909.37 + value: {index: 3, type: Single} + - key: [64, 4288, 256] + speed: 1996.66 + value: {index: 3, type: Single} + - key: [64, 4288, 768] + speed: 2489.27 + value: {index: 3, type: Single} + - key: [64, 4288, 1792] + speed: 2805.95 + value: {index: 3, type: Single} + - key: [64, 4288, 3328] + speed: 2960.93 + value: {index: 3, type: Single} + - key: [64, 5056, 256] + speed: 1948.22 + value: {index: 5, type: Single} + - key: [64, 5056, 768] + speed: 2819.17 + value: {index: 3, type: Single} + - key: [64, 5056, 1792] + speed: 3182.08 + value: {index: 3, type: Single} + - key: [64, 5056, 3328] + speed: 3453.93 + value: {index: 3, type: Single} + - key: [64, 5888, 256] + speed: 2163.27 + value: {index: 5, type: Single} + - key: [64, 5888, 768] + speed: 3148.17 + value: {index: 3, type: Single} + - key: [64, 5888, 1792] + speed: 3662.54 + value: {index: 3, type: Single} + - key: [64, 5888, 3328] + speed: 3945.43 + value: {index: 3, type: Single} + - key: [64, 6784, 256] + speed: 2174.6 + value: {index: 5, type: Single} + - key: [64, 6784, 768] + speed: 3095.87 + value: {index: 3, type: Single} + - key: [64, 6784, 1792] + speed: 3609.38 + value: {index: 5, type: Single} + - key: [64, 6784, 3328] + speed: 3843.57 + value: {index: 5, type: Single} + - key: [128, 4, 256] + speed: 10.4083 + value: {index: 6, type: Single} + - key: [128, 4, 768] + speed: 13.4728 + value: {index: 6, type: Single} + - key: [128, 4, 1792] + speed: 14.2696 + value: {index: 6, type: Single} + - key: [128, 4, 3328] + speed: 13.1218 + value: {index: 6, type: Single} + - key: [128, 64, 256] + speed: 142.984 + value: {index: 3, type: Single} + - key: [128, 64, 768] + speed: 206.149 + value: {index: 3, type: Single} + - key: [128, 64, 1792] + speed: 227.526 + value: {index: 3, type: Single} + - key: [128, 64, 3328] + speed: 219.464 + value: {index: 3, type: Single} + - key: [128, 128, 256] + speed: 307.726 + value: {index: 3, type: Single} + - key: [128, 128, 768] + speed: 411.294 + value: {index: 3, type: Single} + - key: [128, 128, 1792] + speed: 450.91 + value: {index: 3, type: Single} + - key: [128, 128, 3328] + speed: 404.884 + value: {index: 3, type: Single} + - key: [128, 256, 256] + speed: 605.588 + value: {index: 3, type: Single} + - key: [128, 256, 768] + speed: 797.485 + value: {index: 3, type: Single} + - key: [128, 256, 1792] + speed: 813.864 + value: {index: 3, type: Single} + - key: [128, 256, 3328] + speed: 816.504 + value: {index: 3, type: Single} + - key: [128, 448, 256] + speed: 921.738 + value: {index: 3, type: Single} + - key: [128, 448, 768] + speed: 1412.18 + value: {index: 3, type: Single} + - key: [128, 448, 1792] + speed: 1369.43 + value: {index: 3, type: Single} + - key: [128, 448, 3328] + speed: 1403.2 + value: {index: 3, type: Single} + - key: [128, 704, 256] + speed: 1371.91 + value: {index: 3, type: Single} + - key: [128, 704, 768] + speed: 2009.15 + value: {index: 3, type: Single} + - key: [128, 704, 1792] + speed: 2076.12 + value: {index: 3, type: Single} + - key: [128, 704, 3328] + speed: 2169.58 + value: {index: 3, type: Single} + - key: [128, 1024, 256] + speed: 2167.32 + value: {index: 3, type: Single} + - key: [128, 1024, 768] + speed: 2675.05 + value: {index: 3, type: Single} + - key: [128, 1024, 1792] + speed: 2874.71 + value: {index: 3, type: Single} + - key: [128, 1024, 3328] + speed: 2993.71 + value: {index: 3, type: Single} + - key: [128, 1408, 256] + speed: 2193.05 + value: {index: 4, type: Single} + - key: [128, 1408, 768] + speed: 2874.63 + value: {index: 3, type: Single} + - key: [128, 1408, 1792] + speed: 3127.58 + value: {index: 3, type: Single} + - key: [128, 1408, 3328] + speed: 3270.14 + value: {index: 3, type: Single} + - key: [128, 1856, 256] + speed: 2831.09 + value: {index: 3, type: Single} + - key: [128, 1856, 768] + speed: 3698.27 + value: {index: 3, type: Single} + - key: [128, 1856, 1792] + speed: 4099.21 + value: {index: 3, type: Single} + - key: [128, 1856, 3328] + speed: 4288.13 + value: {index: 3, type: Single} + - key: [128, 2368, 256] + speed: 2573.7 + value: {index: 3, type: Single} + - key: [128, 2368, 768] + speed: 3045.06 + value: {index: 3, type: Single} + - key: [128, 2368, 1792] + speed: 3326.9 + value: {index: 3, type: Single} + - key: [128, 2368, 3328] + speed: 3433.55 + value: {index: 3, type: Single} + - key: [128, 2944, 256] + speed: 2724.46 + value: {index: 3, type: Single} + - key: [128, 2944, 768] + speed: 3641.09 + value: {index: 3, type: Single} + - key: [128, 2944, 1792] + speed: 3984.29 + value: {index: 3, type: Single} + - key: [128, 2944, 3328] + speed: 4209.3 + value: {index: 3, type: Single} + - key: [128, 3584, 256] + speed: 3151.92 + value: {index: 5, type: Single} + - key: [128, 3584, 768] + speed: 3924.28 + value: {index: 5, type: Single} + - key: [128, 3584, 1792] + speed: 4161.16 + value: {index: 5, type: Single} + - key: [128, 3584, 3328] + speed: 4242.53 + value: {index: 3, type: Single} + - key: [128, 4288, 256] + speed: 2973.08 + value: {index: 3, type: Single} + - key: [128, 4288, 768] + speed: 3585.67 + value: {index: 3, type: Single} + - key: [128, 4288, 1792] + speed: 3885.79 + value: {index: 3, type: Single} + - key: [128, 4288, 3328] + speed: 3995.89 + value: {index: 3, type: Single} + - key: [128, 5056, 256] + speed: 3289.03 + value: {index: 3, type: Single} + - key: [128, 5056, 768] + speed: 4059.07 + value: {index: 3, type: Single} + - key: [128, 5056, 1792] + speed: 4516.97 + value: {index: 3, type: Single} + - key: [128, 5056, 3328] + speed: 4659.48 + value: {index: 3, type: Single} + - key: [128, 5888, 256] + speed: 3427.08 + value: {index: 5, type: Single} + - key: [128, 5888, 768] + speed: 4212.27 + value: {index: 3, type: Single} + - key: [128, 5888, 1792] + speed: 4538.74 + value: {index: 3, type: Single} + - key: [128, 5888, 3328] + speed: 4684.51 + value: {index: 3, type: Single} + - key: [128, 6784, 256] + speed: 3808.27 + value: {index: 5, type: Single} + - key: [128, 6784, 768] + speed: 4487.94 + value: {index: 5, type: Single} + - key: [128, 6784, 1792] + speed: 4651.58 + value: {index: 5, type: Single} + - key: [128, 6784, 3328] + speed: 4750.64 + value: {index: 0, type: Single} + - key: [256, 4, 256] + speed: 19.7703 + value: {index: 6, type: Single} + - key: [256, 4, 768] + speed: 25.9574 + value: {index: 6, type: Single} + - key: [256, 4, 1792] + speed: 28.7378 + value: {index: 6, type: Single} + - key: [256, 4, 3328] + speed: 25.4031 + value: {index: 6, type: Single} + - key: [256, 64, 256] + speed: 298.007 + value: {index: 3, type: Single} + - key: [256, 64, 768] + speed: 413.293 + value: {index: 3, type: Single} + - key: [256, 64, 1792] + speed: 442.848 + value: {index: 3, type: Single} + - key: [256, 64, 3328] + speed: 407.575 + value: {index: 3, type: Single} + - key: [256, 128, 256] + speed: 526.708 + value: {index: 3, type: Single} + - key: [256, 128, 768] + speed: 814.705 + value: {index: 3, type: Single} + - key: [256, 128, 1792] + speed: 837.951 + value: {index: 3, type: Single} + - key: [256, 128, 3328] + speed: 830.789 + value: {index: 3, type: Single} + - key: [256, 256, 256] + speed: 1048.54 + value: {index: 3, type: Single} + - key: [256, 256, 768] + speed: 1551.29 + value: {index: 3, type: Single} + - key: [256, 256, 1792] + speed: 1585.41 + value: {index: 3, type: Single} + - key: [256, 256, 3328] + speed: 1624.0 + value: {index: 3, type: Single} + - key: [256, 448, 256] + speed: 1826.45 + value: {index: 3, type: Single} + - key: [256, 448, 768] + speed: 2551.61 + value: {index: 3, type: Single} + - key: [256, 448, 1792] + speed: 2683.22 + value: {index: 3, type: Single} + - key: [256, 448, 3328] + speed: 2812.53 + value: {index: 3, type: Single} + - key: [256, 704, 256] + speed: 2208.64 + value: {index: 4, type: Single} + - key: [256, 704, 768] + speed: 2965.89 + value: {index: 3, type: Single} + - key: [256, 704, 1792] + speed: 3189.36 + value: {index: 3, type: Single} + - key: [256, 704, 3328] + speed: 3298.12 + value: {index: 3, type: Single} + - key: [256, 1024, 256] + speed: 3123.96 + value: {index: 3, type: Single} + - key: [256, 1024, 768] + speed: 4155.69 + value: {index: 3, type: Single} + - key: [256, 1024, 1792] + speed: 4523.25 + value: {index: 3, type: Single} + - key: [256, 1024, 3328] + speed: 4737.44 + value: {index: 3, type: Single} + - key: [256, 1408, 256] + speed: 3053.08 + value: {index: 3, type: Single} + - key: [256, 1408, 768] + speed: 3718.4 + value: {index: 3, type: Single} + - key: [256, 1408, 1792] + speed: 3992.54 + value: {index: 3, type: Single} + - key: [256, 1408, 3328] + speed: 4108.0 + value: {index: 3, type: Single} + - key: [256, 1856, 256] + speed: 3297.23 + value: {index: 3, type: Single} + - key: [256, 1856, 768] + speed: 4047.7 + value: {index: 3, type: Single} + - key: [256, 1856, 1792] + speed: 4332.52 + value: {index: 3, type: Single} + - key: [256, 1856, 3328] + speed: 4437.9 + value: {index: 3, type: Single} + - key: [256, 2368, 256] + speed: 3434.42 + value: {index: 3, type: Single} + - key: [256, 2368, 768] + speed: 4094.45 + value: {index: 3, type: Single} + - key: [256, 2368, 1792] + speed: 4346.46 + value: {index: 3, type: Single} + - key: [256, 2368, 3328] + speed: 4460.36 + value: {index: 3, type: Single} + - key: [256, 2944, 256] + speed: 3683.99 + value: {index: 3, type: Single} + - key: [256, 2944, 768] + speed: 4333.77 + value: {index: 3, type: Single} + - key: [256, 2944, 1792] + speed: 4664.14 + value: {index: 3, type: Single} + - key: [256, 2944, 3328] + speed: 4773.67 + value: {index: 3, type: Single} + - key: [256, 3584, 256] + speed: 4279.08 + value: {index: 0, type: Single} + - key: [256, 3584, 768] + speed: 4794.57 + value: {index: 0, type: Single} + - key: [256, 3584, 1792] + speed: 5012.56 + value: {index: 0, type: Single} + - key: [256, 3584, 3328] + speed: 5075.78 + value: {index: 0, type: Single} + - key: [256, 4288, 256] + speed: 3708.34 + value: {index: 3, type: Single} + - key: [256, 4288, 768] + speed: 4355.5 + value: {index: 3, type: Single} + - key: [256, 4288, 1792] + speed: 4608.73 + value: {index: 3, type: Single} + - key: [256, 4288, 3328] + speed: 4714.85 + value: {index: 3, type: Single} + - key: [256, 5056, 256] + speed: 4219.89 + value: {index: 0, type: Single} + - key: [256, 5056, 768] + speed: 4834.04 + value: {index: 0, type: Single} + - key: [256, 5056, 1792] + speed: 5001.07 + value: {index: 0, type: Single} + - key: [256, 5056, 3328] + speed: 5036.84 + value: {index: 0, type: Single} + - key: [256, 5888, 256] + speed: 4841.24 + value: {index: 0, type: Single} + - key: [256, 5888, 768] + speed: 5567.35 + value: {index: 0, type: Single} + - key: [256, 5888, 1792] + speed: 5796.25 + value: {index: 0, type: Single} + - key: [256, 5888, 3328] + speed: 5873.82 + value: {index: 0, type: Single} + - key: [256, 6784, 256] + speed: 4489.06 + value: {index: 0, type: Single} + - key: [256, 6784, 768] + speed: 5005.74 + value: {index: 0, type: Single} + - key: [256, 6784, 1792] + speed: 5137.97 + value: {index: 0, type: Single} + - key: [256, 6784, 3328] + speed: 5190.9 + value: {index: 0, type: Single} + - key: [448, 4, 256] + speed: 28.4075 + value: {index: 6, type: Single} + - key: [448, 4, 768] + speed: 37.916 + value: {index: 6, type: Single} + - key: [448, 4, 1792] + speed: 42.0067 + value: {index: 6, type: Single} + - key: [448, 4, 3328] + speed: 39.3685 + value: {index: 6, type: Single} + - key: [448, 64, 256] + speed: 516.086 + value: {index: 3, type: Single} + - key: [448, 64, 768] + speed: 716.299 + value: {index: 3, type: Single} + - key: [448, 64, 1792] + speed: 707.049 + value: {index: 3, type: Single} + - key: [448, 64, 3328] + speed: 715.234 + value: {index: 3, type: Single} + - key: [448, 128, 256] + speed: 896.711 + value: {index: 3, type: Single} + - key: [448, 128, 768] + speed: 1385.83 + value: {index: 3, type: Single} + - key: [448, 128, 1792] + speed: 1388.62 + value: {index: 3, type: Single} + - key: [448, 128, 3328] + speed: 1420.22 + value: {index: 3, type: Single} + - key: [448, 256, 256] + speed: 1708.42 + value: {index: 3, type: Single} + - key: [448, 256, 768] + speed: 2684.11 + value: {index: 3, type: Single} + - key: [448, 256, 1792] + speed: 2717.4 + value: {index: 3, type: Single} + - key: [448, 256, 3328] + speed: 2771.68 + value: {index: 3, type: Single} + - key: [448, 448, 256] + speed: 2391.78 + value: {index: 3, type: Single} + - key: [448, 448, 768] + speed: 3201.26 + value: {index: 3, type: Single} + - key: [448, 448, 1792] + speed: 3528.57 + value: {index: 3, type: Single} + - key: [448, 448, 3328] + speed: 3646.17 + value: {index: 3, type: Single} + - key: [448, 704, 256] + speed: 2831.06 + value: {index: 3, type: Single} + - key: [448, 704, 768] + speed: 3309.62 + value: {index: 3, type: Single} + - key: [448, 704, 1792] + speed: 3514.4 + value: {index: 3, type: Single} + - key: [448, 704, 3328] + speed: 3601.81 + value: {index: 3, type: Single} + - key: [448, 1024, 256] + speed: 3248.79 + value: {index: 3, type: Single} + - key: [448, 1024, 768] + speed: 3934.0 + value: {index: 3, type: Single} + - key: [448, 1024, 1792] + speed: 4183.12 + value: {index: 3, type: Single} + - key: [448, 1024, 3328] + speed: 4291.11 + value: {index: 3, type: Single} + - key: [448, 1408, 256] + speed: 3603.19 + value: {index: 3, type: Single} + - key: [448, 1408, 768] + speed: 4316.68 + value: {index: 3, type: Single} + - key: [448, 1408, 1792] + speed: 4500.0 + value: {index: 3, type: Single} + - key: [448, 1408, 3328] + speed: 4610.22 + value: {index: 3, type: Single} + - key: [448, 1856, 256] + speed: 3771.05 + value: {index: 5, type: Single} + - key: [448, 1856, 768] + speed: 4278.22 + value: {index: 0, type: Single} + - key: [448, 1856, 1792] + speed: 4488.91 + value: {index: 0, type: Single} + - key: [448, 1856, 3328] + speed: 4566.2 + value: {index: 0, type: Single} + - key: [448, 2368, 256] + speed: 3684.68 + value: {index: 3, type: Single} + - key: [448, 2368, 768] + speed: 4264.72 + value: {index: 3, type: Single} + - key: [448, 2368, 1792] + speed: 4420.2 + value: {index: 3, type: Single} + - key: [448, 2368, 3328] + speed: 4511.66 + value: {index: 3, type: Single} + - key: [448, 2944, 256] + speed: 4340.98 + value: {index: 5, type: Single} + - key: [448, 2944, 768] + speed: 4808.04 + value: {index: 0, type: Single} + - key: [448, 2944, 1792] + speed: 5016.7 + value: {index: 0, type: Single} + - key: [448, 2944, 3328] + speed: 5091.9 + value: {index: 0, type: Single} + - key: [448, 3584, 256] + speed: 4104.22 + value: {index: 5, type: Single} + - key: [448, 3584, 768] + speed: 4521.11 + value: {index: 0, type: Single} + - key: [448, 3584, 1792] + speed: 4720.75 + value: {index: 3, type: Single} + - key: [448, 3584, 3328] + speed: 4797.85 + value: {index: 3, type: Single} + - key: [448, 4288, 256] + speed: 4705.08 + value: {index: 5, type: Single} + - key: [448, 4288, 768] + speed: 5198.8 + value: {index: 5, type: Single} + - key: [448, 4288, 1792] + speed: 5357.63 + value: {index: 5, type: Single} + - key: [448, 4288, 3328] + speed: 5424.26 + value: {index: 5, type: Single} + - key: [448, 5056, 256] + speed: 4648.4 + value: {index: 5, type: Single} + - key: [448, 5056, 768] + speed: 5197.82 + value: {index: 0, type: Single} + - key: [448, 5056, 1792] + speed: 5399.1 + value: {index: 0, type: Single} + - key: [448, 5056, 3328] + speed: 5470.24 + value: {index: 0, type: Single} + - key: [448, 5888, 256] + speed: 4701.41 + value: {index: 5, type: Single} + - key: [448, 5888, 768] + speed: 5110.86 + value: {index: 0, type: Single} + - key: [448, 5888, 1792] + speed: 5308.85 + value: {index: 0, type: Single} + - key: [448, 5888, 3328] + speed: 5369.92 + value: {index: 0, type: Single} + - key: [448, 6784, 256] + speed: 5272.73 + value: {index: 5, type: Single} + - key: [448, 6784, 768] + speed: 5664.18 + value: {index: 5, type: Single} + - key: [448, 6784, 1792] + speed: 5791.04 + value: {index: 5, type: Single} + - key: [448, 6784, 3328] + speed: 5832.92 + value: {index: 5, type: Single} + - key: [704, 4, 256] + speed: 41.4118 + value: {index: 6, type: Single} + - key: [704, 4, 768] + speed: 56.3618 + value: {index: 6, type: Single} + - key: [704, 4, 1792] + speed: 61.0419 + value: {index: 6, type: Single} + - key: [704, 4, 3328] + speed: 58.8437 + value: {index: 6, type: Single} + - key: [704, 64, 256] + speed: 819.52 + value: {index: 3, type: Single} + - key: [704, 64, 768] + speed: 1109.57 + value: {index: 3, type: Single} + - key: [704, 64, 1792] + speed: 1082.38 + value: {index: 3, type: Single} + - key: [704, 64, 3328] + speed: 1123.94 + value: {index: 3, type: Single} + - key: [704, 128, 256] + speed: 1384.09 + value: {index: 3, type: Single} + - key: [704, 128, 768] + speed: 2152.66 + value: {index: 3, type: Single} + - key: [704, 128, 1792] + speed: 2141.38 + value: {index: 3, type: Single} + - key: [704, 128, 3328] + speed: 2205.03 + value: {index: 3, type: Single} + - key: [704, 256, 256] + speed: 2216.49 + value: {index: 4, type: Single} + - key: [704, 256, 768] + speed: 2951.85 + value: {index: 3, type: Single} + - key: [704, 256, 1792] + speed: 3196.37 + value: {index: 3, type: Single} + - key: [704, 256, 3328] + speed: 3255.68 + value: {index: 3, type: Single} + - key: [704, 448, 256] + speed: 2868.32 + value: {index: 3, type: Single} + - key: [704, 448, 768] + speed: 3326.44 + value: {index: 3, type: Single} + - key: [704, 448, 1792] + speed: 3496.67 + value: {index: 3, type: Single} + - key: [704, 448, 3328] + speed: 3584.5 + value: {index: 3, type: Single} + - key: [704, 704, 256] + speed: 3425.61 + value: {index: 3, type: Single} + - key: [704, 704, 768] + speed: 4292.74 + value: {index: 3, type: Single} + - key: [704, 704, 1792] + speed: 4522.67 + value: {index: 3, type: Single} + - key: [704, 704, 3328] + speed: 4611.93 + value: {index: 3, type: Single} + - key: [704, 1024, 256] + speed: 3595.0 + value: {index: 3, type: Single} + - key: [704, 1024, 768] + speed: 4263.57 + value: {index: 3, type: Single} + - key: [704, 1024, 1792] + speed: 4463.64 + value: {index: 3, type: Single} + - key: [704, 1024, 3328] + speed: 4550.08 + value: {index: 3, type: Single} + - key: [704, 1408, 256] + speed: 4705.49 + value: {index: 5, type: Single} + - key: [704, 1408, 768] + speed: 5169.42 + value: {index: 5, type: Single} + - key: [704, 1408, 1792] + speed: 5333.45 + value: {index: 5, type: Single} + - key: [704, 1408, 3328] + speed: 5391.38 + value: {index: 5, type: Single} + - key: [704, 1856, 256] + speed: 4341.87 + value: {index: 0, type: Single} + - key: [704, 1856, 768] + speed: 4883.46 + value: {index: 0, type: Single} + - key: [704, 1856, 1792] + speed: 5026.07 + value: {index: 0, type: Single} + - key: [704, 1856, 3328] + speed: 5085.09 + value: {index: 0, type: Single} + - key: [704, 2368, 256] + speed: 4322.01 + value: {index: 0, type: Single} + - key: [704, 2368, 768] + speed: 4753.5 + value: {index: 0, type: Single} + - key: [704, 2368, 1792] + speed: 4924.73 + value: {index: 0, type: Single} + - key: [704, 2368, 3328] + speed: 4973.84 + value: {index: 0, type: Single} + - key: [704, 2944, 256] + speed: 5251.18 + value: {index: 5, type: Single} + - key: [704, 2944, 768] + speed: 5656.19 + value: {index: 5, type: Single} + - key: [704, 2944, 1792] + speed: 5807.08 + value: {index: 5, type: Single} + - key: [704, 2944, 3328] + speed: 5865.45 + value: {index: 5, type: Single} + - key: [704, 3584, 256] + speed: 5256.0 + value: {index: 5, type: Single} + - key: [704, 3584, 768] + speed: 5581.2 + value: {index: 5, type: Single} + - key: [704, 3584, 1792] + speed: 5723.74 + value: {index: 5, type: Single} + - key: [704, 3584, 3328] + speed: 5765.83 + value: {index: 5, type: Single} + - key: [704, 4288, 256] + speed: 5213.65 + value: {index: 5, type: Single} + - key: [704, 4288, 768] + speed: 5572.92 + value: {index: 5, type: Single} + - key: [704, 4288, 1792] + speed: 5733.95 + value: {index: 5, type: Single} + - key: [704, 4288, 3328] + speed: 5789.41 + value: {index: 5, type: Single} + - key: [704, 5056, 256] + speed: 5247.86 + value: {index: 5, type: Single} + - key: [704, 5056, 768] + speed: 5653.04 + value: {index: 5, type: Single} + - key: [704, 5056, 1792] + speed: 5824.35 + value: {index: 5, type: Single} + - key: [704, 5056, 3328] + speed: 5881.97 + value: {index: 5, type: Single} + - key: [704, 5888, 256] + speed: 5571.85 + value: {index: 5, type: Single} + - key: [704, 5888, 768] + speed: 5847.83 + value: {index: 5, type: Single} + - key: [704, 5888, 1792] + speed: 5973.11 + value: {index: 5, type: Single} + - key: [704, 5888, 3328] + speed: 6027.77 + value: {index: 5, type: Single} + - key: [704, 6784, 256] + speed: 5338.02 + value: {index: 0, type: Single} + - key: [704, 6784, 768] + speed: 5798.75 + value: {index: 0, type: Single} + - key: [704, 6784, 1792] + speed: 5932.84 + value: {index: 0, type: Single} + - key: [704, 6784, 3328] + speed: 5992.66 + value: {index: 0, type: Single} + - key: [1024, 4, 256] + speed: 59.2265 + value: {index: 6, type: Single} + - key: [1024, 4, 768] + speed: 77.3514 + value: {index: 6, type: Single} + - key: [1024, 4, 1792] + speed: 82.7801 + value: {index: 6, type: Single} + - key: [1024, 4, 3328] + speed: 82.1518 + value: {index: 6, type: Single} + - key: [1024, 64, 256] + speed: 1029.5 + value: {index: 3, type: Single} + - key: [1024, 64, 768] + speed: 1202.58 + value: {index: 3, type: Single} + - key: [1024, 64, 1792] + speed: 1241.51 + value: {index: 3, type: Single} + - key: [1024, 64, 3328] + speed: 1282.37 + value: {index: 3, type: Single} + - key: [1024, 128, 256] + speed: 1671.5 + value: {index: 4, type: Single} + - key: [1024, 128, 768] + speed: 2034.32 + value: {index: 1, type: Single} + - key: [1024, 128, 1792] + speed: 2140.98 + value: {index: 1, type: Single} + - key: [1024, 128, 3328] + speed: 2249.29 + value: {index: 1, type: Single} + - key: [1024, 256, 256] + speed: 2912.96 + value: {index: 1, type: Single} + - key: [1024, 256, 768] + speed: 3623.78 + value: {index: 1, type: Single} + - key: [1024, 256, 1792] + speed: 3907.34 + value: {index: 1, type: Single} + - key: [1024, 256, 3328] + speed: 4025.04 + value: {index: 1, type: Single} + - key: [1024, 448, 256] + speed: 3209.33 + value: {index: 0, type: Single} + - key: [1024, 448, 768] + speed: 3885.8 + value: {index: 0, type: Single} + - key: [1024, 448, 1792] + speed: 4077.09 + value: {index: 5, type: Single} + - key: [1024, 448, 3328] + speed: 4189.07 + value: {index: 5, type: Single} + - key: [1024, 704, 256] + speed: 3426.88 + value: {index: 0, type: Single} + - key: [1024, 704, 768] + speed: 3821.08 + value: {index: 0, type: Single} + - key: [1024, 704, 1792] + speed: 4025.72 + value: {index: 3, type: Single} + - key: [1024, 704, 3328] + speed: 4072.87 + value: {index: 1, type: Single} + - key: [1024, 1024, 256] + speed: 4724.6 + value: {index: 0, type: Single} + - key: [1024, 1024, 768] + speed: 5344.8 + value: {index: 5, type: Single} + - key: [1024, 1024, 1792] + speed: 5564.03 + value: {index: 0, type: Single} + - key: [1024, 1024, 3328] + speed: 5664.87 + value: {index: 0, type: Single} + - key: [1024, 1408, 256] + speed: 4777.27 + value: {index: 0, type: Single} + - key: [1024, 1408, 768] + speed: 5336.69 + value: {index: 0, type: Single} + - key: [1024, 1408, 1792] + speed: 5542.48 + value: {index: 0, type: Single} + - key: [1024, 1408, 3328] + speed: 5597.56 + value: {index: 0, type: Single} + - key: [1024, 1856, 256] + speed: 4923.61 + value: {index: 0, type: Single} + - key: [1024, 1856, 768] + speed: 5391.05 + value: {index: 0, type: Single} + - key: [1024, 1856, 1792] + speed: 5578.34 + value: {index: 0, type: Single} + - key: [1024, 1856, 3328] + speed: 5656.89 + value: {index: 0, type: Single} + - key: [1024, 2368, 256] + speed: 5147.41 + value: {index: 0, type: Single} + - key: [1024, 2368, 768] + speed: 5629.18 + value: {index: 0, type: Single} + - key: [1024, 2368, 1792] + speed: 5811.36 + value: {index: 0, type: Single} + - key: [1024, 2368, 3328] + speed: 5867.38 + value: {index: 0, type: Single} + - key: [1024, 2944, 256] + speed: 5409.31 + value: {index: 0, type: Single} + - key: [1024, 2944, 768] + speed: 5886.08 + value: {index: 0, type: Single} + - key: [1024, 2944, 1792] + speed: 6060.2 + value: {index: 0, type: Single} + - key: [1024, 2944, 3328] + speed: 6144.57 + value: {index: 0, type: Single} + - key: [1024, 3584, 256] + speed: 5705.44 + value: {index: 0, type: Single} + - key: [1024, 3584, 768] + speed: 6206.1 + value: {index: 0, type: Single} + - key: [1024, 3584, 1792] + speed: 6388.15 + value: {index: 0, type: Single} + - key: [1024, 3584, 3328] + speed: 6449.51 + value: {index: 0, type: Single} + - key: [1024, 4288, 256] + speed: 5548.28 + value: {index: 0, type: Single} + - key: [1024, 4288, 768] + speed: 5867.96 + value: {index: 0, type: Single} + - key: [1024, 4288, 1792] + speed: 6038.08 + value: {index: 0, type: Single} + - key: [1024, 4288, 3328] + speed: 6083.49 + value: {index: 0, type: Single} + - key: [1024, 5056, 256] + speed: 5852.92 + value: {index: 0, type: Single} + - key: [1024, 5056, 768] + speed: 6237.9 + value: {index: 0, type: Single} + - key: [1024, 5056, 1792] + speed: 6405.17 + value: {index: 0, type: Single} + - key: [1024, 5056, 3328] + speed: 6479.32 + value: {index: 0, type: Single} + - key: [1024, 5888, 256] + speed: 5767.15 + value: {index: 0, type: Single} + - key: [1024, 5888, 768] + speed: 6132.65 + value: {index: 0, type: Single} + - key: [1024, 5888, 1792] + speed: 6263.45 + value: {index: 0, type: Single} + - key: [1024, 5888, 3328] + speed: 6329.2 + value: {index: 0, type: Single} + - key: [1024, 6784, 256] + speed: 5766.87 + value: {index: 0, type: Single} + - key: [1024, 6784, 768] + speed: 6090.7 + value: {index: 0, type: Single} + - key: [1024, 6784, 1792] + speed: 6236.18 + value: {index: 0, type: Single} + - key: [1024, 6784, 3328] + speed: 6282.16 + value: {index: 0, type: Single} + - key: [1408, 4, 256] + speed: 86.8916 + value: {index: 6, type: Single} + - key: [1408, 4, 768] + speed: 105.019 + value: {index: 6, type: Single} + - key: [1408, 4, 1792] + speed: 114.396 + value: {index: 6, type: Single} + - key: [1408, 4, 3328] + speed: 115.802 + value: {index: 6, type: Single} + - key: [1408, 64, 256] + speed: 1564.93 + value: {index: 3, type: Single} + - key: [1408, 64, 768] + speed: 1979.35 + value: {index: 3, type: Single} + - key: [1408, 64, 1792] + speed: 2114.38 + value: {index: 3, type: Single} + - key: [1408, 64, 3328] + speed: 2221.98 + value: {index: 3, type: Single} + - key: [1408, 128, 256] + speed: 2155.15 + value: {index: 4, type: Single} + - key: [1408, 128, 768] + speed: 2883.52 + value: {index: 3, type: Single} + - key: [1408, 128, 1792] + speed: 3103.09 + value: {index: 3, type: Single} + - key: [1408, 128, 3328] + speed: 3233.58 + value: {index: 3, type: Single} + - key: [1408, 256, 256] + speed: 3121.98 + value: {index: 3, type: Single} + - key: [1408, 256, 768] + speed: 3707.37 + value: {index: 3, type: Single} + - key: [1408, 256, 1792] + speed: 4018.31 + value: {index: 3, type: Single} + - key: [1408, 256, 3328] + speed: 4084.17 + value: {index: 3, type: Single} + - key: [1408, 448, 256] + speed: 3621.13 + value: {index: 3, type: Single} + - key: [1408, 448, 768] + speed: 4257.67 + value: {index: 3, type: Single} + - key: [1408, 448, 1792] + speed: 4518.65 + value: {index: 3, type: Single} + - key: [1408, 448, 3328] + speed: 4614.73 + value: {index: 3, type: Single} + - key: [1408, 704, 256] + speed: 4711.96 + value: {index: 0, type: Single} + - key: [1408, 704, 768] + speed: 5208.73 + value: {index: 0, type: Single} + - key: [1408, 704, 1792] + speed: 5417.8 + value: {index: 0, type: Single} + - key: [1408, 704, 3328] + speed: 5448.1 + value: {index: 0, type: Single} + - key: [1408, 1024, 256] + speed: 4957.91 + value: {index: 0, type: Single} + - key: [1408, 1024, 768] + speed: 5427.76 + value: {index: 0, type: Single} + - key: [1408, 1024, 1792] + speed: 5574.37 + value: {index: 0, type: Single} + - key: [1408, 1024, 3328] + speed: 5625.76 + value: {index: 0, type: Single} + - key: [1408, 1408, 256] + speed: 5327.53 + value: {index: 0, type: Single} + - key: [1408, 1408, 768] + speed: 5730.02 + value: {index: 0, type: Single} + - key: [1408, 1408, 1792] + speed: 5888.8 + value: {index: 0, type: Single} + - key: [1408, 1408, 3328] + speed: 5935.73 + value: {index: 0, type: Single} + - key: [1408, 1856, 256] + speed: 5726.78 + value: {index: 0, type: Single} + - key: [1408, 1856, 768] + speed: 6170.2 + value: {index: 0, type: Single} + - key: [1408, 1856, 1792] + speed: 6310.43 + value: {index: 0, type: Single} + - key: [1408, 1856, 3328] + speed: 6364.08 + value: {index: 0, type: Single} + - key: [1408, 2368, 256] + speed: 5466.07 + value: {index: 0, type: Single} + - key: [1408, 2368, 768] + speed: 5757.4 + value: {index: 0, type: Single} + - key: [1408, 2368, 1792] + speed: 5880.96 + value: {index: 0, type: Single} + - key: [1408, 2368, 3328] + speed: 5910.59 + value: {index: 0, type: Single} + - key: [1408, 2944, 256] + speed: 5956.43 + value: {index: 0, type: Single} + - key: [1408, 2944, 768] + speed: 6311.6 + value: {index: 0, type: Single} + - key: [1408, 2944, 1792] + speed: 6420.58 + value: {index: 0, type: Single} + - key: [1408, 2944, 3328] + speed: 6466.48 + value: {index: 0, type: Single} + - key: [1408, 3584, 256] + speed: 5947.96 + value: {index: 0, type: Single} + - key: [1408, 3584, 768] + speed: 6210.62 + value: {index: 0, type: Single} + - key: [1408, 3584, 1792] + speed: 6314.38 + value: {index: 0, type: Single} + - key: [1408, 3584, 3328] + speed: 6351.55 + value: {index: 0, type: Single} + - key: [1408, 4288, 256] + speed: 5978.51 + value: {index: 0, type: Single} + - key: [1408, 4288, 768] + speed: 6244.52 + value: {index: 0, type: Single} + - key: [1408, 4288, 1792] + speed: 6339.2 + value: {index: 0, type: Single} + - key: [1408, 4288, 3328] + speed: 6369.2 + value: {index: 0, type: Single} + - key: [1408, 5056, 256] + speed: 6101.7 + value: {index: 0, type: Single} + - key: [1408, 5056, 768] + speed: 6346.18 + value: {index: 0, type: Single} + - key: [1408, 5056, 1792] + speed: 6432.38 + value: {index: 0, type: Single} + - key: [1408, 5056, 3328] + speed: 6463.1 + value: {index: 0, type: Single} + - key: [1408, 5888, 256] + speed: 6255.56 + value: {index: 0, type: Single} + - key: [1408, 5888, 768] + speed: 6487.39 + value: {index: 0, type: Single} + - key: [1408, 5888, 1792] + speed: 6567.55 + value: {index: 0, type: Single} + - key: [1408, 5888, 3328] + speed: 6605.59 + value: {index: 0, type: Single} + - key: [1408, 6784, 256] + speed: 6134.61 + value: {index: 0, type: Single} + - key: [1408, 6784, 768] + speed: 6344.48 + value: {index: 0, type: Single} + - key: [1408, 6784, 1792] + speed: 6407.81 + value: {index: 0, type: Single} + - key: [1408, 6784, 3328] + speed: 6433.31 + value: {index: 0, type: Single} + - key: [1856, 4, 256] + speed: 110.116 + value: {index: 6, type: Single} + - key: [1856, 4, 768] + speed: 136.23 + value: {index: 6, type: Single} + - key: [1856, 4, 1792] + speed: 149.789 + value: {index: 6, type: Single} + - key: [1856, 4, 3328] + speed: 151.332 + value: {index: 6, type: Single} + - key: [1856, 64, 256] + speed: 1732.1 + value: {index: 3, type: Single} + - key: [1856, 64, 768] + speed: 2463.04 + value: {index: 3, type: Single} + - key: [1856, 64, 1792] + speed: 2752.43 + value: {index: 3, type: Single} + - key: [1856, 64, 3328] + speed: 2867.58 + value: {index: 3, type: Single} + - key: [1856, 128, 256] + speed: 2870.64 + value: {index: 3, type: Single} + - key: [1856, 128, 768] + speed: 3783.43 + value: {index: 3, type: Single} + - key: [1856, 128, 1792] + speed: 4128.65 + value: {index: 3, type: Single} + - key: [1856, 128, 3328] + speed: 4277.82 + value: {index: 3, type: Single} + - key: [1856, 256, 256] + speed: 3378.65 + value: {index: 5, type: Single} + - key: [1856, 256, 768] + speed: 4051.03 + value: {index: 5, type: Single} + - key: [1856, 256, 1792] + speed: 4288.87 + value: {index: 3, type: Single} + - key: [1856, 256, 3328] + speed: 4408.57 + value: {index: 3, type: Single} + - key: [1856, 448, 256] + speed: 3857.08 + value: {index: 0, type: Single} + - key: [1856, 448, 768] + speed: 4358.26 + value: {index: 0, type: Single} + - key: [1856, 448, 1792] + speed: 4538.53 + value: {index: 0, type: Single} + - key: [1856, 448, 3328] + speed: 4597.1 + value: {index: 0, type: Single} + - key: [1856, 704, 256] + speed: 4414.04 + value: {index: 0, type: Single} + - key: [1856, 704, 768] + speed: 4886.97 + value: {index: 0, type: Single} + - key: [1856, 704, 1792] + speed: 5042.1 + value: {index: 0, type: Single} + - key: [1856, 704, 3328] + speed: 5085.53 + value: {index: 0, type: Single} + - key: [1856, 1024, 256] + speed: 4919.93 + value: {index: 0, type: Single} + - key: [1856, 1024, 768] + speed: 5488.66 + value: {index: 0, type: Single} + - key: [1856, 1024, 1792] + speed: 5630.25 + value: {index: 0, type: Single} + - key: [1856, 1024, 3328] + speed: 5686.28 + value: {index: 0, type: Single} + - key: [1856, 1408, 256] + speed: 5470.11 + value: {index: 5, type: Single} + - key: [1856, 1408, 768] + speed: 5804.07 + value: {index: 5, type: Single} + - key: [1856, 1408, 1792] + speed: 5939.3 + value: {index: 5, type: Single} + - key: [1856, 1408, 3328] + speed: 5976.0 + value: {index: 5, type: Single} + - key: [1856, 1856, 256] + speed: 5486.03 + value: {index: 0, type: Single} + - key: [1856, 1856, 768] + speed: 5905.09 + value: {index: 0, type: Single} + - key: [1856, 1856, 1792] + speed: 6051.75 + value: {index: 0, type: Single} + - key: [1856, 1856, 3328] + speed: 6101.59 + value: {index: 0, type: Single} + - key: [1856, 2368, 256] + speed: 5682.28 + value: {index: 0, type: Single} + - key: [1856, 2368, 768] + speed: 5978.24 + value: {index: 0, type: Single} + - key: [1856, 2368, 1792] + speed: 6081.39 + value: {index: 0, type: Single} + - key: [1856, 2368, 3328] + speed: 6120.87 + value: {index: 0, type: Single} + - key: [1856, 2944, 256] + speed: 5831.77 + value: {index: 0, type: Single} + - key: [1856, 2944, 768] + speed: 6101.22 + value: {index: 0, type: Single} + - key: [1856, 2944, 1792] + speed: 6223.01 + value: {index: 0, type: Single} + - key: [1856, 2944, 3328] + speed: 6269.5 + value: {index: 0, type: Single} + - key: [1856, 3584, 256] + speed: 5728.46 + value: {index: 0, type: Single} + - key: [1856, 3584, 768] + speed: 5930.43 + value: {index: 0, type: Single} + - key: [1856, 3584, 1792] + speed: 6008.45 + value: {index: 0, type: Single} + - key: [1856, 3584, 3328] + speed: 6039.17 + value: {index: 0, type: Single} + - key: [1856, 4288, 256] + speed: 5984.31 + value: {index: 0, type: Single} + - key: [1856, 4288, 768] + speed: 6219.31 + value: {index: 0, type: Single} + - key: [1856, 4288, 1792] + speed: 6312.37 + value: {index: 0, type: Single} + - key: [1856, 4288, 3328] + speed: 6339.1 + value: {index: 0, type: Single} + - key: [1856, 5056, 256] + speed: 5998.88 + value: {index: 0, type: Single} + - key: [1856, 5056, 768] + speed: 6224.56 + value: {index: 0, type: Single} + - key: [1856, 5056, 1792] + speed: 6287.66 + value: {index: 0, type: Single} + - key: [1856, 5056, 3328] + speed: 6319.16 + value: {index: 0, type: Single} + - key: [1856, 5888, 256] + speed: 6063.02 + value: {index: 0, type: Single} + - key: [1856, 5888, 768] + speed: 6281.51 + value: {index: 0, type: Single} + - key: [1856, 5888, 1792] + speed: 6342.9 + value: {index: 0, type: Single} + - key: [1856, 5888, 3328] + speed: 6372.69 + value: {index: 0, type: Single} + - key: [1856, 6784, 256] + speed: 6127.82 + value: {index: 0, type: Single} + - key: [1856, 6784, 768] + speed: 6353.28 + value: {index: 0, type: Single} + - key: [1856, 6784, 1792] + speed: 6444.0 + value: {index: 0, type: Single} + - key: [1856, 6784, 3328] + speed: 6471.68 + value: {index: 0, type: Single} + - key: [2368, 4, 256] + speed: 133.611 + value: {index: 6, type: Single} + - key: [2368, 4, 768] + speed: 172.894 + value: {index: 6, type: Single} + - key: [2368, 4, 1792] + speed: 183.607 + value: {index: 6, type: Single} + - key: [2368, 4, 3328] + speed: 188.964 + value: {index: 6, type: Single} + - key: [2368, 64, 256] + speed: 1863.87 + value: {index: 3, type: Single} + - key: [2368, 64, 768] + speed: 2439.83 + value: {index: 3, type: Single} + - key: [2368, 64, 1792] + speed: 2631.89 + value: {index: 3, type: Single} + - key: [2368, 64, 3328] + speed: 2731.14 + value: {index: 3, type: Single} + - key: [2368, 128, 256] + speed: 2580.04 + value: {index: 3, type: Single} + - key: [2368, 128, 768] + speed: 3074.86 + value: {index: 3, type: Single} + - key: [2368, 128, 1792] + speed: 3329.92 + value: {index: 3, type: Single} + - key: [2368, 128, 3328] + speed: 3406.92 + value: {index: 3, type: Single} + - key: [2368, 256, 256] + speed: 3384.49 + value: {index: 3, type: Single} + - key: [2368, 256, 768] + speed: 4031.43 + value: {index: 3, type: Single} + - key: [2368, 256, 1792] + speed: 4289.26 + value: {index: 3, type: Single} + - key: [2368, 256, 3328] + speed: 4372.28 + value: {index: 3, type: Single} + - key: [2368, 448, 256] + speed: 3658.92 + value: {index: 0, type: Single} + - key: [2368, 448, 768] + speed: 4202.8 + value: {index: 3, type: Single} + - key: [2368, 448, 1792] + speed: 4401.25 + value: {index: 3, type: Single} + - key: [2368, 448, 3328] + speed: 4477.75 + value: {index: 3, type: Single} + - key: [2368, 704, 256] + speed: 4407.99 + value: {index: 0, type: Single} + - key: [2368, 704, 768] + speed: 4813.06 + value: {index: 0, type: Single} + - key: [2368, 704, 1792] + speed: 4931.36 + value: {index: 0, type: Single} + - key: [2368, 704, 3328] + speed: 4980.78 + value: {index: 0, type: Single} + - key: [2368, 1024, 256] + speed: 5297.06 + value: {index: 0, type: Single} + - key: [2368, 1024, 768] + speed: 5680.05 + value: {index: 0, type: Single} + - key: [2368, 1024, 1792] + speed: 5851.36 + value: {index: 0, type: Single} + - key: [2368, 1024, 3328] + speed: 5886.41 + value: {index: 0, type: Single} + - key: [2368, 1408, 256] + speed: 5424.89 + value: {index: 0, type: Single} + - key: [2368, 1408, 768] + speed: 5759.31 + value: {index: 0, type: Single} + - key: [2368, 1408, 1792] + speed: 5866.84 + value: {index: 0, type: Single} + - key: [2368, 1408, 3328] + speed: 5903.6 + value: {index: 0, type: Single} + - key: [2368, 1856, 256] + speed: 5673.78 + value: {index: 0, type: Single} + - key: [2368, 1856, 768] + speed: 5971.97 + value: {index: 0, type: Single} + - key: [2368, 1856, 1792] + speed: 6071.67 + value: {index: 0, type: Single} + - key: [2368, 1856, 3328] + speed: 6119.54 + value: {index: 0, type: Single} + - key: [2368, 2368, 256] + speed: 5957.18 + value: {index: 0, type: Single} + - key: [2368, 2368, 768] + speed: 6259.27 + value: {index: 0, type: Single} + - key: [2368, 2368, 1792] + speed: 6384.77 + value: {index: 0, type: Single} + - key: [2368, 2368, 3328] + speed: 6430.55 + value: {index: 0, type: Single} + - key: [2368, 2944, 256] + speed: 5948.75 + value: {index: 0, type: Single} + - key: [2368, 2944, 768] + speed: 6219.01 + value: {index: 0, type: Single} + - key: [2368, 2944, 1792] + speed: 6294.68 + value: {index: 0, type: Single} + - key: [2368, 2944, 3328] + speed: 6325.77 + value: {index: 0, type: Single} + - key: [2368, 3584, 256] + speed: 6037.45 + value: {index: 0, type: Single} + - key: [2368, 3584, 768] + speed: 6269.31 + value: {index: 0, type: Single} + - key: [2368, 3584, 1792] + speed: 6341.41 + value: {index: 0, type: Single} + - key: [2368, 3584, 3328] + speed: 6373.3 + value: {index: 0, type: Single} + - key: [2368, 4288, 256] + speed: 6147.71 + value: {index: 0, type: Single} + - key: [2368, 4288, 768] + speed: 6394.94 + value: {index: 0, type: Single} + - key: [2368, 4288, 1792] + speed: 6468.59 + value: {index: 0, type: Single} + - key: [2368, 4288, 3328] + speed: 6499.2 + value: {index: 0, type: Single} + - key: [2368, 5056, 256] + speed: 6115.3 + value: {index: 0, type: Single} + - key: [2368, 5056, 768] + speed: 6312.79 + value: {index: 0, type: Single} + - key: [2368, 5056, 1792] + speed: 6379.16 + value: {index: 0, type: Single} + - key: [2368, 5056, 3328] + speed: 6406.12 + value: {index: 0, type: Single} + - key: [2368, 5888, 256] + speed: 6121.05 + value: {index: 0, type: Single} + - key: [2368, 5888, 768] + speed: 6327.35 + value: {index: 0, type: Single} + - key: [2368, 5888, 1792] + speed: 6384.03 + value: {index: 0, type: Single} + - key: [2368, 5888, 3328] + speed: 6251.78 + value: {index: 2, type: Single} + - key: [2368, 6784, 256] + speed: 6207.22 + value: {index: 0, type: Single} + - key: [2368, 6784, 768] + speed: 6388.16 + value: {index: 0, type: Single} + - key: [2368, 6784, 1792] + speed: 6447.15 + value: {index: 0, type: Single} + - key: [2368, 6784, 3328] + speed: 6471.48 + value: {index: 0, type: Single} + - key: [2944, 4, 256] + speed: 160.222 + value: {index: 6, type: Single} + - key: [2944, 4, 768] + speed: 205.54 + value: {index: 6, type: Single} + - key: [2944, 4, 1792] + speed: 229.554 + value: {index: 6, type: Single} + - key: [2944, 4, 3328] + speed: 235.033 + value: {index: 6, type: Single} + - key: [2944, 64, 256] + speed: 2367.8 + value: {index: 3, type: Single} + - key: [2944, 64, 768] + speed: 3052.27 + value: {index: 3, type: Single} + - key: [2944, 64, 1792] + speed: 3314.92 + value: {index: 3, type: Single} + - key: [2944, 64, 3328] + speed: 3417.4 + value: {index: 3, type: Single} + - key: [2944, 128, 256] + speed: 3071.43 + value: {index: 3, type: Single} + - key: [2944, 128, 768] + speed: 3830.26 + value: {index: 3, type: Single} + - key: [2944, 128, 1792] + speed: 4097.11 + value: {index: 3, type: Single} + - key: [2944, 128, 3328] + speed: 4192.62 + value: {index: 3, type: Single} + - key: [2944, 256, 256] + speed: 3668.44 + value: {index: 3, type: Single} + - key: [2944, 256, 768] + speed: 4362.79 + value: {index: 3, type: Single} + - key: [2944, 256, 1792] + speed: 4615.73 + value: {index: 3, type: Single} + - key: [2944, 256, 3328] + speed: 4690.35 + value: {index: 3, type: Single} + - key: [2944, 448, 256] + speed: 4580.92 + value: {index: 0, type: Single} + - key: [2944, 448, 768] + speed: 4988.72 + value: {index: 0, type: Single} + - key: [2944, 448, 1792] + speed: 5101.73 + value: {index: 0, type: Single} + - key: [2944, 448, 3328] + speed: 5140.49 + value: {index: 0, type: Single} + - key: [2944, 704, 256] + speed: 5552.4 + value: {index: 0, type: Single} + - key: [2944, 704, 768] + speed: 5998.84 + value: {index: 0, type: Single} + - key: [2944, 704, 1792] + speed: 6141.39 + value: {index: 0, type: Single} + - key: [2944, 704, 3328] + speed: 6200.99 + value: {index: 0, type: Single} + - key: [2944, 1024, 256] + speed: 5634.58 + value: {index: 0, type: Single} + - key: [2944, 1024, 768] + speed: 6021.01 + value: {index: 0, type: Single} + - key: [2944, 1024, 1792] + speed: 6139.28 + value: {index: 0, type: Single} + - key: [2944, 1024, 3328] + speed: 6180.46 + value: {index: 0, type: Single} + - key: [2944, 1408, 256] + speed: 5981.31 + value: {index: 0, type: Single} + - key: [2944, 1408, 768] + speed: 6311.6 + value: {index: 0, type: Single} + - key: [2944, 1408, 1792] + speed: 6423.05 + value: {index: 0, type: Single} + - key: [2944, 1408, 3328] + speed: 6458.18 + value: {index: 0, type: Single} + - key: [2944, 1856, 256] + speed: 5880.8 + value: {index: 0, type: Single} + - key: [2944, 1856, 768] + speed: 6151.57 + value: {index: 0, type: Single} + - key: [2944, 1856, 1792] + speed: 6232.39 + value: {index: 0, type: Single} + - key: [2944, 1856, 3328] + speed: 6262.15 + value: {index: 0, type: Single} + - key: [2944, 2368, 256] + speed: 5978.27 + value: {index: 0, type: Single} + - key: [2944, 2368, 768] + speed: 6220.08 + value: {index: 0, type: Single} + - key: [2944, 2368, 1792] + speed: 6297.5 + value: {index: 0, type: Single} + - key: [2944, 2368, 3328] + speed: 6325.9 + value: {index: 0, type: Single} + - key: [2944, 2944, 256] + speed: 6166.92 + value: {index: 0, type: Single} + - key: [2944, 2944, 768] + speed: 6390.6 + value: {index: 0, type: Single} + - key: [2944, 2944, 1792] + speed: 6470.48 + value: {index: 0, type: Single} + - key: [2944, 2944, 3328] + speed: 6502.44 + value: {index: 0, type: Single} + - key: [2944, 3584, 256] + speed: 6147.04 + value: {index: 0, type: Single} + - key: [2944, 3584, 768] + speed: 6338.67 + value: {index: 0, type: Single} + - key: [2944, 3584, 1792] + speed: 6407.67 + value: {index: 0, type: Single} + - key: [2944, 3584, 3328] + speed: 6432.91 + value: {index: 0, type: Single} + - key: [2944, 4288, 256] + speed: 6203.16 + value: {index: 0, type: Single} + - key: [2944, 4288, 768] + speed: 6404.1 + value: {index: 0, type: Single} + - key: [2944, 4288, 1792] + speed: 6459.27 + value: {index: 0, type: Single} + - key: [2944, 4288, 3328] + speed: 6487.49 + value: {index: 0, type: Single} + - key: [2944, 5056, 256] + speed: 6304.77 + value: {index: 0, type: Single} + - key: [2944, 5056, 768] + speed: 6518.62 + value: {index: 0, type: Single} + - key: [2944, 5056, 1792] + speed: 6578.32 + value: {index: 0, type: Single} + - key: [2944, 5056, 3328] + speed: 6602.31 + value: {index: 0, type: Single} + - key: [2944, 5888, 256] + speed: 6314.48 + value: {index: 0, type: Single} + - key: [2944, 5888, 768] + speed: 6496.64 + value: {index: 0, type: Single} + - key: [2944, 5888, 1792] + speed: 6546.03 + value: {index: 0, type: Single} + - key: [2944, 5888, 3328] + speed: 6572.75 + value: {index: 0, type: Single} + - key: [2944, 6784, 256] + speed: 6352.07 + value: {index: 0, type: Single} + - key: [2944, 6784, 768] + speed: 6533.66 + value: {index: 0, type: Single} + - key: [2944, 6784, 1792] + speed: 6584.05 + value: {index: 0, type: Single} + - key: [2944, 6784, 3328] + speed: 6607.85 + value: {index: 0, type: Single} + - key: [3584, 4, 256] + speed: 192.778 + value: {index: 6, type: Single} + - key: [3584, 4, 768] + speed: 250.641 + value: {index: 6, type: Single} + - key: [3584, 4, 1792] + speed: 272.219 + value: {index: 6, type: Single} + - key: [3584, 4, 3328] + speed: 283.982 + value: {index: 6, type: Single} + - key: [3584, 64, 256] + speed: 2724.02 + value: {index: 3, type: Single} + - key: [3584, 64, 768] + speed: 3636.23 + value: {index: 3, type: Single} + - key: [3584, 64, 1792] + speed: 3943.77 + value: {index: 3, type: Single} + - key: [3584, 64, 3328] + speed: 4044.39 + value: {index: 3, type: Single} + - key: [3584, 128, 256] + speed: 3282.43 + value: {index: 0, type: Single} + - key: [3584, 128, 768] + speed: 3943.78 + value: {index: 5, type: Single} + - key: [3584, 128, 1792] + speed: 4159.59 + value: {index: 5, type: Single} + - key: [3584, 128, 3328] + speed: 4232.95 + value: {index: 5, type: Single} + - key: [3584, 256, 256] + speed: 4367.48 + value: {index: 0, type: Single} + - key: [3584, 256, 768] + speed: 4843.39 + value: {index: 0, type: Single} + - key: [3584, 256, 1792] + speed: 5026.18 + value: {index: 0, type: Single} + - key: [3584, 256, 3328] + speed: 5093.97 + value: {index: 0, type: Single} + - key: [3584, 448, 256] + speed: 4352.09 + value: {index: 0, type: Single} + - key: [3584, 448, 768] + speed: 4673.42 + value: {index: 0, type: Single} + - key: [3584, 448, 1792] + speed: 4779.42 + value: {index: 0, type: Single} + - key: [3584, 448, 3328] + speed: 4817.08 + value: {index: 0, type: Single} + - key: [3584, 704, 256] + speed: 5511.83 + value: {index: 0, type: Single} + - key: [3584, 704, 768] + speed: 5961.51 + value: {index: 0, type: Single} + - key: [3584, 704, 1792] + speed: 6074.03 + value: {index: 0, type: Single} + - key: [3584, 704, 3328] + speed: 6134.99 + value: {index: 0, type: Single} + - key: [3584, 1024, 256] + speed: 5855.59 + value: {index: 0, type: Single} + - key: [3584, 1024, 768] + speed: 6292.32 + value: {index: 0, type: Single} + - key: [3584, 1024, 1792] + speed: 6443.32 + value: {index: 0, type: Single} + - key: [3584, 1024, 3328] + speed: 6489.87 + value: {index: 0, type: Single} + - key: [3584, 1408, 256] + speed: 5795.76 + value: {index: 0, type: Single} + - key: [3584, 1408, 768] + speed: 6195.17 + value: {index: 0, type: Single} + - key: [3584, 1408, 1792] + speed: 6289.33 + value: {index: 0, type: Single} + - key: [3584, 1408, 3328] + speed: 6338.24 + value: {index: 0, type: Single} + - key: [3584, 1856, 256] + speed: 5989.66 + value: {index: 0, type: Single} + - key: [3584, 1856, 768] + speed: 6333.45 + value: {index: 0, type: Single} + - key: [3584, 1856, 1792] + speed: 6436.47 + value: {index: 0, type: Single} + - key: [3584, 1856, 3328] + speed: 6474.04 + value: {index: 0, type: Single} + - key: [3584, 2368, 256] + speed: 5954.1 + value: {index: 0, type: Single} + - key: [3584, 2368, 768] + speed: 6232.45 + value: {index: 0, type: Single} + - key: [3584, 2368, 1792] + speed: 6325.78 + value: {index: 0, type: Single} + - key: [3584, 2368, 3328] + speed: 6357.15 + value: {index: 0, type: Single} + - key: [3584, 2944, 256] + speed: 6083.45 + value: {index: 0, type: Single} + - key: [3584, 2944, 768] + speed: 6316.71 + value: {index: 0, type: Single} + - key: [3584, 2944, 1792] + speed: 6390.83 + value: {index: 0, type: Single} + - key: [3584, 2944, 3328] + speed: 6425.59 + value: {index: 0, type: Single} + - key: [3584, 3584, 256] + speed: 6228.6 + value: {index: 0, type: Single} + - key: [3584, 3584, 768] + speed: 6468.51 + value: {index: 0, type: Single} + - key: [3584, 3584, 1792] + speed: 6547.52 + value: {index: 0, type: Single} + - key: [3584, 3584, 3328] + speed: 6584.43 + value: {index: 0, type: Single} + - key: [3584, 4288, 256] + speed: 6216.16 + value: {index: 0, type: Single} + - key: [3584, 4288, 768] + speed: 6461.71 + value: {index: 0, type: Single} + - key: [3584, 4288, 1792] + speed: 6547.21 + value: {index: 0, type: Single} + - key: [3584, 4288, 3328] + speed: 6580.47 + value: {index: 0, type: Single} + - key: [3584, 5056, 256] + speed: 6354.48 + value: {index: 0, type: Single} + - key: [3584, 5056, 768] + speed: 6549.65 + value: {index: 0, type: Single} + - key: [3584, 5056, 1792] + speed: 6628.41 + value: {index: 0, type: Single} + - key: [3584, 5056, 3328] + speed: 6660.86 + value: {index: 0, type: Single} + - key: [3584, 5888, 256] + speed: 6343.81 + value: {index: 0, type: Single} + - key: [3584, 5888, 768] + speed: 6548.89 + value: {index: 0, type: Single} + - key: [3584, 5888, 1792] + speed: 6606.61 + value: {index: 0, type: Single} + - key: [3584, 5888, 3328] + speed: 6631.11 + value: {index: 0, type: Single} + - key: [3584, 6784, 256] + speed: 6372.38 + value: {index: 0, type: Single} + - key: [3584, 6784, 768] + speed: 6597.89 + value: {index: 0, type: Single} + - key: [3584, 6784, 1792] + speed: 6636.02 + value: {index: 0, type: Single} + - key: [3584, 6784, 3328] + speed: 6672.95 + value: {index: 0, type: Single} + - key: [4288, 4, 256] + speed: 222.844 + value: {index: 6, type: Single} + - key: [4288, 4, 768] + speed: 288.68 + value: {index: 6, type: Single} + - key: [4288, 4, 1792] + speed: 310.577 + value: {index: 6, type: Single} + - key: [4288, 4, 3328] + speed: 319.744 + value: {index: 6, type: Single} + - key: [4288, 64, 256] + speed: 2359.2 + value: {index: 3, type: Single} + - key: [4288, 64, 768] + speed: 2853.79 + value: {index: 3, type: Single} + - key: [4288, 64, 1792] + speed: 2963.78 + value: {index: 3, type: Single} + - key: [4288, 64, 3328] + speed: 3004.22 + value: {index: 3, type: Single} + - key: [4288, 128, 256] + speed: 3020.4 + value: {index: 3, type: Single} + - key: [4288, 128, 768] + speed: 3615.28 + value: {index: 3, type: Single} + - key: [4288, 128, 1792] + speed: 3732.85 + value: {index: 3, type: Single} + - key: [4288, 128, 3328] + speed: 3806.52 + value: {index: 3, type: Single} + - key: [4288, 256, 256] + speed: 3744.94 + value: {index: 0, type: Single} + - key: [4288, 256, 768] + speed: 4131.0 + value: {index: 0, type: Single} + - key: [4288, 256, 1792] + speed: 4300.48 + value: {index: 3, type: Single} + - key: [4288, 256, 3328] + speed: 4355.11 + value: {index: 3, type: Single} + - key: [4288, 448, 256] + speed: 5075.6 + value: {index: 0, type: Single} + - key: [4288, 448, 768] + speed: 5560.19 + value: {index: 0, type: Single} + - key: [4288, 448, 1792] + speed: 5695.11 + value: {index: 0, type: Single} + - key: [4288, 448, 3328] + speed: 5745.26 + value: {index: 0, type: Single} + - key: [4288, 704, 256] + speed: 5561.04 + value: {index: 0, type: Single} + - key: [4288, 704, 768] + speed: 5987.66 + value: {index: 0, type: Single} + - key: [4288, 704, 1792] + speed: 6126.98 + value: {index: 0, type: Single} + - key: [4288, 704, 3328] + speed: 6183.21 + value: {index: 0, type: Single} + - key: [4288, 1024, 256] + speed: 5668.49 + value: {index: 0, type: Single} + - key: [4288, 1024, 768] + speed: 5986.8 + value: {index: 0, type: Single} + - key: [4288, 1024, 1792] + speed: 6077.81 + value: {index: 0, type: Single} + - key: [4288, 1024, 3328] + speed: 6108.92 + value: {index: 0, type: Single} + - key: [4288, 1408, 256] + speed: 5937.69 + value: {index: 0, type: Single} + - key: [4288, 1408, 768] + speed: 6227.74 + value: {index: 0, type: Single} + - key: [4288, 1408, 1792] + speed: 6327.11 + value: {index: 0, type: Single} + - key: [4288, 1408, 3328] + speed: 6363.08 + value: {index: 0, type: Single} + - key: [4288, 1856, 256] + speed: 5958.38 + value: {index: 0, type: Single} + - key: [4288, 1856, 768] + speed: 6229.18 + value: {index: 0, type: Single} + - key: [4288, 1856, 1792] + speed: 6308.02 + value: {index: 0, type: Single} + - key: [4288, 1856, 3328] + speed: 6326.65 + value: {index: 0, type: Single} + - key: [4288, 2368, 256] + speed: 6156.34 + value: {index: 0, type: Single} + - key: [4288, 2368, 768] + speed: 6398.44 + value: {index: 0, type: Single} + - key: [4288, 2368, 1792] + speed: 6460.93 + value: {index: 0, type: Single} + - key: [4288, 2368, 3328] + speed: 6489.58 + value: {index: 0, type: Single} + - key: [4288, 2944, 256] + speed: 6155.9 + value: {index: 0, type: Single} + - key: [4288, 2944, 768] + speed: 6389.72 + value: {index: 0, type: Single} + - key: [4288, 2944, 1792] + speed: 6457.5 + value: {index: 0, type: Single} + - key: [4288, 2944, 3328] + speed: 6477.85 + value: {index: 0, type: Single} + - key: [4288, 3584, 256] + speed: 6277.15 + value: {index: 0, type: Single} + - key: [4288, 3584, 768] + speed: 6495.96 + value: {index: 0, type: Single} + - key: [4288, 3584, 1792] + speed: 6565.94 + value: {index: 0, type: Single} + - key: [4288, 3584, 3328] + speed: 6588.32 + value: {index: 0, type: Single} + - key: [4288, 4288, 256] + speed: 6300.79 + value: {index: 0, type: Single} + - key: [4288, 4288, 768] + speed: 6501.72 + value: {index: 0, type: Single} + - key: [4288, 4288, 1792] + speed: 6561.11 + value: {index: 0, type: Single} + - key: [4288, 4288, 3328] + speed: 6584.66 + value: {index: 0, type: Single} + - key: [4288, 5056, 256] + speed: 6330.21 + value: {index: 0, type: Single} + - key: [4288, 5056, 768] + speed: 6570.01 + value: {index: 0, type: Single} + - key: [4288, 5056, 1792] + speed: 6640.88 + value: {index: 0, type: Single} + - key: [4288, 5056, 3328] + speed: 6662.95 + value: {index: 0, type: Single} + - key: [4288, 5888, 256] + speed: 6378.19 + value: {index: 0, type: Single} + - key: [4288, 5888, 768] + speed: 6573.94 + value: {index: 0, type: Single} + - key: [4288, 5888, 1792] + speed: 6638.65 + value: {index: 0, type: Single} + - key: [4288, 5888, 3328] + speed: 6656.8 + value: {index: 0, type: Single} + - key: [4288, 6784, 256] + speed: 6338.68 + value: {index: 0, type: Single} + - key: [4288, 6784, 768] + speed: 6533.53 + value: {index: 0, type: Single} + - key: [4288, 6784, 1792] + speed: 6585.62 + value: {index: 0, type: Single} + - key: [4288, 6784, 3328] + speed: 6603.52 + value: {index: 0, type: Single} + - key: [5056, 4, 256] + speed: 250.513 + value: {index: 6, type: Single} + - key: [5056, 4, 768] + speed: 323.574 + value: {index: 6, type: Single} + - key: [5056, 4, 1792] + speed: 350.213 + value: {index: 6, type: Single} + - key: [5056, 4, 3328] + speed: 362.714 + value: {index: 6, type: Single} + - key: [5056, 64, 256] + speed: 2656.24 + value: {index: 3, type: Single} + - key: [5056, 64, 768] + speed: 3266.63 + value: {index: 3, type: Single} + - key: [5056, 64, 1792] + speed: 3437.82 + value: {index: 3, type: Single} + - key: [5056, 64, 3328] + speed: 3500.5 + value: {index: 3, type: Single} + - key: [5056, 128, 256] + speed: 3388.73 + value: {index: 3, type: Single} + - key: [5056, 128, 768] + speed: 4098.75 + value: {index: 3, type: Single} + - key: [5056, 128, 1792] + speed: 4302.23 + value: {index: 3, type: Single} + - key: [5056, 128, 3328] + speed: 4393.34 + value: {index: 3, type: Single} + - key: [5056, 256, 256] + speed: 4455.28 + value: {index: 0, type: Single} + - key: [5056, 256, 768] + speed: 4876.19 + value: {index: 0, type: Single} + - key: [5056, 256, 1792] + speed: 4987.52 + value: {index: 0, type: Single} + - key: [5056, 256, 3328] + speed: 5048.64 + value: {index: 0, type: Single} + - key: [5056, 448, 256] + speed: 4982.77 + value: {index: 0, type: Single} + - key: [5056, 448, 768] + speed: 5348.15 + value: {index: 0, type: Single} + - key: [5056, 448, 1792] + speed: 5464.79 + value: {index: 0, type: Single} + - key: [5056, 448, 3328] + speed: 5502.19 + value: {index: 0, type: Single} + - key: [5056, 704, 256] + speed: 5713.43 + value: {index: 0, type: Single} + - key: [5056, 704, 768] + speed: 6129.05 + value: {index: 0, type: Single} + - key: [5056, 704, 1792] + speed: 6244.17 + value: {index: 0, type: Single} + - key: [5056, 704, 3328] + speed: 6295.55 + value: {index: 0, type: Single} + - key: [5056, 1024, 256] + speed: 5938.4 + value: {index: 0, type: Single} + - key: [5056, 1024, 768] + speed: 6341.07 + value: {index: 0, type: Single} + - key: [5056, 1024, 1792] + speed: 6461.01 + value: {index: 0, type: Single} + - key: [5056, 1024, 3328] + speed: 6497.42 + value: {index: 0, type: Single} + - key: [5056, 1408, 256] + speed: 6025.47 + value: {index: 0, type: Single} + - key: [5056, 1408, 768] + speed: 6313.61 + value: {index: 0, type: Single} + - key: [5056, 1408, 1792] + speed: 6413.94 + value: {index: 0, type: Single} + - key: [5056, 1408, 3328] + speed: 6449.28 + value: {index: 0, type: Single} + - key: [5056, 1856, 256] + speed: 5992.23 + value: {index: 0, type: Single} + - key: [5056, 1856, 768] + speed: 6214.22 + value: {index: 0, type: Single} + - key: [5056, 1856, 1792] + speed: 6281.05 + value: {index: 0, type: Single} + - key: [5056, 1856, 3328] + speed: 6306.97 + value: {index: 0, type: Single} + - key: [5056, 2368, 256] + speed: 6090.1 + value: {index: 0, type: Single} + - key: [5056, 2368, 768] + speed: 6306.7 + value: {index: 0, type: Single} + - key: [5056, 2368, 1792] + speed: 6375.23 + value: {index: 0, type: Single} + - key: [5056, 2368, 3328] + speed: 6397.05 + value: {index: 0, type: Single} + - key: [5056, 2944, 256] + speed: 6257.22 + value: {index: 0, type: Single} + - key: [5056, 2944, 768] + speed: 6501.32 + value: {index: 0, type: Single} + - key: [5056, 2944, 1792] + speed: 6568.12 + value: {index: 0, type: Single} + - key: [5056, 2944, 3328] + speed: 6592.48 + value: {index: 0, type: Single} + - key: [5056, 3584, 256] + speed: 6275.51 + value: {index: 0, type: Single} + - key: [5056, 3584, 768] + speed: 6549.87 + value: {index: 0, type: Single} + - key: [5056, 3584, 1792] + speed: 6642.68 + value: {index: 0, type: Single} + - key: [5056, 3584, 3328] + speed: 6666.37 + value: {index: 0, type: Single} + - key: [5056, 4288, 256] + speed: 6329.68 + value: {index: 0, type: Single} + - key: [5056, 4288, 768] + speed: 6560.04 + value: {index: 0, type: Single} + - key: [5056, 4288, 1792] + speed: 6637.19 + value: {index: 0, type: Single} + - key: [5056, 4288, 3328] + speed: 6655.94 + value: {index: 0, type: Single} + - key: [5056, 5056, 256] + speed: 6324.72 + value: {index: 0, type: Single} + - key: [5056, 5056, 768] + speed: 6524.0 + value: {index: 0, type: Single} + - key: [5056, 5056, 1792] + speed: 6582.77 + value: {index: 0, type: Single} + - key: [5056, 5056, 3328] + speed: 6604.03 + value: {index: 0, type: Single} + - key: [5056, 5888, 256] + speed: 6359.33 + value: {index: 0, type: Single} + - key: [5056, 5888, 768] + speed: 6555.17 + value: {index: 0, type: Single} + - key: [5056, 5888, 1792] + speed: 6617.55 + value: {index: 0, type: Single} + - key: [5056, 5888, 3328] + speed: 6635.16 + value: {index: 0, type: Single} + - key: [5056, 6784, 256] + speed: 6362.7 + value: {index: 0, type: Single} + - key: [5056, 6784, 768] + speed: 6552.39 + value: {index: 0, type: Single} + - key: [5056, 6784, 1792] + speed: 6607.1 + value: {index: 0, type: Single} + - key: [5056, 6784, 3328] + speed: 6625.97 + value: {index: 0, type: Single} + - key: [5888, 4, 256] + speed: 290.688 + value: {index: 6, type: Single} + - key: [5888, 4, 768] + speed: 379.755 + value: {index: 6, type: Single} + - key: [5888, 4, 1792] + speed: 414.972 + value: {index: 6, type: Single} + - key: [5888, 4, 3328] + speed: 425.459 + value: {index: 6, type: Single} + - key: [5888, 64, 256] + speed: 2718.78 + value: {index: 3, type: Single} + - key: [5888, 64, 768] + speed: 3756.63 + value: {index: 3, type: Single} + - key: [5888, 64, 1792] + speed: 3996.51 + value: {index: 3, type: Single} + - key: [5888, 64, 3328] + speed: 4098.24 + value: {index: 3, type: Single} + - key: [5888, 128, 256] + speed: 3617.51 + value: {index: 0, type: Single} + - key: [5888, 128, 768] + speed: 4223.66 + value: {index: 3, type: Single} + - key: [5888, 128, 1792] + speed: 4458.82 + value: {index: 3, type: Single} + - key: [5888, 128, 3328] + speed: 4530.32 + value: {index: 3, type: Single} + - key: [5888, 256, 256] + speed: 5137.27 + value: {index: 0, type: Single} + - key: [5888, 256, 768] + speed: 5653.96 + value: {index: 0, type: Single} + - key: [5888, 256, 1792] + speed: 5814.74 + value: {index: 0, type: Single} + - key: [5888, 256, 3328] + speed: 5871.78 + value: {index: 0, type: Single} + - key: [5888, 448, 256] + speed: 4976.0 + value: {index: 0, type: Single} + - key: [5888, 448, 768] + speed: 5303.12 + value: {index: 0, type: Single} + - key: [5888, 448, 1792] + speed: 5388.2 + value: {index: 0, type: Single} + - key: [5888, 448, 3328] + speed: 5418.78 + value: {index: 0, type: Single} + - key: [5888, 704, 256] + speed: 5919.52 + value: {index: 0, type: Single} + - key: [5888, 704, 768] + speed: 6278.41 + value: {index: 0, type: Single} + - key: [5888, 704, 1792] + speed: 6407.04 + value: {index: 0, type: Single} + - key: [5888, 704, 3328] + speed: 6453.25 + value: {index: 0, type: Single} + - key: [5888, 1024, 256] + speed: 5887.74 + value: {index: 0, type: Single} + - key: [5888, 1024, 768] + speed: 6210.64 + value: {index: 0, type: Single} + - key: [5888, 1024, 1792] + speed: 6304.87 + value: {index: 0, type: Single} + - key: [5888, 1024, 3328] + speed: 6339.27 + value: {index: 0, type: Single} + - key: [5888, 1408, 256] + speed: 6178.67 + value: {index: 0, type: Single} + - key: [5888, 1408, 768] + speed: 6453.78 + value: {index: 0, type: Single} + - key: [5888, 1408, 1792] + speed: 6545.26 + value: {index: 0, type: Single} + - key: [5888, 1408, 3328] + speed: 6588.64 + value: {index: 0, type: Single} + - key: [5888, 1856, 256] + speed: 6245.49 + value: {index: 0, type: Single} + - key: [5888, 1856, 768] + speed: 6534.38 + value: {index: 0, type: Single} + - key: [5888, 1856, 1792] + speed: 6609.47 + value: {index: 0, type: Single} + - key: [5888, 1856, 3328] + speed: 6647.42 + value: {index: 0, type: Single} + - key: [5888, 2368, 256] + speed: 6250.47 + value: {index: 0, type: Single} + - key: [5888, 2368, 768] + speed: 6495.38 + value: {index: 0, type: Single} + - key: [5888, 2368, 1792] + speed: 6594.34 + value: {index: 0, type: Single} + - key: [5888, 2368, 3328] + speed: 6623.07 + value: {index: 0, type: Single} + - key: [5888, 2944, 256] + speed: 6280.06 + value: {index: 0, type: Single} + - key: [5888, 2944, 768] + speed: 6472.31 + value: {index: 0, type: Single} + - key: [5888, 2944, 1792] + speed: 6532.36 + value: {index: 0, type: Single} + - key: [5888, 2944, 3328] + speed: 6559.35 + value: {index: 0, type: Single} + - key: [5888, 3584, 256] + speed: 6291.82 + value: {index: 0, type: Single} + - key: [5888, 3584, 768] + speed: 6548.7 + value: {index: 0, type: Single} + - key: [5888, 3584, 1792] + speed: 6600.88 + value: {index: 0, type: Single} + - key: [5888, 3584, 3328] + speed: 6630.14 + value: {index: 0, type: Single} + - key: [5888, 4288, 256] + speed: 6354.04 + value: {index: 0, type: Single} + - key: [5888, 4288, 768] + speed: 6549.92 + value: {index: 0, type: Single} + - key: [5888, 4288, 1792] + speed: 6617.29 + value: {index: 0, type: Single} + - key: [5888, 4288, 3328] + speed: 6647.28 + value: {index: 0, type: Single} + - key: [5888, 5056, 256] + speed: 6427.26 + value: {index: 0, type: Single} + - key: [5888, 5056, 768] + speed: 6644.93 + value: {index: 0, type: Single} + - key: [5888, 5056, 1792] + speed: 6708.42 + value: {index: 0, type: Single} + - key: [5888, 5056, 3328] + speed: 6742.43 + value: {index: 0, type: Single} + - key: [5888, 5888, 256] + speed: 6421.43 + value: {index: 0, type: Single} + - key: [5888, 5888, 768] + speed: 6602.74 + value: {index: 0, type: Single} + - key: [5888, 5888, 1792] + speed: 6670.3 + value: {index: 0, type: Single} + - key: [5888, 5888, 3328] + speed: 6690.28 + value: {index: 0, type: Single} + - key: [5888, 6784, 256] + speed: 6408.39 + value: {index: 0, type: Single} + - key: [5888, 6784, 768] + speed: 6616.54 + value: {index: 0, type: Single} + - key: [5888, 6784, 1792] + speed: 6688.28 + value: {index: 0, type: Single} + - key: [5888, 6784, 3328] + speed: 6708.94 + value: {index: 0, type: Single} + - key: [6784, 4, 256] + speed: 301.544 + value: {index: 6, type: Single} + - key: [6784, 4, 768] + speed: 396.81 + value: {index: 6, type: Single} + - key: [6784, 4, 1792] + speed: 428.217 + value: {index: 6, type: Single} + - key: [6784, 4, 3328] + speed: 440.438 + value: {index: 6, type: Single} + - key: [6784, 64, 256] + speed: 3062.17 + value: {index: 0, type: Single} + - key: [6784, 64, 768] + speed: 3683.62 + value: {index: 0, type: Single} + - key: [6784, 64, 1792] + speed: 3900.22 + value: {index: 0, type: Single} + - key: [6784, 64, 3328] + speed: 3953.4 + value: {index: 0, type: Single} + - key: [6784, 128, 256] + speed: 4110.88 + value: {index: 0, type: Single} + - key: [6784, 128, 768] + speed: 4600.32 + value: {index: 0, type: Single} + - key: [6784, 128, 1792] + speed: 4698.4 + value: {index: 0, type: Single} + - key: [6784, 128, 3328] + speed: 4813.35 + value: {index: 0, type: Single} + - key: [6784, 256, 256] + speed: 4688.97 + value: {index: 0, type: Single} + - key: [6784, 256, 768] + speed: 5042.18 + value: {index: 0, type: Single} + - key: [6784, 256, 1792] + speed: 5161.32 + value: {index: 0, type: Single} + - key: [6784, 256, 3328] + speed: 5200.59 + value: {index: 0, type: Single} + - key: [6784, 448, 256] + speed: 5653.01 + value: {index: 0, type: Single} + - key: [6784, 448, 768] + speed: 6056.09 + value: {index: 0, type: Single} + - key: [6784, 448, 1792] + speed: 6176.33 + value: {index: 0, type: Single} + - key: [6784, 448, 3328] + speed: 6224.03 + value: {index: 0, type: Single} + - key: [6784, 704, 256] + speed: 5585.49 + value: {index: 0, type: Single} + - key: [6784, 704, 768] + speed: 5875.12 + value: {index: 0, type: Single} + - key: [6784, 704, 1792] + speed: 5966.85 + value: {index: 0, type: Single} + - key: [6784, 704, 3328] + speed: 6002.21 + value: {index: 0, type: Single} + - key: [6784, 1024, 256] + speed: 5900.1 + value: {index: 0, type: Single} + - key: [6784, 1024, 768] + speed: 6174.77 + value: {index: 0, type: Single} + - key: [6784, 1024, 1792] + speed: 6258.24 + value: {index: 0, type: Single} + - key: [6784, 1024, 3328] + speed: 6287.73 + value: {index: 0, type: Single} + - key: [6784, 1408, 256] + speed: 6032.59 + value: {index: 0, type: Single} + - key: [6784, 1408, 768] + speed: 6299.68 + value: {index: 0, type: Single} + - key: [6784, 1408, 1792] + speed: 6381.27 + value: {index: 0, type: Single} + - key: [6784, 1408, 3328] + speed: 6414.95 + value: {index: 0, type: Single} + - key: [6784, 1856, 256] + speed: 6177.4 + value: {index: 0, type: Single} + - key: [6784, 1856, 768] + speed: 6363.19 + value: {index: 0, type: Single} + - key: [6784, 1856, 1792] + speed: 6430.13 + value: {index: 0, type: Single} + - key: [6784, 1856, 3328] + speed: 6458.97 + value: {index: 0, type: Single} + - key: [6784, 2368, 256] + speed: 6295.91 + value: {index: 0, type: Single} + - key: [6784, 2368, 768] + speed: 6544.8 + value: {index: 0, type: Single} + - key: [6784, 2368, 1792] + speed: 6624.87 + value: {index: 0, type: Single} + - key: [6784, 2368, 3328] + speed: 6650.32 + value: {index: 0, type: Single} + - key: [6784, 2944, 256] + speed: 6255.92 + value: {index: 0, type: Single} + - key: [6784, 2944, 768] + speed: 6501.66 + value: {index: 0, type: Single} + - key: [6784, 2944, 1792] + speed: 6567.76 + value: {index: 0, type: Single} + - key: [6784, 2944, 3328] + speed: 6590.96 + value: {index: 0, type: Single} + - key: [6784, 3584, 256] + speed: 6333.0 + value: {index: 0, type: Single} + - key: [6784, 3584, 768] + speed: 6567.12 + value: {index: 0, type: Single} + - key: [6784, 3584, 1792] + speed: 6642.69 + value: {index: 0, type: Single} + - key: [6784, 3584, 3328] + speed: 6664.89 + value: {index: 0, type: Single} + - key: [6784, 4288, 256] + speed: 6389.44 + value: {index: 0, type: Single} + - key: [6784, 4288, 768] + speed: 6605.8 + value: {index: 0, type: Single} + - key: [6784, 4288, 1792] + speed: 6678.37 + value: {index: 0, type: Single} + - key: [6784, 4288, 3328] + speed: 6700.31 + value: {index: 0, type: Single} + - key: [6784, 5056, 256] + speed: 6400.49 + value: {index: 0, type: Single} + - key: [6784, 5056, 768] + speed: 6614.55 + value: {index: 0, type: Single} + - key: [6784, 5056, 1792] + speed: 6687.56 + value: {index: 0, type: Single} + - key: [6784, 5056, 3328] + speed: 6709.14 + value: {index: 0, type: Single} + - key: [6784, 5888, 256] + speed: 6406.61 + value: {index: 0, type: Single} + - key: [6784, 5888, 768] + speed: 6610.52 + value: {index: 0, type: Single} + - key: [6784, 5888, 1792] + speed: 6680.42 + value: {index: 0, type: Single} + - key: [6784, 5888, 3328] + speed: 6705.63 + value: {index: 0, type: Single} + - key: [6784, 6784, 256] + speed: 6452.06 + value: {index: 0, type: Single} + - key: [6784, 6784, 768] + speed: 6672.01 + value: {index: 0, type: Single} + - key: [6784, 6784, 1792] + speed: 6739.88 + value: {index: 0, type: Single} + - key: [6784, 6784, 3328] + speed: 6759.07 + value: {index: 0, type: Single} + type: Matching + predicate: + type: And + value: + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + type: Problem + Contraction_l_Alik_Bjlk_Cijk_Dijk: + rows: + - library: + distance: Euclidean + properties: + - {index: 0, type: FreeSizeA} + - {index: 0, type: FreeSizeB} + - {index: 0, type: BoundSize} + table: + - key: [4, 4, 256] + speed: 0.228489 + value: {index: 12, type: Single} + - key: [4, 4, 768] + speed: 0.320861 + value: {index: 12, type: Single} + - key: [4, 4, 1792] + speed: 0.347453 + value: {index: 12, type: Single} + - key: [4, 4, 3328] + speed: 0.375368 + value: {index: 12, type: Single} + - key: [4, 64, 256] + speed: 5.11401 + value: {index: 12, type: Single} + - key: [4, 64, 768] + speed: 5.59945 + value: {index: 12, type: Single} + - key: [4, 64, 1792] + speed: 6.16834 + value: {index: 12, type: Single} + - key: [4, 64, 3328] + speed: 6.41098 + value: {index: 12, type: Single} + - key: [4, 128, 256] + speed: 10.3475 + value: {index: 12, type: Single} + - key: [4, 128, 768] + speed: 13.4728 + value: {index: 12, type: Single} + - key: [4, 128, 1792] + speed: 14.6407 + value: {index: 12, type: Single} + - key: [4, 128, 3328] + speed: 13.0179 + value: {index: 12, type: Single} + - key: [4, 256, 256] + speed: 19.7695 + value: {index: 12, type: Single} + - key: [4, 256, 768] + speed: 26.021 + value: {index: 12, type: Single} + - key: [4, 256, 1792] + speed: 29.2122 + value: {index: 12, type: Single} + - key: [4, 256, 3328] + speed: 26.1097 + value: {index: 12, type: Single} + - key: [4, 448, 256] + speed: 27.6473 + value: {index: 12, type: Single} + - key: [4, 448, 768] + speed: 38.7874 + value: {index: 12, type: Single} + - key: [4, 448, 1792] + speed: 42.8792 + value: {index: 12, type: Single} + - key: [4, 448, 3328] + speed: 39.311 + value: {index: 12, type: Single} + - key: [4, 704, 256] + speed: 41.0627 + value: {index: 12, type: Single} + - key: [4, 704, 768] + speed: 58.1587 + value: {index: 12, type: Single} + - key: [4, 704, 1792] + speed: 62.0996 + value: {index: 12, type: Single} + - key: [4, 704, 3328] + speed: 58.5443 + value: {index: 12, type: Single} + - key: [4, 1024, 256] + speed: 55.7309 + value: {index: 12, type: Single} + - key: [4, 1024, 768] + speed: 79.3764 + value: {index: 12, type: Single} + - key: [4, 1024, 1792] + speed: 80.8222 + value: {index: 12, type: Single} + - key: [4, 1024, 3328] + speed: 80.7818 + value: {index: 12, type: Single} + - key: [4, 1408, 256] + speed: 81.4388 + value: {index: 12, type: Single} + - key: [4, 1408, 768] + speed: 112.724 + value: {index: 12, type: Single} + - key: [4, 1408, 1792] + speed: 113.918 + value: {index: 12, type: Single} + - key: [4, 1408, 3328] + speed: 118.515 + value: {index: 12, type: Single} + - key: [4, 1856, 256] + speed: 103.456 + value: {index: 12, type: Single} + - key: [4, 1856, 768] + speed: 148.879 + value: {index: 12, type: Single} + - key: [4, 1856, 1792] + speed: 148.55 + value: {index: 12, type: Single} + - key: [4, 1856, 3328] + speed: 154.702 + value: {index: 12, type: Single} + - key: [4, 2368, 256] + speed: 133.611 + value: {index: 12, type: Single} + - key: [4, 2368, 768] + speed: 185.991 + value: {index: 12, type: Single} + - key: [4, 2368, 1792] + speed: 187.514 + value: {index: 12, type: Single} + - key: [4, 2368, 3328] + speed: 193.519 + value: {index: 12, type: Single} + - key: [4, 2944, 256] + speed: 162.787 + value: {index: 12, type: Single} + - key: [4, 2944, 768] + speed: 216.091 + value: {index: 12, type: Single} + - key: [4, 2944, 1792] + speed: 225.022 + value: {index: 12, type: Single} + - key: [4, 2944, 3328] + speed: 232.964 + value: {index: 12, type: Single} + - key: [4, 3584, 256] + speed: 193.531 + value: {index: 12, type: Single} + - key: [4, 3584, 768] + speed: 254.506 + value: {index: 12, type: Single} + - key: [4, 3584, 1792] + speed: 274.158 + value: {index: 12, type: Single} + - key: [4, 3584, 3328] + speed: 283.234 + value: {index: 12, type: Single} + - key: [4, 4288, 256] + speed: 224.53 + value: {index: 12, type: Single} + - key: [4, 4288, 768] + speed: 292.477 + value: {index: 12, type: Single} + - key: [4, 4288, 1792] + speed: 311.978 + value: {index: 12, type: Single} + - key: [4, 4288, 3328] + speed: 322.691 + value: {index: 12, type: Single} + - key: [4, 5056, 256] + speed: 247.844 + value: {index: 12, type: Single} + - key: [4, 5056, 768] + speed: 327.621 + value: {index: 12, type: Single} + - key: [4, 5056, 1792] + speed: 351.471 + value: {index: 12, type: Single} + - key: [4, 5056, 3328] + speed: 366.225 + value: {index: 12, type: Single} + - key: [4, 5888, 256] + speed: 273.135 + value: {index: 12, type: Single} + - key: [4, 5888, 768] + speed: 345.374 + value: {index: 12, type: Single} + - key: [4, 5888, 1792] + speed: 367.112 + value: {index: 12, type: Single} + - key: [4, 5888, 3328] + speed: 375.087 + value: {index: 12, type: Single} + - key: [4, 6784, 256] + speed: 277.456 + value: {index: 12, type: Single} + - key: [4, 6784, 768] + speed: 350.36 + value: {index: 12, type: Single} + - key: [4, 6784, 1792] + speed: 369.42 + value: {index: 12, type: Single} + - key: [4, 6784, 3328] + speed: 373.625 + value: {index: 12, type: Single} + - key: [64, 4, 256] + speed: 4.05833 + value: {index: 12, type: Single} + - key: [64, 4, 768] + speed: 5.41657 + value: {index: 12, type: Single} + - key: [64, 4, 1792] + speed: 6.01263 + value: {index: 12, type: Single} + - key: [64, 4, 3328] + speed: 6.18018 + value: {index: 12, type: Single} + - key: [64, 64, 256] + speed: 56.8488 + value: {index: 7, type: Single} + - key: [64, 64, 768] + speed: 95.8596 + value: {index: 7, type: Single} + - key: [64, 64, 1792] + speed: 116.029 + value: {index: 7, type: Single} + - key: [64, 64, 3328] + speed: 118.954 + value: {index: 7, type: Single} + - key: [64, 128, 256] + speed: 151.397 + value: {index: 7, type: Single} + - key: [64, 128, 768] + speed: 207.659 + value: {index: 7, type: Single} + - key: [64, 128, 1792] + speed: 229.636 + value: {index: 7, type: Single} + - key: [64, 128, 3328] + speed: 220.648 + value: {index: 7, type: Single} + - key: [64, 256, 256] + speed: 304.42 + value: {index: 7, type: Single} + - key: [64, 256, 768] + speed: 412.291 + value: {index: 7, type: Single} + - key: [64, 256, 1792] + speed: 452.457 + value: {index: 7, type: Single} + - key: [64, 256, 3328] + speed: 410.989 + value: {index: 7, type: Single} + - key: [64, 448, 256] + speed: 483.358 + value: {index: 7, type: Single} + - key: [64, 448, 768] + speed: 718.027 + value: {index: 7, type: Single} + - key: [64, 448, 1792] + speed: 692.923 + value: {index: 7, type: Single} + - key: [64, 448, 3328] + speed: 710.113 + value: {index: 7, type: Single} + - key: [64, 704, 256] + speed: 714.267 + value: {index: 7, type: Single} + - key: [64, 704, 768] + speed: 1099.14 + value: {index: 7, type: Single} + - key: [64, 704, 1792] + speed: 1081.32 + value: {index: 7, type: Single} + - key: [64, 704, 3328] + speed: 1114.05 + value: {index: 7, type: Single} + - key: [64, 1024, 256] + speed: 980.464 + value: {index: 7, type: Single} + - key: [64, 1024, 768] + speed: 1253.61 + value: {index: 7, type: Single} + - key: [64, 1024, 1792] + speed: 1246.39 + value: {index: 7, type: Single} + - key: [64, 1024, 3328] + speed: 1308.01 + value: {index: 7, type: Single} + - key: [64, 1408, 256] + speed: 1371.91 + value: {index: 7, type: Single} + - key: [64, 1408, 768] + speed: 1902.75 + value: {index: 7, type: Single} + - key: [64, 1408, 1792] + speed: 2116.45 + value: {index: 7, type: Single} + - key: [64, 1408, 3328] + speed: 2189.53 + value: {index: 7, type: Single} + - key: [64, 1856, 256] + speed: 1732.1 + value: {index: 7, type: Single} + - key: [64, 1856, 768] + speed: 2382.04 + value: {index: 7, type: Single} + - key: [64, 1856, 1792] + speed: 2680.55 + value: {index: 7, type: Single} + - key: [64, 1856, 3328] + speed: 2794.03 + value: {index: 7, type: Single} + - key: [64, 2368, 256] + speed: 1932.67 + value: {index: 7, type: Single} + - key: [64, 2368, 768] + speed: 2474.4 + value: {index: 7, type: Single} + - key: [64, 2368, 1792] + speed: 2703.73 + value: {index: 7, type: Single} + - key: [64, 2368, 3328] + speed: 2763.3 + value: {index: 7, type: Single} + - key: [64, 2944, 256] + speed: 2237.58 + value: {index: 7, type: Single} + - key: [64, 2944, 768] + speed: 3019.25 + value: {index: 7, type: Single} + - key: [64, 2944, 1792] + speed: 3293.37 + value: {index: 7, type: Single} + - key: [64, 2944, 3328] + speed: 3416.04 + value: {index: 7, type: Single} + - key: [64, 3584, 256] + speed: 2469.47 + value: {index: 9, type: Single} + - key: [64, 3584, 768] + speed: 3363.64 + value: {index: 9, type: Single} + - key: [64, 3584, 1792] + speed: 3696.82 + value: {index: 9, type: Single} + - key: [64, 3584, 3328] + speed: 3828.06 + value: {index: 9, type: Single} + - key: [64, 4288, 256] + speed: 2296.39 + value: {index: 7, type: Single} + - key: [64, 4288, 768] + speed: 2797.69 + value: {index: 7, type: Single} + - key: [64, 4288, 1792] + speed: 2974.42 + value: {index: 7, type: Single} + - key: [64, 4288, 3328] + speed: 3069.3 + value: {index: 7, type: Single} + - key: [64, 5056, 256] + speed: 2570.76 + value: {index: 7, type: Single} + - key: [64, 5056, 768] + speed: 3253.97 + value: {index: 7, type: Single} + - key: [64, 5056, 1792] + speed: 3500.87 + value: {index: 7, type: Single} + - key: [64, 5056, 3328] + speed: 3593.09 + value: {index: 7, type: Single} + - key: [64, 5888, 256] + speed: 2724.5 + value: {index: 7, type: Single} + - key: [64, 5888, 768] + speed: 3800.49 + value: {index: 7, type: Single} + - key: [64, 5888, 1792] + speed: 4102.66 + value: {index: 7, type: Single} + - key: [64, 5888, 3328] + speed: 4045.37 + value: {index: 7, type: Single} + - key: [64, 6784, 256] + speed: 2971.22 + value: {index: 7, type: Single} + - key: [64, 6784, 768] + speed: 3683.64 + value: {index: 10, type: Single} + - key: [64, 6784, 1792] + speed: 3929.41 + value: {index: 10, type: Single} + - key: [64, 6784, 3328] + speed: 4036.04 + value: {index: 10, type: Single} + - key: [128, 4, 256] + speed: 10.4703 + value: {index: 12, type: Single} + - key: [128, 4, 768] + speed: 13.1394 + value: {index: 12, type: Single} + - key: [128, 4, 1792] + speed: 11.9556 + value: {index: 12, type: Single} + - key: [128, 4, 3328] + speed: 12.3803 + value: {index: 12, type: Single} + - key: [128, 64, 256] + speed: 130.465 + value: {index: 7, type: Single} + - key: [128, 64, 768] + speed: 206.145 + value: {index: 7, type: Single} + - key: [128, 64, 1792] + speed: 231.244 + value: {index: 7, type: Single} + - key: [128, 64, 3328] + speed: 223.597 + value: {index: 7, type: Single} + - key: [128, 128, 256] + speed: 299.582 + value: {index: 7, type: Single} + - key: [128, 128, 768] + speed: 410.308 + value: {index: 7, type: Single} + - key: [128, 128, 1792] + speed: 460.342 + value: {index: 7, type: Single} + - key: [128, 128, 3328] + speed: 408.028 + value: {index: 7, type: Single} + - key: [128, 256, 256] + speed: 592.897 + value: {index: 7, type: Single} + - key: [128, 256, 768] + speed: 822.587 + value: {index: 7, type: Single} + - key: [128, 256, 1792] + speed: 840.614 + value: {index: 7, type: Single} + - key: [128, 256, 3328] + speed: 827.524 + value: {index: 7, type: Single} + - key: [128, 448, 256] + speed: 850.525 + value: {index: 7, type: Single} + - key: [128, 448, 768] + speed: 1385.85 + value: {index: 7, type: Single} + - key: [128, 448, 1792] + speed: 1368.08 + value: {index: 7, type: Single} + - key: [128, 448, 3328] + speed: 1417.88 + value: {index: 7, type: Single} + - key: [128, 704, 256] + speed: 1336.54 + value: {index: 7, type: Single} + - key: [128, 704, 768] + speed: 2062.37 + value: {index: 7, type: Single} + - key: [128, 704, 1792] + speed: 2137.19 + value: {index: 7, type: Single} + - key: [128, 704, 3328] + speed: 2188.35 + value: {index: 7, type: Single} + - key: [128, 1024, 256] + speed: 1856.45 + value: {index: 7, type: Single} + - key: [128, 1024, 768] + speed: 2493.42 + value: {index: 7, type: Single} + - key: [128, 1024, 1792] + speed: 2828.56 + value: {index: 7, type: Single} + - key: [128, 1024, 3328] + speed: 2993.72 + value: {index: 7, type: Single} + - key: [128, 1408, 256] + speed: 2177.78 + value: {index: 11, type: Single} + - key: [128, 1408, 768] + speed: 2797.19 + value: {index: 7, type: Single} + - key: [128, 1408, 1792] + speed: 3132.1 + value: {index: 7, type: Single} + - key: [128, 1408, 3328] + speed: 3225.86 + value: {index: 7, type: Single} + - key: [128, 1856, 256] + speed: 2783.09 + value: {index: 7, type: Single} + - key: [128, 1856, 768] + speed: 3748.95 + value: {index: 7, type: Single} + - key: [128, 1856, 1792] + speed: 4108.0 + value: {index: 7, type: Single} + - key: [128, 1856, 3328] + speed: 4279.54 + value: {index: 7, type: Single} + - key: [128, 2368, 256] + speed: 2735.0 + value: {index: 7, type: Single} + - key: [128, 2368, 768] + speed: 3180.67 + value: {index: 7, type: Single} + - key: [128, 2368, 1792] + speed: 3343.6 + value: {index: 7, type: Single} + - key: [128, 2368, 3328] + speed: 3455.35 + value: {index: 7, type: Single} + - key: [128, 2944, 256] + speed: 3207.61 + value: {index: 7, type: Single} + - key: [128, 2944, 768] + speed: 3731.52 + value: {index: 7, type: Single} + - key: [128, 2944, 1792] + speed: 4071.5 + value: {index: 7, type: Single} + - key: [128, 2944, 3328] + speed: 4233.53 + value: {index: 7, type: Single} + - key: [128, 3584, 256] + speed: 3170.81 + value: {index: 10, type: Single} + - key: [128, 3584, 768] + speed: 3885.8 + value: {index: 10, type: Single} + - key: [128, 3584, 1792] + speed: 4263.49 + value: {index: 10, type: Single} + - key: [128, 3584, 3328] + speed: 4272.45 + value: {index: 10, type: Single} + - key: [128, 4288, 256] + speed: 3069.29 + value: {index: 7, type: Single} + - key: [128, 4288, 768] + speed: 3673.65 + value: {index: 7, type: Single} + - key: [128, 4288, 1792] + speed: 3908.68 + value: {index: 7, type: Single} + - key: [128, 4288, 3328] + speed: 3995.26 + value: {index: 7, type: Single} + - key: [128, 5056, 256] + speed: 3550.08 + value: {index: 7, type: Single} + - key: [128, 5056, 768] + speed: 4287.32 + value: {index: 7, type: Single} + - key: [128, 5056, 1792] + speed: 4575.08 + value: {index: 7, type: Single} + - key: [128, 5056, 3328] + speed: 4688.05 + value: {index: 7, type: Single} + - key: [128, 5888, 256] + speed: 3577.76 + value: {index: 7, type: Single} + - key: [128, 5888, 768] + speed: 4258.21 + value: {index: 7, type: Single} + - key: [128, 5888, 1792] + speed: 4547.82 + value: {index: 7, type: Single} + - key: [128, 5888, 3328] + speed: 4649.15 + value: {index: 7, type: Single} + - key: [128, 6784, 256] + speed: 3907.51 + value: {index: 10, type: Single} + - key: [128, 6784, 768] + speed: 4597.99 + value: {index: 10, type: Single} + - key: [128, 6784, 1792] + speed: 4789.48 + value: {index: 10, type: Single} + - key: [128, 6784, 3328] + speed: 4829.46 + value: {index: 10, type: Single} + - key: [256, 4, 256] + speed: 19.8812 + value: {index: 12, type: Single} + - key: [256, 4, 768] + speed: 25.8942 + value: {index: 12, type: Single} + - key: [256, 4, 1792] + speed: 24.0272 + value: {index: 12, type: Single} + - key: [256, 4, 3328] + speed: 24.9756 + value: {index: 12, type: Single} + - key: [256, 64, 256] + speed: 260.929 + value: {index: 7, type: Single} + - key: [256, 64, 768] + speed: 411.3 + value: {index: 7, type: Single} + - key: [256, 64, 1792] + speed: 446.844 + value: {index: 7, type: Single} + - key: [256, 64, 3328] + speed: 404.886 + value: {index: 7, type: Single} + - key: [256, 128, 256] + speed: 534.17 + value: {index: 7, type: Single} + - key: [256, 128, 768] + speed: 820.602 + value: {index: 7, type: Single} + - key: [256, 128, 1792] + speed: 833.55 + value: {index: 7, type: Single} + - key: [256, 128, 3328] + speed: 813.799 + value: {index: 7, type: Single} + - key: [256, 256, 256] + speed: 1034.19 + value: {index: 7, type: Single} + - key: [256, 256, 768] + speed: 1629.38 + value: {index: 7, type: Single} + - key: [256, 256, 1792] + speed: 1594.98 + value: {index: 7, type: Single} + - key: [256, 256, 3328] + speed: 1665.34 + value: {index: 7, type: Single} + - key: [256, 448, 256] + speed: 1604.69 + value: {index: 7, type: Single} + - key: [256, 448, 768] + speed: 2696.27 + value: {index: 7, type: Single} + - key: [256, 448, 1792] + speed: 2610.04 + value: {index: 7, type: Single} + - key: [256, 448, 3328] + speed: 2734.91 + value: {index: 7, type: Single} + - key: [256, 704, 256] + speed: 2208.64 + value: {index: 11, type: Single} + - key: [256, 704, 768] + speed: 2879.06 + value: {index: 7, type: Single} + - key: [256, 704, 1792] + speed: 3123.12 + value: {index: 7, type: Single} + - key: [256, 704, 3328] + speed: 3227.14 + value: {index: 7, type: Single} + - key: [256, 1024, 256] + speed: 2980.1 + value: {index: 7, type: Single} + - key: [256, 1024, 768] + speed: 3938.89 + value: {index: 7, type: Single} + - key: [256, 1024, 1792] + speed: 4370.53 + value: {index: 7, type: Single} + - key: [256, 1024, 3328] + speed: 4547.22 + value: {index: 7, type: Single} + - key: [256, 1408, 256] + speed: 3023.52 + value: {index: 7, type: Single} + - key: [256, 1408, 768] + speed: 3759.59 + value: {index: 7, type: Single} + - key: [256, 1408, 1792] + speed: 3977.99 + value: {index: 7, type: Single} + - key: [256, 1408, 3328] + speed: 4093.48 + value: {index: 7, type: Single} + - key: [256, 1856, 256] + speed: 3344.23 + value: {index: 7, type: Single} + - key: [256, 1856, 768] + speed: 4034.48 + value: {index: 7, type: Single} + - key: [256, 1856, 1792] + speed: 4265.01 + value: {index: 10, type: Single} + - key: [256, 1856, 3328] + speed: 4532.14 + value: {index: 10, type: Single} + - key: [256, 2368, 256] + speed: 3503.34 + value: {index: 7, type: Single} + - key: [256, 2368, 768] + speed: 4091.81 + value: {index: 7, type: Single} + - key: [256, 2368, 1792] + speed: 4288.02 + value: {index: 7, type: Single} + - key: [256, 2368, 3328] + speed: 4418.4 + value: {index: 7, type: Single} + - key: [256, 2944, 256] + speed: 3642.81 + value: {index: 7, type: Single} + - key: [256, 2944, 768] + speed: 4355.5 + value: {index: 7, type: Single} + - key: [256, 2944, 1792] + speed: 4622.76 + value: {index: 7, type: Single} + - key: [256, 2944, 3328] + speed: 4729.02 + value: {index: 7, type: Single} + - key: [256, 3584, 256] + speed: 4059.96 + value: {index: 10, type: Single} + - key: [256, 3584, 768] + speed: 4794.58 + value: {index: 10, type: Single} + - key: [256, 3584, 1792] + speed: 5013.71 + value: {index: 10, type: Single} + - key: [256, 3584, 3328] + speed: 5083.32 + value: {index: 10, type: Single} + - key: [256, 4288, 256] + speed: 3744.96 + value: {index: 7, type: Single} + - key: [256, 4288, 768] + speed: 4333.95 + value: {index: 7, type: Single} + - key: [256, 4288, 1792] + speed: 4560.46 + value: {index: 7, type: Single} + - key: [256, 4288, 3328] + speed: 4653.47 + value: {index: 7, type: Single} + - key: [256, 5056, 256] + speed: 3972.57 + value: {index: 7, type: Single} + - key: [256, 5056, 768] + speed: 4628.96 + value: {index: 7, type: Single} + - key: [256, 5056, 1792] + speed: 4859.06 + value: {index: 7, type: Single} + - key: [256, 5056, 3328] + speed: 4967.16 + value: {index: 7, type: Single} + - key: [256, 5888, 256] + speed: 4414.58 + value: {index: 10, type: Single} + - key: [256, 5888, 768] + speed: 5306.48 + value: {index: 10, type: Single} + - key: [256, 5888, 1792] + speed: 5562.83 + value: {index: 10, type: Single} + - key: [256, 5888, 3328] + speed: 5706.54 + value: {index: 10, type: Single} + - key: [256, 6784, 256] + speed: 4432.71 + value: {index: 8, type: Single} + - key: [256, 6784, 768] + speed: 4884.89 + value: {index: 8, type: Single} + - key: [256, 6784, 1792] + speed: 5051.5 + value: {index: 8, type: Single} + - key: [256, 6784, 3328] + speed: 5098.99 + value: {index: 8, type: Single} + - key: [448, 4, 256] + speed: 31.1198 + value: {index: 12, type: Single} + - key: [448, 4, 768] + speed: 41.3787 + value: {index: 12, type: Single} + - key: [448, 4, 1792] + speed: 44.7838 + value: {index: 12, type: Single} + - key: [448, 4, 3328] + speed: 43.331 + value: {index: 12, type: Single} + - key: [448, 64, 256] + speed: 458.738 + value: {index: 11, type: Single} + - key: [448, 64, 768] + speed: 725.037 + value: {index: 7, type: Single} + - key: [448, 64, 1792] + speed: 706.325 + value: {index: 7, type: Single} + - key: [448, 64, 3328] + speed: 710.896 + value: {index: 7, type: Single} + - key: [448, 128, 256] + speed: 921.738 + value: {index: 7, type: Single} + - key: [448, 128, 768] + speed: 1443.02 + value: {index: 7, type: Single} + - key: [448, 128, 1792] + speed: 1394.2 + value: {index: 7, type: Single} + - key: [448, 128, 3328] + speed: 1432.07 + value: {index: 7, type: Single} + - key: [448, 256, 256] + speed: 1730.78 + value: {index: 7, type: Single} + - key: [448, 256, 768] + speed: 2817.67 + value: {index: 7, type: Single} + - key: [448, 256, 1792] + speed: 2768.93 + value: {index: 7, type: Single} + - key: [448, 256, 3328] + speed: 2902.85 + value: {index: 7, type: Single} + - key: [448, 448, 256] + speed: 2416.81 + value: {index: 7, type: Single} + - key: [448, 448, 768] + speed: 3266.66 + value: {index: 7, type: Single} + - key: [448, 448, 1792] + speed: 3513.25 + value: {index: 7, type: Single} + - key: [448, 448, 3328] + speed: 3588.14 + value: {index: 7, type: Single} + - key: [448, 704, 256] + speed: 2883.48 + value: {index: 7, type: Single} + - key: [448, 704, 768] + speed: 3392.02 + value: {index: 7, type: Single} + - key: [448, 704, 1792] + speed: 3560.32 + value: {index: 7, type: Single} + - key: [448, 704, 3328] + speed: 3635.09 + value: {index: 7, type: Single} + - key: [448, 1024, 256] + speed: 3183.58 + value: {index: 10, type: Single} + - key: [448, 1024, 768] + speed: 3921.04 + value: {index: 10, type: Single} + - key: [448, 1024, 1792] + speed: 4158.06 + value: {index: 10, type: Single} + - key: [448, 1024, 3328] + speed: 4271.57 + value: {index: 10, type: Single} + - key: [448, 1408, 256] + speed: 3639.29 + value: {index: 7, type: Single} + - key: [448, 1408, 768] + speed: 4342.49 + value: {index: 7, type: Single} + - key: [448, 1408, 1792] + speed: 4530.74 + value: {index: 7, type: Single} + - key: [448, 1408, 3328] + speed: 4643.48 + value: {index: 7, type: Single} + - key: [448, 1856, 256] + speed: 3790.97 + value: {index: 10, type: Single} + - key: [448, 1856, 768] + speed: 4334.16 + value: {index: 10, type: Single} + - key: [448, 1856, 1792] + speed: 4537.53 + value: {index: 10, type: Single} + - key: [448, 1856, 3328] + speed: 4611.88 + value: {index: 10, type: Single} + - key: [448, 2368, 256] + speed: 3695.81 + value: {index: 7, type: Single} + - key: [448, 2368, 768] + speed: 4286.35 + value: {index: 7, type: Single} + - key: [448, 2368, 1792] + speed: 4453.2 + value: {index: 7, type: Single} + - key: [448, 2368, 3328] + speed: 4531.83 + value: {index: 7, type: Single} + - key: [448, 2944, 256] + speed: 4320.43 + value: {index: 10, type: Single} + - key: [448, 2944, 768] + speed: 4756.23 + value: {index: 10, type: Single} + - key: [448, 2944, 1792] + speed: 4934.48 + value: {index: 10, type: Single} + - key: [448, 2944, 3328] + speed: 5009.69 + value: {index: 10, type: Single} + - key: [448, 3584, 256] + speed: 4172.12 + value: {index: 10, type: Single} + - key: [448, 3584, 768] + speed: 4501.57 + value: {index: 10, type: Single} + - key: [448, 3584, 1792] + speed: 4637.36 + value: {index: 10, type: Single} + - key: [448, 3584, 3328] + speed: 4693.0 + value: {index: 10, type: Single} + - key: [448, 4288, 256] + speed: 4779.62 + value: {index: 10, type: Single} + - key: [448, 4288, 768] + speed: 5311.12 + value: {index: 10, type: Single} + - key: [448, 4288, 1792] + speed: 5516.0 + value: {index: 10, type: Single} + - key: [448, 4288, 3328] + speed: 5587.57 + value: {index: 10, type: Single} + - key: [448, 5056, 256] + speed: 4604.68 + value: {index: 10, type: Single} + - key: [448, 5056, 768] + speed: 5096.33 + value: {index: 10, type: Single} + - key: [448, 5056, 1792] + speed: 5268.31 + value: {index: 10, type: Single} + - key: [448, 5056, 3328] + speed: 5331.55 + value: {index: 10, type: Single} + - key: [448, 5888, 256] + speed: 4762.84 + value: {index: 10, type: Single} + - key: [448, 5888, 768] + speed: 5059.81 + value: {index: 10, type: Single} + - key: [448, 5888, 1792] + speed: 5193.09 + value: {index: 10, type: Single} + - key: [448, 5888, 3328] + speed: 5259.36 + value: {index: 8, type: Single} + - key: [448, 6784, 256] + speed: 5411.29 + value: {index: 10, type: Single} + - key: [448, 6784, 768] + speed: 5800.8 + value: {index: 10, type: Single} + - key: [448, 6784, 1792] + speed: 5962.96 + value: {index: 10, type: Single} + - key: [448, 6784, 3328] + speed: 6021.49 + value: {index: 10, type: Single} + - key: [704, 4, 256] + speed: 46.565 + value: {index: 12, type: Single} + - key: [704, 4, 768] + speed: 60.0738 + value: {index: 12, type: Single} + - key: [704, 4, 1792] + speed: 61.7617 + value: {index: 12, type: Single} + - key: [704, 4, 3328] + speed: 56.5301 + value: {index: 12, type: Single} + - key: [704, 64, 256] + speed: 704.559 + value: {index: 7, type: Single} + - key: [704, 64, 768] + speed: 1076.35 + value: {index: 7, type: Single} + - key: [704, 64, 1792] + speed: 1082.39 + value: {index: 7, type: Single} + - key: [704, 64, 3328] + speed: 1114.66 + value: {index: 7, type: Single} + - key: [704, 128, 256] + speed: 1475.92 + value: {index: 7, type: Single} + - key: [704, 128, 768] + speed: 2118.5 + value: {index: 7, type: Single} + - key: [704, 128, 1792] + speed: 2171.24 + value: {index: 7, type: Single} + - key: [704, 128, 3328] + speed: 2252.89 + value: {index: 7, type: Single} + - key: [704, 256, 256] + speed: 2404.8 + value: {index: 7, type: Single} + - key: [704, 256, 768] + speed: 2999.24 + value: {index: 7, type: Single} + - key: [704, 256, 1792] + speed: 3111.97 + value: {index: 7, type: Single} + - key: [704, 256, 3328] + speed: 3216.89 + value: {index: 7, type: Single} + - key: [704, 448, 256] + speed: 2914.34 + value: {index: 7, type: Single} + - key: [704, 448, 768] + speed: 3378.0 + value: {index: 7, type: Single} + - key: [704, 448, 1792] + speed: 3561.99 + value: {index: 7, type: Single} + - key: [704, 448, 3328] + speed: 3619.31 + value: {index: 7, type: Single} + - key: [704, 704, 256] + speed: 3502.68 + value: {index: 7, type: Single} + - key: [704, 704, 768] + speed: 4299.92 + value: {index: 7, type: Single} + - key: [704, 704, 1792] + speed: 4490.51 + value: {index: 7, type: Single} + - key: [704, 704, 3328] + speed: 4591.03 + value: {index: 7, type: Single} + - key: [704, 1024, 256] + speed: 3445.85 + value: {index: 10, type: Single} + - key: [704, 1024, 768] + speed: 3842.71 + value: {index: 10, type: Single} + - key: [704, 1024, 1792] + speed: 3961.71 + value: {index: 10, type: Single} + - key: [704, 1024, 3328] + speed: 4043.4 + value: {index: 9, type: Single} + - key: [704, 1408, 256] + speed: 4724.99 + value: {index: 10, type: Single} + - key: [704, 1408, 768] + speed: 5272.89 + value: {index: 10, type: Single} + - key: [704, 1408, 1792] + speed: 5437.49 + value: {index: 10, type: Single} + - key: [704, 1408, 3328] + speed: 5518.34 + value: {index: 10, type: Single} + - key: [704, 1856, 256] + speed: 4135.14 + value: {index: 10, type: Single} + - key: [704, 1856, 768] + speed: 4746.58 + value: {index: 7, type: Single} + - key: [704, 1856, 1792] + speed: 4951.3 + value: {index: 7, type: Single} + - key: [704, 1856, 3328] + speed: 5031.5 + value: {index: 7, type: Single} + - key: [704, 2368, 256] + speed: 4211.44 + value: {index: 10, type: Single} + - key: [704, 2368, 768] + speed: 4708.2 + value: {index: 7, type: Single} + - key: [704, 2368, 1792] + speed: 4904.39 + value: {index: 7, type: Single} + - key: [704, 2368, 3328] + speed: 4980.47 + value: {index: 7, type: Single} + - key: [704, 2944, 256] + speed: 5365.27 + value: {index: 10, type: Single} + - key: [704, 2944, 768] + speed: 5784.1 + value: {index: 10, type: Single} + - key: [704, 2944, 1792] + speed: 5958.23 + value: {index: 10, type: Single} + - key: [704, 2944, 3328] + speed: 6031.51 + value: {index: 10, type: Single} + - key: [704, 3584, 256] + speed: 5362.69 + value: {index: 10, type: Single} + - key: [704, 3584, 768] + speed: 5750.55 + value: {index: 10, type: Single} + - key: [704, 3584, 1792] + speed: 5891.15 + value: {index: 10, type: Single} + - key: [704, 3584, 3328] + speed: 5946.11 + value: {index: 10, type: Single} + - key: [704, 4288, 256] + speed: 5290.33 + value: {index: 10, type: Single} + - key: [704, 4288, 768] + speed: 5756.42 + value: {index: 10, type: Single} + - key: [704, 4288, 1792] + speed: 5911.75 + value: {index: 10, type: Single} + - key: [704, 4288, 3328] + speed: 5973.01 + value: {index: 10, type: Single} + - key: [704, 5056, 256] + speed: 5459.86 + value: {index: 10, type: Single} + - key: [704, 5056, 768] + speed: 5862.3 + value: {index: 10, type: Single} + - key: [704, 5056, 1792] + speed: 6013.09 + value: {index: 10, type: Single} + - key: [704, 5056, 3328] + speed: 6073.22 + value: {index: 10, type: Single} + - key: [704, 5888, 256] + speed: 5646.55 + value: {index: 10, type: Single} + - key: [704, 5888, 768] + speed: 6037.64 + value: {index: 10, type: Single} + - key: [704, 5888, 1792] + speed: 6175.08 + value: {index: 10, type: Single} + - key: [704, 5888, 3328] + speed: 6224.02 + value: {index: 10, type: Single} + - key: [704, 6784, 256] + speed: 5376.3 + value: {index: 10, type: Single} + - key: [704, 6784, 768] + speed: 5666.06 + value: {index: 8, type: Single} + - key: [704, 6784, 1792] + speed: 5792.18 + value: {index: 8, type: Single} + - key: [704, 6784, 3328] + speed: 5861.39 + value: {index: 8, type: Single} + - key: [1024, 4, 256] + speed: 66.1479 + value: {index: 12, type: Single} + - key: [1024, 4, 768] + speed: 86.8434 + value: {index: 12, type: Single} + - key: [1024, 4, 1792] + speed: 83.2675 + value: {index: 12, type: Single} + - key: [1024, 4, 3328] + speed: 82.9671 + value: {index: 12, type: Single} + - key: [1024, 64, 256] + speed: 1011.1 + value: {index: 7, type: Single} + - key: [1024, 64, 768] + speed: 1415.54 + value: {index: 7, type: Single} + - key: [1024, 64, 1792] + speed: 1542.23 + value: {index: 7, type: Single} + - key: [1024, 64, 3328] + speed: 1620.43 + value: {index: 7, type: Single} + - key: [1024, 128, 256] + speed: 2040.4 + value: {index: 7, type: Single} + - key: [1024, 128, 768] + speed: 2819.31 + value: {index: 7, type: Single} + - key: [1024, 128, 1792] + speed: 3014.08 + value: {index: 7, type: Single} + - key: [1024, 128, 3328] + speed: 3159.15 + value: {index: 7, type: Single} + - key: [1024, 256, 256] + speed: 3113.23 + value: {index: 7, type: Single} + - key: [1024, 256, 768] + speed: 4111.68 + value: {index: 7, type: Single} + - key: [1024, 256, 1792] + speed: 4484.9 + value: {index: 7, type: Single} + - key: [1024, 256, 3328] + speed: 4647.71 + value: {index: 7, type: Single} + - key: [1024, 448, 256] + speed: 3202.85 + value: {index: 7, type: Single} + - key: [1024, 448, 768] + speed: 3870.01 + value: {index: 7, type: Single} + - key: [1024, 448, 1792] + speed: 4110.33 + value: {index: 10, type: Single} + - key: [1024, 448, 3328] + speed: 4231.23 + value: {index: 10, type: Single} + - key: [1024, 704, 256] + speed: 3455.4 + value: {index: 7, type: Single} + - key: [1024, 704, 768] + speed: 4145.34 + value: {index: 7, type: Single} + - key: [1024, 704, 1792] + speed: 4387.3 + value: {index: 7, type: Single} + - key: [1024, 704, 3328] + speed: 4503.91 + value: {index: 7, type: Single} + - key: [1024, 1024, 256] + speed: 4718.5 + value: {index: 10, type: Single} + - key: [1024, 1024, 768] + speed: 5485.05 + value: {index: 10, type: Single} + - key: [1024, 1024, 1792] + speed: 5676.11 + value: {index: 10, type: Single} + - key: [1024, 1024, 3328] + speed: 5841.92 + value: {index: 10, type: Single} + - key: [1024, 1408, 256] + speed: 4554.58 + value: {index: 10, type: Single} + - key: [1024, 1408, 768] + speed: 5179.53 + value: {index: 10, type: Single} + - key: [1024, 1408, 1792] + speed: 5381.73 + value: {index: 10, type: Single} + - key: [1024, 1408, 3328] + speed: 5452.0 + value: {index: 10, type: Single} + - key: [1024, 1856, 256] + speed: 4658.24 + value: {index: 8, type: Single} + - key: [1024, 1856, 768] + speed: 5269.96 + value: {index: 8, type: Single} + - key: [1024, 1856, 1792] + speed: 5481.26 + value: {index: 8, type: Single} + - key: [1024, 1856, 3328] + speed: 5548.48 + value: {index: 8, type: Single} + - key: [1024, 2368, 256] + speed: 4955.67 + value: {index: 10, type: Single} + - key: [1024, 2368, 768] + speed: 5469.99 + value: {index: 10, type: Single} + - key: [1024, 2368, 1792] + speed: 5614.48 + value: {index: 10, type: Single} + - key: [1024, 2368, 3328] + speed: 5694.13 + value: {index: 10, type: Single} + - key: [1024, 2944, 256] + speed: 5243.27 + value: {index: 8, type: Single} + - key: [1024, 2944, 768] + speed: 5767.71 + value: {index: 8, type: Single} + - key: [1024, 2944, 1792] + speed: 5953.86 + value: {index: 8, type: Single} + - key: [1024, 2944, 3328] + speed: 6021.66 + value: {index: 8, type: Single} + - key: [1024, 3584, 256] + speed: 5349.33 + value: {index: 10, type: Single} + - key: [1024, 3584, 768] + speed: 5924.91 + value: {index: 10, type: Single} + - key: [1024, 3584, 1792] + speed: 6154.79 + value: {index: 10, type: Single} + - key: [1024, 3584, 3328] + speed: 6245.07 + value: {index: 10, type: Single} + - key: [1024, 4288, 256] + speed: 5171.98 + value: {index: 10, type: Single} + - key: [1024, 4288, 768] + speed: 5607.08 + value: {index: 10, type: Single} + - key: [1024, 4288, 1792] + speed: 5800.06 + value: {index: 10, type: Single} + - key: [1024, 4288, 3328] + speed: 5857.19 + value: {index: 10, type: Single} + - key: [1024, 5056, 256] + speed: 5537.74 + value: {index: 8, type: Single} + - key: [1024, 5056, 768] + speed: 6094.85 + value: {index: 8, type: Single} + - key: [1024, 5056, 1792] + speed: 6278.69 + value: {index: 8, type: Single} + - key: [1024, 5056, 3328] + speed: 6349.17 + value: {index: 8, type: Single} + - key: [1024, 5888, 256] + speed: 5543.18 + value: {index: 8, type: Single} + - key: [1024, 5888, 768] + speed: 6007.73 + value: {index: 8, type: Single} + - key: [1024, 5888, 1792] + speed: 6161.87 + value: {index: 8, type: Single} + - key: [1024, 5888, 3328] + speed: 6214.94 + value: {index: 8, type: Single} + - key: [1024, 6784, 256] + speed: 5491.22 + value: {index: 8, type: Single} + - key: [1024, 6784, 768] + speed: 5962.65 + value: {index: 8, type: Single} + - key: [1024, 6784, 1792] + speed: 6120.82 + value: {index: 8, type: Single} + - key: [1024, 6784, 3328] + speed: 6165.65 + value: {index: 8, type: Single} + - key: [1408, 4, 256] + speed: 90.1092 + value: {index: 12, type: Single} + - key: [1408, 4, 768] + speed: 107.932 + value: {index: 12, type: Single} + - key: [1408, 4, 1792] + speed: 110.41 + value: {index: 12, type: Single} + - key: [1408, 4, 3328] + speed: 117.034 + value: {index: 12, type: Single} + - key: [1408, 64, 256] + speed: 1428.53 + value: {index: 7, type: Single} + - key: [1408, 64, 768] + speed: 1971.01 + value: {index: 7, type: Single} + - key: [1408, 64, 1792] + speed: 2130.93 + value: {index: 7, type: Single} + - key: [1408, 64, 3328] + speed: 2206.25 + value: {index: 7, type: Single} + - key: [1408, 128, 256] + speed: 2341.52 + value: {index: 7, type: Single} + - key: [1408, 128, 768] + speed: 2892.44 + value: {index: 7, type: Single} + - key: [1408, 128, 1792] + speed: 3127.59 + value: {index: 7, type: Single} + - key: [1408, 128, 3328] + speed: 3219.44 + value: {index: 7, type: Single} + - key: [1408, 256, 256] + speed: 2980.11 + value: {index: 7, type: Single} + - key: [1408, 256, 768] + speed: 3663.75 + value: {index: 7, type: Single} + - key: [1408, 256, 1792] + speed: 3920.76 + value: {index: 7, type: Single} + - key: [1408, 256, 3328] + speed: 3995.51 + value: {index: 7, type: Single} + - key: [1408, 448, 256] + speed: 3657.63 + value: {index: 7, type: Single} + - key: [1408, 448, 768] + speed: 4241.12 + value: {index: 7, type: Single} + - key: [1408, 448, 1792] + speed: 4521.35 + value: {index: 7, type: Single} + - key: [1408, 448, 3328] + speed: 4621.52 + value: {index: 7, type: Single} + - key: [1408, 704, 256] + speed: 3810.48 + value: {index: 7, type: Single} + - key: [1408, 704, 768] + speed: 4410.66 + value: {index: 7, type: Single} + - key: [1408, 704, 1792] + speed: 4648.95 + value: {index: 7, type: Single} + - key: [1408, 704, 3328] + speed: 4735.03 + value: {index: 7, type: Single} + - key: [1408, 1024, 256] + speed: 4635.08 + value: {index: 10, type: Single} + - key: [1408, 1024, 768] + speed: 5215.68 + value: {index: 10, type: Single} + - key: [1408, 1024, 1792] + speed: 5412.62 + value: {index: 10, type: Single} + - key: [1408, 1024, 3328] + speed: 5459.81 + value: {index: 10, type: Single} + - key: [1408, 1408, 256] + speed: 5109.03 + value: {index: 10, type: Single} + - key: [1408, 1408, 768] + speed: 5611.16 + value: {index: 8, type: Single} + - key: [1408, 1408, 1792] + speed: 5765.64 + value: {index: 8, type: Single} + - key: [1408, 1408, 3328] + speed: 5820.92 + value: {index: 8, type: Single} + - key: [1408, 1856, 256] + speed: 4676.94 + value: {index: 10, type: Single} + - key: [1408, 1856, 768] + speed: 5021.03 + value: {index: 8, type: Single} + - key: [1408, 1856, 1792] + speed: 5169.53 + value: {index: 8, type: Single} + - key: [1408, 1856, 3328] + speed: 5222.18 + value: {index: 8, type: Single} + - key: [1408, 2368, 256] + speed: 5067.07 + value: {index: 10, type: Single} + - key: [1408, 2368, 768] + speed: 5503.51 + value: {index: 10, type: Single} + - key: [1408, 2368, 1792] + speed: 5631.34 + value: {index: 10, type: Single} + - key: [1408, 2368, 3328] + speed: 5688.19 + value: {index: 10, type: Single} + - key: [1408, 2944, 256] + speed: 5673.39 + value: {index: 10, type: Single} + - key: [1408, 2944, 768] + speed: 6119.31 + value: {index: 8, type: Single} + - key: [1408, 2944, 1792] + speed: 6266.93 + value: {index: 8, type: Single} + - key: [1408, 2944, 3328] + speed: 6321.2 + value: {index: 8, type: Single} + - key: [1408, 3584, 256] + speed: 5629.26 + value: {index: 10, type: Single} + - key: [1408, 3584, 768] + speed: 6022.62 + value: {index: 8, type: Single} + - key: [1408, 3584, 1792] + speed: 6157.7 + value: {index: 8, type: Single} + - key: [1408, 3584, 3328] + speed: 6209.79 + value: {index: 8, type: Single} + - key: [1408, 4288, 256] + speed: 5613.42 + value: {index: 10, type: Single} + - key: [1408, 4288, 768] + speed: 6021.67 + value: {index: 8, type: Single} + - key: [1408, 4288, 1792] + speed: 6165.29 + value: {index: 8, type: Single} + - key: [1408, 4288, 3328] + speed: 6220.1 + value: {index: 8, type: Single} + - key: [1408, 5056, 256] + speed: 5676.54 + value: {index: 10, type: Single} + - key: [1408, 5056, 768] + speed: 6124.0 + value: {index: 8, type: Single} + - key: [1408, 5056, 1792] + speed: 6260.77 + value: {index: 8, type: Single} + - key: [1408, 5056, 3328] + speed: 6312.72 + value: {index: 8, type: Single} + - key: [1408, 5888, 256] + speed: 5939.17 + value: {index: 8, type: Single} + - key: [1408, 5888, 768] + speed: 6306.07 + value: {index: 8, type: Single} + - key: [1408, 5888, 1792] + speed: 6422.46 + value: {index: 8, type: Single} + - key: [1408, 5888, 3328] + speed: 6469.98 + value: {index: 8, type: Single} + - key: [1408, 6784, 256] + speed: 5817.84 + value: {index: 10, type: Single} + - key: [1408, 6784, 768] + speed: 6057.33 + value: {index: 10, type: Single} + - key: [1408, 6784, 1792] + speed: 6135.45 + value: {index: 10, type: Single} + - key: [1408, 6784, 3328] + speed: 6166.1 + value: {index: 10, type: Single} + - key: [1856, 4, 256] + speed: 117.155 + value: {index: 12, type: Single} + - key: [1856, 4, 768] + speed: 139.693 + value: {index: 12, type: Single} + - key: [1856, 4, 1792] + speed: 141.193 + value: {index: 12, type: Single} + - key: [1856, 4, 3328] + speed: 144.077 + value: {index: 12, type: Single} + - key: [1856, 64, 256] + speed: 1900.48 + value: {index: 7, type: Single} + - key: [1856, 64, 768] + speed: 2487.93 + value: {index: 7, type: Single} + - key: [1856, 64, 1792] + speed: 2771.03 + value: {index: 7, type: Single} + - key: [1856, 64, 3328] + speed: 2900.33 + value: {index: 7, type: Single} + - key: [1856, 128, 256] + speed: 2901.11 + value: {index: 7, type: Single} + - key: [1856, 128, 768] + speed: 3818.67 + value: {index: 7, type: Single} + - key: [1856, 128, 1792] + speed: 4110.95 + value: {index: 7, type: Single} + - key: [1856, 128, 3328] + speed: 4225.33 + value: {index: 7, type: Single} + - key: [1856, 256, 256] + speed: 3284.05 + value: {index: 10, type: Single} + - key: [1856, 256, 768] + speed: 4081.27 + value: {index: 10, type: Single} + - key: [1856, 256, 1792] + speed: 4332.53 + value: {index: 10, type: Single} + - key: [1856, 256, 3328] + speed: 4425.96 + value: {index: 10, type: Single} + - key: [1856, 448, 256] + speed: 3605.46 + value: {index: 10, type: Single} + - key: [1856, 448, 768] + speed: 4280.38 + value: {index: 10, type: Single} + - key: [1856, 448, 1792] + speed: 4570.52 + value: {index: 10, type: Single} + - key: [1856, 448, 3328] + speed: 4598.82 + value: {index: 10, type: Single} + - key: [1856, 704, 256] + speed: 4120.07 + value: {index: 10, type: Single} + - key: [1856, 704, 768] + speed: 4705.36 + value: {index: 10, type: Single} + - key: [1856, 704, 1792] + speed: 4915.11 + value: {index: 7, type: Single} + - key: [1856, 704, 3328] + speed: 5001.07 + value: {index: 7, type: Single} + - key: [1856, 1024, 256] + speed: 4890.63 + value: {index: 10, type: Single} + - key: [1856, 1024, 768] + speed: 5338.49 + value: {index: 10, type: Single} + - key: [1856, 1024, 1792] + speed: 5487.8 + value: {index: 8, type: Single} + - key: [1856, 1024, 3328] + speed: 5570.92 + value: {index: 8, type: Single} + - key: [1856, 1408, 256] + speed: 5554.22 + value: {index: 10, type: Single} + - key: [1856, 1408, 768] + speed: 5976.94 + value: {index: 10, type: Single} + - key: [1856, 1408, 1792] + speed: 6106.26 + value: {index: 10, type: Single} + - key: [1856, 1408, 3328] + speed: 6161.06 + value: {index: 10, type: Single} + - key: [1856, 1856, 256] + speed: 5225.95 + value: {index: 10, type: Single} + - key: [1856, 1856, 768] + speed: 5697.86 + value: {index: 10, type: Single} + - key: [1856, 1856, 1792] + speed: 5826.26 + value: {index: 10, type: Single} + - key: [1856, 1856, 3328] + speed: 5880.88 + value: {index: 10, type: Single} + - key: [1856, 2368, 256] + speed: 5351.93 + value: {index: 10, type: Single} + - key: [1856, 2368, 768] + speed: 5740.28 + value: {index: 10, type: Single} + - key: [1856, 2368, 1792] + speed: 5840.23 + value: {index: 10, type: Single} + - key: [1856, 2368, 3328] + speed: 5886.24 + value: {index: 10, type: Single} + - key: [1856, 2944, 256] + speed: 5641.87 + value: {index: 10, type: Single} + - key: [1856, 2944, 768] + speed: 5913.98 + value: {index: 10, type: Single} + - key: [1856, 2944, 1792] + speed: 5985.75 + value: {index: 10, type: Single} + - key: [1856, 2944, 3328] + speed: 6029.61 + value: {index: 10, type: Single} + - key: [1856, 3584, 256] + speed: 5797.83 + value: {index: 10, type: Single} + - key: [1856, 3584, 768] + speed: 6110.69 + value: {index: 10, type: Single} + - key: [1856, 3584, 1792] + speed: 6189.43 + value: {index: 10, type: Single} + - key: [1856, 3584, 3328] + speed: 6229.27 + value: {index: 10, type: Single} + - key: [1856, 4288, 256] + speed: 5681.47 + value: {index: 10, type: Single} + - key: [1856, 4288, 768] + speed: 6034.66 + value: {index: 8, type: Single} + - key: [1856, 4288, 1792] + speed: 6154.0 + value: {index: 8, type: Single} + - key: [1856, 4288, 3328] + speed: 6204.38 + value: {index: 8, type: Single} + - key: [1856, 5056, 256] + speed: 5659.7 + value: {index: 10, type: Single} + - key: [1856, 5056, 768] + speed: 5947.2 + value: {index: 10, type: Single} + - key: [1856, 5056, 1792] + speed: 6028.54 + value: {index: 10, type: Single} + - key: [1856, 5056, 3328] + speed: 6059.26 + value: {index: 10, type: Single} + - key: [1856, 5888, 256] + speed: 6029.16 + value: {index: 10, type: Single} + - key: [1856, 5888, 768] + speed: 6271.8 + value: {index: 10, type: Single} + - key: [1856, 5888, 1792] + speed: 6353.75 + value: {index: 10, type: Single} + - key: [1856, 5888, 3328] + speed: 6383.74 + value: {index: 10, type: Single} + - key: [1856, 6784, 256] + speed: 5914.61 + value: {index: 10, type: Single} + - key: [1856, 6784, 768] + speed: 6103.77 + value: {index: 10, type: Single} + - key: [1856, 6784, 1792] + speed: 6177.55 + value: {index: 10, type: Single} + - key: [1856, 6784, 3328] + speed: 6214.14 + value: {index: 10, type: Single} + - key: [2368, 4, 256] + speed: 141.098 + value: {index: 12, type: Single} + - key: [2368, 4, 768] + speed: 173.504 + value: {index: 12, type: Single} + - key: [2368, 4, 1792] + speed: 175.319 + value: {index: 12, type: Single} + - key: [2368, 4, 3328] + speed: 180.241 + value: {index: 12, type: Single} + - key: [2368, 64, 256] + speed: 1890.8 + value: {index: 7, type: Single} + - key: [2368, 64, 768] + speed: 2478.32 + value: {index: 7, type: Single} + - key: [2368, 64, 1792] + speed: 2664.43 + value: {index: 7, type: Single} + - key: [2368, 64, 3328] + speed: 2759.94 + value: {index: 7, type: Single} + - key: [2368, 128, 256] + speed: 2658.62 + value: {index: 7, type: Single} + - key: [2368, 128, 768] + speed: 3148.79 + value: {index: 7, type: Single} + - key: [2368, 128, 1792] + speed: 3385.28 + value: {index: 7, type: Single} + - key: [2368, 128, 3328] + speed: 3461.5 + value: {index: 7, type: Single} + - key: [2368, 256, 256] + speed: 3183.89 + value: {index: 7, type: Single} + - key: [2368, 256, 768] + speed: 3357.38 + value: {index: 7, type: Single} + - key: [2368, 256, 1792] + speed: 3490.01 + value: {index: 9, type: Single} + - key: [2368, 256, 3328] + speed: 3581.67 + value: {index: 9, type: Single} + - key: [2368, 448, 256] + speed: 3597.91 + value: {index: 7, type: Single} + - key: [2368, 448, 768] + speed: 4196.4 + value: {index: 7, type: Single} + - key: [2368, 448, 1792] + speed: 4418.7 + value: {index: 7, type: Single} + - key: [2368, 448, 3328] + speed: 4505.28 + value: {index: 7, type: Single} + - key: [2368, 704, 256] + speed: 4118.11 + value: {index: 8, type: Single} + - key: [2368, 704, 768] + speed: 4625.04 + value: {index: 7, type: Single} + - key: [2368, 704, 1792] + speed: 4860.65 + value: {index: 7, type: Single} + - key: [2368, 704, 3328] + speed: 4954.12 + value: {index: 7, type: Single} + - key: [2368, 1024, 256] + speed: 5128.53 + value: {index: 10, type: Single} + - key: [2368, 1024, 768] + speed: 5533.8 + value: {index: 10, type: Single} + - key: [2368, 1024, 1792] + speed: 5645.28 + value: {index: 10, type: Single} + - key: [2368, 1024, 3328] + speed: 5317.01 + value: {index: 10, type: Single} + - key: [2368, 1408, 256] + speed: 5263.83 + value: {index: 10, type: Single} + - key: [2368, 1408, 768] + speed: 5573.61 + value: {index: 10, type: Single} + - key: [2368, 1408, 1792] + speed: 5668.14 + value: {index: 10, type: Single} + - key: [2368, 1408, 3328] + speed: 5704.43 + value: {index: 10, type: Single} + - key: [2368, 1856, 256] + speed: 5353.82 + value: {index: 10, type: Single} + - key: [2368, 1856, 768] + speed: 5739.55 + value: {index: 10, type: Single} + - key: [2368, 1856, 1792] + speed: 5825.83 + value: {index: 10, type: Single} + - key: [2368, 1856, 3328] + speed: 5885.54 + value: {index: 10, type: Single} + - key: [2368, 2368, 256] + speed: 5635.0 + value: {index: 10, type: Single} + - key: [2368, 2368, 768] + speed: 6025.11 + value: {index: 10, type: Single} + - key: [2368, 2368, 1792] + speed: 6129.75 + value: {index: 10, type: Single} + - key: [2368, 2368, 3328] + speed: 6172.04 + value: {index: 10, type: Single} + - key: [2368, 2944, 256] + speed: 5690.26 + value: {index: 10, type: Single} + - key: [2368, 2944, 768] + speed: 6045.3 + value: {index: 8, type: Single} + - key: [2368, 2944, 1792] + speed: 6141.36 + value: {index: 8, type: Single} + - key: [2368, 2944, 3328] + speed: 6185.87 + value: {index: 8, type: Single} + - key: [2368, 3584, 256] + speed: 5760.02 + value: {index: 10, type: Single} + - key: [2368, 3584, 768] + speed: 6006.97 + value: {index: 10, type: Single} + - key: [2368, 3584, 1792] + speed: 6093.92 + value: {index: 10, type: Single} + - key: [2368, 3584, 3328] + speed: 6116.6 + value: {index: 10, type: Single} + - key: [2368, 4288, 256] + speed: 5806.94 + value: {index: 10, type: Single} + - key: [2368, 4288, 768] + speed: 6092.94 + value: {index: 10, type: Single} + - key: [2368, 4288, 1792] + speed: 6180.38 + value: {index: 10, type: Single} + - key: [2368, 4288, 3328] + speed: 6222.63 + value: {index: 10, type: Single} + - key: [2368, 5056, 256] + speed: 5779.62 + value: {index: 8, type: Single} + - key: [2368, 5056, 768] + speed: 6118.34 + value: {index: 8, type: Single} + - key: [2368, 5056, 1792] + speed: 6227.76 + value: {index: 8, type: Single} + - key: [2368, 5056, 3328] + speed: 6271.11 + value: {index: 8, type: Single} + - key: [2368, 5888, 256] + speed: 6028.41 + value: {index: 10, type: Single} + - key: [2368, 5888, 768] + speed: 6249.96 + value: {index: 10, type: Single} + - key: [2368, 5888, 1792] + speed: 6329.2 + value: {index: 10, type: Single} + - key: [2368, 5888, 3328] + speed: 6361.2 + value: {index: 10, type: Single} + - key: [2368, 6784, 256] + speed: 6071.48 + value: {index: 10, type: Single} + - key: [2368, 6784, 768] + speed: 6284.3 + value: {index: 10, type: Single} + - key: [2368, 6784, 1792] + speed: 6361.59 + value: {index: 10, type: Single} + - key: [2368, 6784, 3328] + speed: 6392.01 + value: {index: 10, type: Single} + - key: [2944, 4, 256] + speed: 164.767 + value: {index: 12, type: Single} + - key: [2944, 4, 768] + speed: 212.702 + value: {index: 12, type: Single} + - key: [2944, 4, 1792] + speed: 224.668 + value: {index: 12, type: Single} + - key: [2944, 4, 3328] + speed: 199.947 + value: {index: 12, type: Single} + - key: [2944, 64, 256] + speed: 2342.28 + value: {index: 7, type: Single} + - key: [2944, 64, 768] + speed: 3038.01 + value: {index: 7, type: Single} + - key: [2944, 64, 1792] + speed: 3288.62 + value: {index: 7, type: Single} + - key: [2944, 64, 3328] + speed: 3405.05 + value: {index: 7, type: Single} + - key: [2944, 128, 256] + speed: 3049.87 + value: {index: 7, type: Single} + - key: [2944, 128, 768] + speed: 3800.49 + value: {index: 7, type: Single} + - key: [2944, 128, 1792] + speed: 4075.14 + value: {index: 7, type: Single} + - key: [2944, 128, 3328] + speed: 4174.03 + value: {index: 7, type: Single} + - key: [2944, 256, 256] + speed: 3668.48 + value: {index: 10, type: Single} + - key: [2944, 256, 768] + speed: 4038.13 + value: {index: 9, type: Single} + - key: [2944, 256, 1792] + speed: 4287.91 + value: {index: 9, type: Single} + - key: [2944, 256, 3328] + speed: 4402.52 + value: {index: 9, type: Single} + - key: [2944, 448, 256] + speed: 4048.0 + value: {index: 10, type: Single} + - key: [2944, 448, 768] + speed: 4673.32 + value: {index: 10, type: Single} + - key: [2944, 448, 1792] + speed: 4903.38 + value: {index: 10, type: Single} + - key: [2944, 448, 3328] + speed: 4980.22 + value: {index: 10, type: Single} + - key: [2944, 704, 256] + speed: 4198.5 + value: {index: 10, type: Single} + - key: [2944, 704, 768] + speed: 4692.71 + value: {index: 7, type: Single} + - key: [2944, 704, 1792] + speed: 4941.22 + value: {index: 7, type: Single} + - key: [2944, 704, 3328] + speed: 5041.66 + value: {index: 7, type: Single} + - key: [2944, 1024, 256] + speed: 5329.1 + value: {index: 8, type: Single} + - key: [2944, 1024, 768] + speed: 5804.12 + value: {index: 8, type: Single} + - key: [2944, 1024, 1792] + speed: 5958.72 + value: {index: 8, type: Single} + - key: [2944, 1024, 3328] + speed: 6024.07 + value: {index: 8, type: Single} + - key: [2944, 1408, 256] + speed: 5732.42 + value: {index: 10, type: Single} + - key: [2944, 1408, 768] + speed: 6113.21 + value: {index: 8, type: Single} + - key: [2944, 1408, 1792] + speed: 6262.62 + value: {index: 8, type: Single} + - key: [2944, 1408, 3328] + speed: 6317.12 + value: {index: 8, type: Single} + - key: [2944, 1856, 256] + speed: 5482.97 + value: {index: 10, type: Single} + - key: [2944, 1856, 768] + speed: 5869.25 + value: {index: 10, type: Single} + - key: [2944, 1856, 1792] + speed: 5970.07 + value: {index: 10, type: Single} + - key: [2944, 1856, 3328] + speed: 6012.04 + value: {index: 10, type: Single} + - key: [2944, 2368, 256] + speed: 5544.88 + value: {index: 10, type: Single} + - key: [2944, 2368, 768] + speed: 6003.62 + value: {index: 8, type: Single} + - key: [2944, 2368, 1792] + speed: 6127.97 + value: {index: 8, type: Single} + - key: [2944, 2368, 3328] + speed: 6178.79 + value: {index: 8, type: Single} + - key: [2944, 2944, 256] + speed: 5846.75 + value: {index: 10, type: Single} + - key: [2944, 2944, 768] + speed: 6114.08 + value: {index: 10, type: Single} + - key: [2944, 2944, 1792] + speed: 6196.11 + value: {index: 10, type: Single} + - key: [2944, 2944, 3328] + speed: 6240.19 + value: {index: 10, type: Single} + - key: [2944, 3584, 256] + speed: 5839.91 + value: {index: 10, type: Single} + - key: [2944, 3584, 768] + speed: 6067.29 + value: {index: 10, type: Single} + - key: [2944, 3584, 1792] + speed: 6143.07 + value: {index: 10, type: Single} + - key: [2944, 3584, 3328] + speed: 6172.76 + value: {index: 10, type: Single} + - key: [2944, 4288, 256] + speed: 5830.93 + value: {index: 10, type: Single} + - key: [2944, 4288, 768] + speed: 6087.21 + value: {index: 10, type: Single} + - key: [2944, 4288, 1792] + speed: 6175.09 + value: {index: 10, type: Single} + - key: [2944, 4288, 3328] + speed: 6212.23 + value: {index: 10, type: Single} + - key: [2944, 5056, 256] + speed: 5922.95 + value: {index: 10, type: Single} + - key: [2944, 5056, 768] + speed: 6197.94 + value: {index: 10, type: Single} + - key: [2944, 5056, 1792] + speed: 6278.3 + value: {index: 10, type: Single} + - key: [2944, 5056, 3328] + speed: 6316.95 + value: {index: 10, type: Single} + - key: [2944, 5888, 256] + speed: 6043.15 + value: {index: 8, type: Single} + - key: [2944, 5888, 768] + speed: 6325.85 + value: {index: 8, type: Single} + - key: [2944, 5888, 1792] + speed: 6413.89 + value: {index: 8, type: Single} + - key: [2944, 5888, 3328] + speed: 6442.87 + value: {index: 8, type: Single} + - key: [2944, 6784, 256] + speed: 6035.49 + value: {index: 10, type: Single} + - key: [2944, 6784, 768] + speed: 6245.19 + value: {index: 8, type: Single} + - key: [2944, 6784, 1792] + speed: 6313.33 + value: {index: 8, type: Single} + - key: [2944, 6784, 3328] + speed: 6329.78 + value: {index: 10, type: Single} + - key: [3584, 4, 256] + speed: 186.257 + value: {index: 12, type: Single} + - key: [3584, 4, 768] + speed: 248.548 + value: {index: 12, type: Single} + - key: [3584, 4, 1792] + speed: 192.243 + value: {index: 12, type: Single} + - key: [3584, 4, 3328] + speed: 179.408 + value: {index: 12, type: Single} + - key: [3584, 64, 256] + speed: 2791.22 + value: {index: 7, type: Single} + - key: [3584, 64, 768] + speed: 3686.99 + value: {index: 7, type: Single} + - key: [3584, 64, 1792] + speed: 4003.54 + value: {index: 7, type: Single} + - key: [3584, 64, 3328] + speed: 4156.98 + value: {index: 7, type: Single} + - key: [3584, 128, 256] + speed: 3196.4 + value: {index: 7, type: Single} + - key: [3584, 128, 768] + speed: 3914.6 + value: {index: 10, type: Single} + - key: [3584, 128, 1792] + speed: 4161.17 + value: {index: 10, type: Single} + - key: [3584, 128, 3328] + speed: 4265.37 + value: {index: 10, type: Single} + - key: [3584, 256, 256] + speed: 3973.46 + value: {index: 10, type: Single} + - key: [3584, 256, 768] + speed: 4792.17 + value: {index: 10, type: Single} + - key: [3584, 256, 1792] + speed: 5031.9 + value: {index: 10, type: Single} + - key: [3584, 256, 3328] + speed: 5094.0 + value: {index: 10, type: Single} + - key: [3584, 448, 256] + speed: 3997.78 + value: {index: 8, type: Single} + - key: [3584, 448, 768] + speed: 4485.8 + value: {index: 8, type: Single} + - key: [3584, 448, 1792] + speed: 4694.52 + value: {index: 7, type: Single} + - key: [3584, 448, 3328] + speed: 4777.86 + value: {index: 7, type: Single} + - key: [3584, 704, 256] + speed: 4368.61 + value: {index: 8, type: Single} + - key: [3584, 704, 768] + speed: 4832.67 + value: {index: 8, type: Single} + - key: [3584, 704, 1792] + speed: 4983.93 + value: {index: 8, type: Single} + - key: [3584, 704, 3328] + speed: 5035.61 + value: {index: 8, type: Single} + - key: [3584, 1024, 256] + speed: 5367.45 + value: {index: 10, type: Single} + - key: [3584, 1024, 768] + speed: 5999.65 + value: {index: 10, type: Single} + - key: [3584, 1024, 1792] + speed: 6153.08 + value: {index: 10, type: Single} + - key: [3584, 1024, 3328] + speed: 6068.77 + value: {index: 10, type: Single} + - key: [3584, 1408, 256] + speed: 5553.98 + value: {index: 8, type: Single} + - key: [3584, 1408, 768] + speed: 6006.02 + value: {index: 8, type: Single} + - key: [3584, 1408, 1792] + speed: 6155.84 + value: {index: 8, type: Single} + - key: [3584, 1408, 3328] + speed: 6196.89 + value: {index: 8, type: Single} + - key: [3584, 1856, 256] + speed: 5300.54 + value: {index: 10, type: Single} + - key: [3584, 1856, 768] + speed: 5720.41 + value: {index: 8, type: Single} + - key: [3584, 1856, 1792] + speed: 5841.4 + value: {index: 8, type: Single} + - key: [3584, 1856, 3328] + speed: 5895.75 + value: {index: 8, type: Single} + - key: [3584, 2368, 256] + speed: 5557.03 + value: {index: 10, type: Single} + - key: [3584, 2368, 768] + speed: 5935.65 + value: {index: 10, type: Single} + - key: [3584, 2368, 1792] + speed: 6051.53 + value: {index: 10, type: Single} + - key: [3584, 2368, 3328] + speed: 6098.8 + value: {index: 10, type: Single} + - key: [3584, 2944, 256] + speed: 5801.81 + value: {index: 10, type: Single} + - key: [3584, 2944, 768] + speed: 6049.17 + value: {index: 10, type: Single} + - key: [3584, 2944, 1792] + speed: 6133.18 + value: {index: 10, type: Single} + - key: [3584, 2944, 3328] + speed: 6164.97 + value: {index: 10, type: Single} + - key: [3584, 3584, 256] + speed: 5891.36 + value: {index: 10, type: Single} + - key: [3584, 3584, 768] + speed: 6178.35 + value: {index: 10, type: Single} + - key: [3584, 3584, 1792] + speed: 6276.56 + value: {index: 10, type: Single} + - key: [3584, 3584, 3328] + speed: 6312.22 + value: {index: 10, type: Single} + - key: [3584, 4288, 256] + speed: 5898.62 + value: {index: 8, type: Single} + - key: [3584, 4288, 768] + speed: 6282.87 + value: {index: 8, type: Single} + - key: [3584, 4288, 1792] + speed: 6406.99 + value: {index: 8, type: Single} + - key: [3584, 4288, 3328] + speed: 6450.23 + value: {index: 8, type: Single} + - key: [3584, 5056, 256] + speed: 5871.88 + value: {index: 8, type: Single} + - key: [3584, 5056, 768] + speed: 6220.89 + value: {index: 10, type: Single} + - key: [3584, 5056, 1792] + speed: 6328.27 + value: {index: 10, type: Single} + - key: [3584, 5056, 3328] + speed: 6374.1 + value: {index: 10, type: Single} + - key: [3584, 5888, 256] + speed: 6004.84 + value: {index: 8, type: Single} + - key: [3584, 5888, 768] + speed: 6267.51 + value: {index: 8, type: Single} + - key: [3584, 5888, 1792] + speed: 6334.79 + value: {index: 8, type: Single} + - key: [3584, 5888, 3328] + speed: 6362.14 + value: {index: 8, type: Single} + - key: [3584, 6784, 256] + speed: 6050.74 + value: {index: 8, type: Single} + - key: [3584, 6784, 768] + speed: 6304.36 + value: {index: 8, type: Single} + - key: [3584, 6784, 1792] + speed: 6396.67 + value: {index: 8, type: Single} + - key: [3584, 6784, 3328] + speed: 6421.6 + value: {index: 8, type: Single} + - key: [4288, 4, 256] + speed: 235.217 + value: {index: 12, type: Single} + - key: [4288, 4, 768] + speed: 287.746 + value: {index: 12, type: Single} + - key: [4288, 4, 1792] + speed: 161.076 + value: {index: 12, type: Single} + - key: [4288, 4, 3328] + speed: 161.177 + value: {index: 12, type: Single} + - key: [4288, 64, 256] + speed: 2285.33 + value: {index: 7, type: Single} + - key: [4288, 64, 768] + speed: 2845.24 + value: {index: 7, type: Single} + - key: [4288, 64, 1792] + speed: 3073.56 + value: {index: 7, type: Single} + - key: [4288, 64, 3328] + speed: 3172.77 + value: {index: 7, type: Single} + - key: [4288, 128, 256] + speed: 3084.28 + value: {index: 7, type: Single} + - key: [4288, 128, 768] + speed: 3590.22 + value: {index: 7, type: Single} + - key: [4288, 128, 1792] + speed: 3847.52 + value: {index: 7, type: Single} + - key: [4288, 128, 3328] + speed: 3930.94 + value: {index: 7, type: Single} + - key: [4288, 256, 256] + speed: 3676.01 + value: {index: 10, type: Single} + - key: [4288, 256, 768] + speed: 4027.23 + value: {index: 10, type: Single} + - key: [4288, 256, 1792] + speed: 4191.22 + value: {index: 10, type: Single} + - key: [4288, 256, 3328] + speed: 4189.39 + value: {index: 10, type: Single} + - key: [4288, 448, 256] + speed: 3949.38 + value: {index: 7, type: Single} + - key: [4288, 448, 768] + speed: 4642.59 + value: {index: 7, type: Single} + - key: [4288, 448, 1792] + speed: 4881.03 + value: {index: 7, type: Single} + - key: [4288, 448, 3328] + speed: 4978.69 + value: {index: 7, type: Single} + - key: [4288, 704, 256] + speed: 4528.01 + value: {index: 10, type: Single} + - key: [4288, 704, 768] + speed: 4974.2 + value: {index: 10, type: Single} + - key: [4288, 704, 1792] + speed: 5089.05 + value: {index: 10, type: Single} + - key: [4288, 704, 3328] + speed: 5143.48 + value: {index: 10, type: Single} + - key: [4288, 1024, 256] + speed: 5425.31 + value: {index: 10, type: Single} + - key: [4288, 1024, 768] + speed: 5734.21 + value: {index: 10, type: Single} + - key: [4288, 1024, 1792] + speed: 5788.05 + value: {index: 10, type: Single} + - key: [4288, 1024, 3328] + speed: 5569.45 + value: {index: 10, type: Single} + - key: [4288, 1408, 256] + speed: 5707.08 + value: {index: 10, type: Single} + - key: [4288, 1408, 768] + speed: 6037.94 + value: {index: 8, type: Single} + - key: [4288, 1408, 1792] + speed: 6167.11 + value: {index: 8, type: Single} + - key: [4288, 1408, 3328] + speed: 6217.82 + value: {index: 8, type: Single} + - key: [4288, 1856, 256] + speed: 5618.79 + value: {index: 10, type: Single} + - key: [4288, 1856, 768] + speed: 6004.37 + value: {index: 8, type: Single} + - key: [4288, 1856, 1792] + speed: 6147.13 + value: {index: 8, type: Single} + - key: [4288, 1856, 3328] + speed: 6194.06 + value: {index: 8, type: Single} + - key: [4288, 2368, 256] + speed: 5785.87 + value: {index: 10, type: Single} + - key: [4288, 2368, 768] + speed: 6075.72 + value: {index: 10, type: Single} + - key: [4288, 2368, 1792] + speed: 6183.8 + value: {index: 10, type: Single} + - key: [4288, 2368, 3328] + speed: 6222.03 + value: {index: 10, type: Single} + - key: [4288, 2944, 256] + speed: 5909.12 + value: {index: 10, type: Single} + - key: [4288, 2944, 768] + speed: 6126.82 + value: {index: 10, type: Single} + - key: [4288, 2944, 1792] + speed: 6191.49 + value: {index: 10, type: Single} + - key: [4288, 2944, 3328] + speed: 6221.49 + value: {index: 10, type: Single} + - key: [4288, 3584, 256] + speed: 6002.62 + value: {index: 10, type: Single} + - key: [4288, 3584, 768] + speed: 6325.02 + value: {index: 8, type: Single} + - key: [4288, 3584, 1792] + speed: 6425.92 + value: {index: 8, type: Single} + - key: [4288, 3584, 3328] + speed: 6459.64 + value: {index: 8, type: Single} + - key: [4288, 4288, 256] + speed: 5949.79 + value: {index: 10, type: Single} + - key: [4288, 4288, 768] + speed: 6192.74 + value: {index: 10, type: Single} + - key: [4288, 4288, 1792] + speed: 6268.14 + value: {index: 10, type: Single} + - key: [4288, 4288, 3328] + speed: 6303.51 + value: {index: 10, type: Single} + - key: [4288, 5056, 256] + speed: 5992.06 + value: {index: 10, type: Single} + - key: [4288, 5056, 768] + speed: 6249.8 + value: {index: 10, type: Single} + - key: [4288, 5056, 1792] + speed: 6337.89 + value: {index: 10, type: Single} + - key: [4288, 5056, 3328] + speed: 6374.94 + value: {index: 10, type: Single} + - key: [4288, 5888, 256] + speed: 6095.29 + value: {index: 10, type: Single} + - key: [4288, 5888, 768] + speed: 6306.76 + value: {index: 8, type: Single} + - key: [4288, 5888, 1792] + speed: 6377.02 + value: {index: 8, type: Single} + - key: [4288, 5888, 3328] + speed: 6405.38 + value: {index: 8, type: Single} + - key: [4288, 6784, 256] + speed: 6147.23 + value: {index: 10, type: Single} + - key: [4288, 6784, 768] + speed: 6336.84 + value: {index: 10, type: Single} + - key: [4288, 6784, 1792] + speed: 6407.97 + value: {index: 10, type: Single} + - key: [4288, 6784, 3328] + speed: 6433.22 + value: {index: 10, type: Single} + - key: [5056, 4, 256] + speed: 260.797 + value: {index: 12, type: Single} + - key: [5056, 4, 768] + speed: 306.548 + value: {index: 12, type: Single} + - key: [5056, 4, 1792] + speed: 164.012 + value: {index: 12, type: Single} + - key: [5056, 4, 3328] + speed: 167.639 + value: {index: 12, type: Single} + - key: [5056, 64, 256] + speed: 2631.27 + value: {index: 7, type: Single} + - key: [5056, 64, 768] + speed: 3364.92 + value: {index: 7, type: Single} + - key: [5056, 64, 1792] + speed: 3602.37 + value: {index: 7, type: Single} + - key: [5056, 64, 3328] + speed: 3674.83 + value: {index: 7, type: Single} + - key: [5056, 128, 256] + speed: 3456.78 + value: {index: 7, type: Single} + - key: [5056, 128, 768] + speed: 4206.69 + value: {index: 7, type: Single} + - key: [5056, 128, 1792] + speed: 4442.65 + value: {index: 7, type: Single} + - key: [5056, 128, 3328] + speed: 4565.12 + value: {index: 7, type: Single} + - key: [5056, 256, 256] + speed: 4288.71 + value: {index: 10, type: Single} + - key: [5056, 256, 768] + speed: 4726.79 + value: {index: 10, type: Single} + - key: [5056, 256, 1792] + speed: 4880.27 + value: {index: 10, type: Single} + - key: [5056, 256, 3328] + speed: 4920.92 + value: {index: 10, type: Single} + - key: [5056, 448, 256] + speed: 4430.07 + value: {index: 10, type: Single} + - key: [5056, 448, 768] + speed: 5016.84 + value: {index: 10, type: Single} + - key: [5056, 448, 1792] + speed: 5245.61 + value: {index: 10, type: Single} + - key: [5056, 448, 3328] + speed: 5309.86 + value: {index: 10, type: Single} + - key: [5056, 704, 256] + speed: 4766.01 + value: {index: 8, type: Single} + - key: [5056, 704, 768] + speed: 5229.31 + value: {index: 8, type: Single} + - key: [5056, 704, 1792] + speed: 5355.28 + value: {index: 8, type: Single} + - key: [5056, 704, 3328] + speed: 5414.02 + value: {index: 8, type: Single} + - key: [5056, 1024, 256] + speed: 5766.18 + value: {index: 10, type: Single} + - key: [5056, 1024, 768] + speed: 6099.01 + value: {index: 8, type: Single} + - key: [5056, 1024, 1792] + speed: 6279.32 + value: {index: 8, type: Single} + - key: [5056, 1024, 3328] + speed: 6170.31 + value: {index: 8, type: Single} + - key: [5056, 1408, 256] + speed: 5816.11 + value: {index: 10, type: Single} + - key: [5056, 1408, 768] + speed: 6135.2 + value: {index: 8, type: Single} + - key: [5056, 1408, 1792] + speed: 6262.13 + value: {index: 8, type: Single} + - key: [5056, 1408, 3328] + speed: 6315.46 + value: {index: 8, type: Single} + - key: [5056, 1856, 256] + speed: 5630.23 + value: {index: 10, type: Single} + - key: [5056, 1856, 768] + speed: 5921.86 + value: {index: 10, type: Single} + - key: [5056, 1856, 1792] + speed: 6009.55 + value: {index: 10, type: Single} + - key: [5056, 1856, 3328] + speed: 6049.43 + value: {index: 10, type: Single} + - key: [5056, 2368, 256] + speed: 5758.71 + value: {index: 8, type: Single} + - key: [5056, 2368, 768] + speed: 6116.23 + value: {index: 8, type: Single} + - key: [5056, 2368, 1792] + speed: 6227.89 + value: {index: 8, type: Single} + - key: [5056, 2368, 3328] + speed: 6260.17 + value: {index: 8, type: Single} + - key: [5056, 2944, 256] + speed: 6006.64 + value: {index: 10, type: Single} + - key: [5056, 2944, 768] + speed: 6225.44 + value: {index: 10, type: Single} + - key: [5056, 2944, 1792] + speed: 6299.17 + value: {index: 10, type: Single} + - key: [5056, 2944, 3328] + speed: 6328.72 + value: {index: 10, type: Single} + - key: [5056, 3584, 256] + speed: 6081.14 + value: {index: 10, type: Single} + - key: [5056, 3584, 768] + speed: 6290.25 + value: {index: 10, type: Single} + - key: [5056, 3584, 1792] + speed: 6365.39 + value: {index: 10, type: Single} + - key: [5056, 3584, 3328] + speed: 6391.72 + value: {index: 10, type: Single} + - key: [5056, 4288, 256] + speed: 6029.18 + value: {index: 10, type: Single} + - key: [5056, 4288, 768] + speed: 6265.48 + value: {index: 10, type: Single} + - key: [5056, 4288, 1792] + speed: 6342.26 + value: {index: 10, type: Single} + - key: [5056, 4288, 3328] + speed: 6375.07 + value: {index: 10, type: Single} + - key: [5056, 5056, 256] + speed: 6053.86 + value: {index: 8, type: Single} + - key: [5056, 5056, 768] + speed: 6365.0 + value: {index: 8, type: Single} + - key: [5056, 5056, 1792] + speed: 6446.18 + value: {index: 8, type: Single} + - key: [5056, 5056, 3328] + speed: 6474.64 + value: {index: 8, type: Single} + - key: [5056, 5888, 256] + speed: 6173.89 + value: {index: 10, type: Single} + - key: [5056, 5888, 768] + speed: 6413.4 + value: {index: 8, type: Single} + - key: [5056, 5888, 1792] + speed: 6482.78 + value: {index: 8, type: Single} + - key: [5056, 5888, 3328] + speed: 6512.1 + value: {index: 8, type: Single} + - key: [5056, 6784, 256] + speed: 6162.6 + value: {index: 10, type: Single} + - key: [5056, 6784, 768] + speed: 6347.05 + value: {index: 10, type: Single} + - key: [5056, 6784, 1792] + speed: 6417.15 + value: {index: 10, type: Single} + - key: [5056, 6784, 3328] + speed: 6440.91 + value: {index: 10, type: Single} + - key: [5888, 4, 256] + speed: 290.695 + value: {index: 12, type: Single} + - key: [5888, 4, 768] + speed: 252.514 + value: {index: 12, type: Single} + - key: [5888, 4, 1792] + speed: 168.817 + value: {index: 12, type: Single} + - key: [5888, 4, 3328] + speed: 168.196 + value: {index: 12, type: Single} + - key: [5888, 64, 256] + speed: 2980.11 + value: {index: 7, type: Single} + - key: [5888, 64, 768] + speed: 3845.38 + value: {index: 7, type: Single} + - key: [5888, 64, 1792] + speed: 4139.92 + value: {index: 7, type: Single} + - key: [5888, 64, 3328] + speed: 4265.53 + value: {index: 7, type: Single} + - key: [5888, 128, 256] + speed: 3622.53 + value: {index: 10, type: Single} + - key: [5888, 128, 768] + speed: 4274.5 + value: {index: 7, type: Single} + - key: [5888, 128, 1792] + speed: 4559.19 + value: {index: 7, type: Single} + - key: [5888, 128, 3328] + speed: 4643.41 + value: {index: 7, type: Single} + - key: [5888, 256, 256] + speed: 4787.87 + value: {index: 10, type: Single} + - key: [5888, 256, 768] + speed: 5396.26 + value: {index: 10, type: Single} + - key: [5888, 256, 1792] + speed: 5639.4 + value: {index: 10, type: Single} + - key: [5888, 256, 3328] + speed: 5706.07 + value: {index: 10, type: Single} + - key: [5888, 448, 256] + speed: 4492.9 + value: {index: 8, type: Single} + - key: [5888, 448, 768] + speed: 5002.44 + value: {index: 8, type: Single} + - key: [5888, 448, 1792] + speed: 5190.13 + value: {index: 8, type: Single} + - key: [5888, 448, 3328] + speed: 5258.9 + value: {index: 8, type: Single} + - key: [5888, 704, 256] + speed: 4884.18 + value: {index: 10, type: Single} + - key: [5888, 704, 768] + speed: 5338.62 + value: {index: 10, type: Single} + - key: [5888, 704, 1792] + speed: 5482.32 + value: {index: 10, type: Single} + - key: [5888, 704, 3328] + speed: 5531.66 + value: {index: 10, type: Single} + - key: [5888, 1024, 256] + speed: 5571.33 + value: {index: 10, type: Single} + - key: [5888, 1024, 768] + speed: 5988.73 + value: {index: 8, type: Single} + - key: [5888, 1024, 1792] + speed: 6098.23 + value: {index: 8, type: Single} + - key: [5888, 1024, 3328] + speed: 5953.11 + value: {index: 8, type: Single} + - key: [5888, 1408, 256] + speed: 5879.45 + value: {index: 10, type: Single} + - key: [5888, 1408, 768] + speed: 6296.83 + value: {index: 8, type: Single} + - key: [5888, 1408, 1792] + speed: 6405.23 + value: {index: 8, type: Single} + - key: [5888, 1408, 3328] + speed: 6452.83 + value: {index: 8, type: Single} + - key: [5888, 1856, 256] + speed: 5663.02 + value: {index: 8, type: Single} + - key: [5888, 1856, 768] + speed: 6063.69 + value: {index: 8, type: Single} + - key: [5888, 1856, 1792] + speed: 6184.31 + value: {index: 8, type: Single} + - key: [5888, 1856, 3328] + speed: 6224.78 + value: {index: 8, type: Single} + - key: [5888, 2368, 256] + speed: 5781.05 + value: {index: 8, type: Single} + - key: [5888, 2368, 768] + speed: 6105.04 + value: {index: 8, type: Single} + - key: [5888, 2368, 1792] + speed: 6231.33 + value: {index: 8, type: Single} + - key: [5888, 2368, 3328] + speed: 6264.06 + value: {index: 8, type: Single} + - key: [5888, 2944, 256] + speed: 6026.74 + value: {index: 8, type: Single} + - key: [5888, 2944, 768] + speed: 6317.84 + value: {index: 8, type: Single} + - key: [5888, 2944, 1792] + speed: 6411.54 + value: {index: 8, type: Single} + - key: [5888, 2944, 3328] + speed: 6438.13 + value: {index: 8, type: Single} + - key: [5888, 3584, 256] + speed: 6015.74 + value: {index: 10, type: Single} + - key: [5888, 3584, 768] + speed: 6250.86 + value: {index: 8, type: Single} + - key: [5888, 3584, 1792] + speed: 6332.66 + value: {index: 8, type: Single} + - key: [5888, 3584, 3328] + speed: 6361.89 + value: {index: 10, type: Single} + - key: [5888, 4288, 256] + speed: 6000.97 + value: {index: 8, type: Single} + - key: [5888, 4288, 768] + speed: 6295.08 + value: {index: 8, type: Single} + - key: [5888, 4288, 1792] + speed: 6372.56 + value: {index: 8, type: Single} + - key: [5888, 4288, 3328] + speed: 6399.78 + value: {index: 8, type: Single} + - key: [5888, 5056, 256] + speed: 6119.17 + value: {index: 8, type: Single} + - key: [5888, 5056, 768] + speed: 6396.79 + value: {index: 8, type: Single} + - key: [5888, 5056, 1792] + speed: 6479.74 + value: {index: 8, type: Single} + - key: [5888, 5056, 3328] + speed: 6510.39 + value: {index: 8, type: Single} + - key: [5888, 5888, 256] + speed: 6193.09 + value: {index: 8, type: Single} + - key: [5888, 5888, 768] + speed: 6393.13 + value: {index: 8, type: Single} + - key: [5888, 5888, 1792] + speed: 6468.86 + value: {index: 8, type: Single} + - key: [5888, 5888, 3328] + speed: 6487.04 + value: {index: 8, type: Single} + - key: [5888, 6784, 256] + speed: 6230.82 + value: {index: 8, type: Single} + - key: [5888, 6784, 768] + speed: 6417.16 + value: {index: 8, type: Single} + - key: [5888, 6784, 1792] + speed: 6502.91 + value: {index: 8, type: Single} + - key: [5888, 6784, 3328] + speed: 6521.7 + value: {index: 8, type: Single} + - key: [6784, 4, 256] + speed: 313.647 + value: {index: 12, type: Single} + - key: [6784, 4, 768] + speed: 313.296 + value: {index: 12, type: Single} + - key: [6784, 4, 1792] + speed: 165.814 + value: {index: 12, type: Single} + - key: [6784, 4, 3328] + speed: 168.413 + value: {index: 12, type: Single} + - key: [6784, 64, 256] + speed: 3025.12 + value: {index: 7, type: Single} + - key: [6784, 64, 768] + speed: 3720.18 + value: {index: 7, type: Single} + - key: [6784, 64, 1792] + speed: 3947.13 + value: {index: 7, type: Single} + - key: [6784, 64, 3328] + speed: 4065.48 + value: {index: 7, type: Single} + - key: [6784, 128, 256] + speed: 4077.37 + value: {index: 10, type: Single} + - key: [6784, 128, 768] + speed: 4602.68 + value: {index: 10, type: Single} + - key: [6784, 128, 1792] + speed: 4768.82 + value: {index: 10, type: Single} + - key: [6784, 128, 3328] + speed: 4830.65 + value: {index: 10, type: Single} + - key: [6784, 256, 256] + speed: 4482.38 + value: {index: 8, type: Single} + - key: [6784, 256, 768] + speed: 4904.85 + value: {index: 10, type: Single} + - key: [6784, 256, 1792] + speed: 5048.46 + value: {index: 8, type: Single} + - key: [6784, 256, 3328] + speed: 5096.66 + value: {index: 8, type: Single} + - key: [6784, 448, 256] + speed: 4358.22 + value: {index: 10, type: Single} + - key: [6784, 448, 768] + speed: 4929.6 + value: {index: 10, type: Single} + - key: [6784, 448, 1792] + speed: 5095.52 + value: {index: 10, type: Single} + - key: [6784, 448, 3328] + speed: 5184.27 + value: {index: 10, type: Single} + - key: [6784, 704, 256] + speed: 5137.02 + value: {index: 8, type: Single} + - key: [6784, 704, 768] + speed: 5617.84 + value: {index: 8, type: Single} + - key: [6784, 704, 1792] + speed: 5798.87 + value: {index: 8, type: Single} + - key: [6784, 704, 3328] + speed: 5853.24 + value: {index: 8, type: Single} + - key: [6784, 1024, 256] + speed: 5621.07 + value: {index: 10, type: Single} + - key: [6784, 1024, 768] + speed: 5942.97 + value: {index: 8, type: Single} + - key: [6784, 1024, 1792] + speed: 5977.99 + value: {index: 8, type: Single} + - key: [6784, 1024, 3328] + speed: 5768.18 + value: {index: 8, type: Single} + - key: [6784, 1408, 256] + speed: 5811.7 + value: {index: 10, type: Single} + - key: [6784, 1408, 768] + speed: 6048.46 + value: {index: 10, type: Single} + - key: [6784, 1408, 1792] + speed: 6137.73 + value: {index: 10, type: Single} + - key: [6784, 1408, 3328] + speed: 6166.99 + value: {index: 10, type: Single} + - key: [6784, 1856, 256] + speed: 5736.82 + value: {index: 10, type: Single} + - key: [6784, 1856, 768] + speed: 6060.7 + value: {index: 10, type: Single} + - key: [6784, 1856, 1792] + speed: 6144.65 + value: {index: 10, type: Single} + - key: [6784, 1856, 3328] + speed: 6188.71 + value: {index: 10, type: Single} + - key: [6784, 2368, 256] + speed: 5849.5 + value: {index: 8, type: Single} + - key: [6784, 2368, 768] + speed: 6169.54 + value: {index: 8, type: Single} + - key: [6784, 2368, 1792] + speed: 6291.35 + value: {index: 8, type: Single} + - key: [6784, 2368, 3328] + speed: 6328.51 + value: {index: 8, type: Single} + - key: [6784, 2944, 256] + speed: 6033.38 + value: {index: 10, type: Single} + - key: [6784, 2944, 768] + speed: 6235.98 + value: {index: 10, type: Single} + - key: [6784, 2944, 1792] + speed: 6309.21 + value: {index: 8, type: Single} + - key: [6784, 2944, 3328] + speed: 6329.02 + value: {index: 10, type: Single} + - key: [6784, 3584, 256] + speed: 6096.84 + value: {index: 10, type: Single} + - key: [6784, 3584, 768] + speed: 6325.56 + value: {index: 8, type: Single} + - key: [6784, 3584, 1792] + speed: 6395.35 + value: {index: 8, type: Single} + - key: [6784, 3584, 3328] + speed: 6414.59 + value: {index: 8, type: Single} + - key: [6784, 4288, 256] + speed: 6025.28 + value: {index: 8, type: Single} + - key: [6784, 4288, 768] + speed: 6270.05 + value: {index: 8, type: Single} + - key: [6784, 4288, 1792] + speed: 6348.48 + value: {index: 8, type: Single} + - key: [6784, 4288, 3328] + speed: 6371.21 + value: {index: 8, type: Single} + - key: [6784, 5056, 256] + speed: 6109.55 + value: {index: 8, type: Single} + - key: [6784, 5056, 768] + speed: 6320.66 + value: {index: 8, type: Single} + - key: [6784, 5056, 1792] + speed: 6392.33 + value: {index: 8, type: Single} + - key: [6784, 5056, 3328] + speed: 6417.63 + value: {index: 8, type: Single} + - key: [6784, 5888, 256] + speed: 6241.52 + value: {index: 8, type: Single} + - key: [6784, 5888, 768] + speed: 6440.21 + value: {index: 8, type: Single} + - key: [6784, 5888, 1792] + speed: 6504.35 + value: {index: 8, type: Single} + - key: [6784, 5888, 3328] + speed: 6523.26 + value: {index: 8, type: Single} + - key: [6784, 6784, 256] + speed: 6312.02 + value: {index: 8, type: Single} + - key: [6784, 6784, 768] + speed: 6552.11 + value: {index: 8, type: Single} + - key: [6784, 6784, 1792] + speed: 6628.56 + value: {index: 8, type: Single} + - key: [6784, 6784, 3328] + speed: 6657.53 + value: {index: 8, type: Single} + type: Matching + predicate: + type: And + value: + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + type: Problem + Contraction_l_Alik_Bljk_Cijk_Dijk: + rows: + - library: + distance: Euclidean + properties: + - {index: 0, type: FreeSizeA} + - {index: 0, type: FreeSizeB} + - {index: 0, type: BoundSize} + table: + - key: [4, 4, 256] + speed: 0.280683 + value: {index: 20, type: Single} + - key: [4, 4, 768] + speed: 0.403613 + value: {index: 20, type: Single} + - key: [4, 4, 1792] + speed: 0.420719 + value: {index: 20, type: Single} + - key: [4, 4, 3328] + speed: 0.44372 + value: {index: 20, type: Single} + - key: [4, 64, 256] + speed: 5.23492 + value: {index: 20, type: Single} + - key: [4, 64, 768] + speed: 6.60235 + value: {index: 20, type: Single} + - key: [4, 64, 1792] + speed: 6.92728 + value: {index: 20, type: Single} + - key: [4, 64, 3328] + speed: 7.13924 + value: {index: 20, type: Single} + - key: [4, 128, 256] + speed: 11.718 + value: {index: 20, type: Single} + - key: [4, 128, 768] + speed: 15.7051 + value: {index: 20, type: Single} + - key: [4, 128, 1792] + speed: 14.2532 + value: {index: 20, type: Single} + - key: [4, 128, 3328] + speed: 14.3497 + value: {index: 20, type: Single} + - key: [4, 256, 256] + speed: 22.3978 + value: {index: 20, type: Single} + - key: [4, 256, 768] + speed: 30.7723 + value: {index: 20, type: Single} + - key: [4, 256, 1792] + speed: 27.2519 + value: {index: 20, type: Single} + - key: [4, 256, 3328] + speed: 29.007 + value: {index: 20, type: Single} + - key: [4, 448, 256] + speed: 33.1169 + value: {index: 20, type: Single} + - key: [4, 448, 768] + speed: 47.5169 + value: {index: 20, type: Single} + - key: [4, 448, 1792] + speed: 51.4247 + value: {index: 20, type: Single} + - key: [4, 448, 3328] + speed: 50.1301 + value: {index: 20, type: Single} + - key: [4, 704, 256] + speed: 48.9041 + value: {index: 20, type: Single} + - key: [4, 704, 768] + speed: 66.6571 + value: {index: 20, type: Single} + - key: [4, 704, 1792] + speed: 75.4408 + value: {index: 20, type: Single} + - key: [4, 704, 3328] + speed: 76.9552 + value: {index: 20, type: Single} + - key: [4, 1024, 256] + speed: 70.0756 + value: {index: 20, type: Single} + - key: [4, 1024, 768] + speed: 96.514 + value: {index: 20, type: Single} + - key: [4, 1024, 1792] + speed: 106.317 + value: {index: 20, type: Single} + - key: [4, 1024, 3328] + speed: 109.082 + value: {index: 20, type: Single} + - key: [4, 1408, 256] + speed: 95.4102 + value: {index: 20, type: Single} + - key: [4, 1408, 768] + speed: 128.051 + value: {index: 20, type: Single} + - key: [4, 1408, 1792] + speed: 140.316 + value: {index: 20, type: Single} + - key: [4, 1408, 3328] + speed: 146.259 + value: {index: 20, type: Single} + - key: [4, 1856, 256] + speed: 113.023 + value: {index: 20, type: Single} + - key: [4, 1856, 768] + speed: 159.359 + value: {index: 20, type: Single} + - key: [4, 1856, 1792] + speed: 172.027 + value: {index: 20, type: Single} + - key: [4, 1856, 3328] + speed: 185.092 + value: {index: 20, type: Single} + - key: [4, 2368, 256] + speed: 138.12 + value: {index: 20, type: Single} + - key: [4, 2368, 768] + speed: 184.939 + value: {index: 20, type: Single} + - key: [4, 2368, 1792] + speed: 199.599 + value: {index: 20, type: Single} + - key: [4, 2368, 3328] + speed: 201.015 + value: {index: 20, type: Single} + - key: [4, 2944, 256] + speed: 154.155 + value: {index: 20, type: Single} + - key: [4, 2944, 768] + speed: 204.851 + value: {index: 20, type: Single} + - key: [4, 2944, 1792] + speed: 222.733 + value: {index: 20, type: Single} + - key: [4, 2944, 3328] + speed: 219.163 + value: {index: 20, type: Single} + - key: [4, 3584, 256] + speed: 172.027 + value: {index: 20, type: Single} + - key: [4, 3584, 768] + speed: 239.729 + value: {index: 20, type: Single} + - key: [4, 3584, 1792] + speed: 183.302 + value: {index: 20, type: Single} + - key: [4, 3584, 3328] + speed: 177.969 + value: {index: 20, type: Single} + - key: [4, 4288, 256] + speed: 174.853 + value: {index: 20, type: Single} + - key: [4, 4288, 768] + speed: 241.285 + value: {index: 20, type: Single} + - key: [4, 4288, 1792] + speed: 159.773 + value: {index: 20, type: Single} + - key: [4, 4288, 3328] + speed: 168.25 + value: {index: 20, type: Single} + - key: [4, 5056, 256] + speed: 185.884 + value: {index: 20, type: Single} + - key: [4, 5056, 768] + speed: 265.414 + value: {index: 20, type: Single} + - key: [4, 5056, 1792] + speed: 163.902 + value: {index: 20, type: Single} + - key: [4, 5056, 3328] + speed: 171.564 + value: {index: 20, type: Single} + - key: [4, 5888, 256] + speed: 190.617 + value: {index: 20, type: Single} + - key: [4, 5888, 768] + speed: 240.335 + value: {index: 20, type: Single} + - key: [4, 5888, 1792] + speed: 166.206 + value: {index: 20, type: Single} + - key: [4, 5888, 3328] + speed: 169.597 + value: {index: 20, type: Single} + - key: [4, 6784, 256] + speed: 203.427 + value: {index: 20, type: Single} + - key: [4, 6784, 768] + speed: 244.43 + value: {index: 20, type: Single} + - key: [4, 6784, 1792] + speed: 168.886 + value: {index: 20, type: Single} + - key: [4, 6784, 3328] + speed: 171.179 + value: {index: 20, type: Single} + - key: [64, 4, 256] + speed: 4.86118 + value: {index: 20, type: Single} + - key: [64, 4, 768] + speed: 6.36488 + value: {index: 20, type: Single} + - key: [64, 4, 1792] + speed: 6.8582 + value: {index: 20, type: Single} + - key: [64, 4, 3328] + speed: 6.9959 + value: {index: 20, type: Single} + - key: [64, 64, 256] + speed: 53.8228 + value: {index: 16, type: Single} + - key: [64, 64, 768] + speed: 87.9211 + value: {index: 17, type: Single} + - key: [64, 64, 1792] + speed: 104.744 + value: {index: 18, type: Single} + - key: [64, 64, 3328] + speed: 106.802 + value: {index: 17, type: Single} + - key: [64, 128, 256] + speed: 142.267 + value: {index: 17, type: Single} + - key: [64, 128, 768] + speed: 190.005 + value: {index: 17, type: Single} + - key: [64, 128, 1792] + speed: 206.863 + value: {index: 18, type: Single} + - key: [64, 128, 3328] + speed: 192.692 + value: {index: 18, type: Single} + - key: [64, 256, 256] + speed: 288.884 + value: {index: 17, type: Single} + - key: [64, 256, 768] + speed: 377.48 + value: {index: 17, type: Single} + - key: [64, 256, 1792] + speed: 397.944 + value: {index: 18, type: Single} + - key: [64, 256, 3328] + speed: 330.823 + value: {index: 17, type: Single} + - key: [64, 448, 256] + speed: 469.612 + value: {index: 18, type: Single} + - key: [64, 448, 768] + speed: 650.472 + value: {index: 18, type: Single} + - key: [64, 448, 1792] + speed: 575.615 + value: {index: 17, type: Single} + - key: [64, 448, 3328] + speed: 581.553 + value: {index: 17, type: Single} + - key: [64, 704, 256] + speed: 759.537 + value: {index: 18, type: Single} + - key: [64, 704, 768] + speed: 961.168 + value: {index: 17, type: Single} + - key: [64, 704, 1792] + speed: 879.007 + value: {index: 17, type: Single} + - key: [64, 704, 3328] + speed: 896.071 + value: {index: 17, type: Single} + - key: [64, 1024, 256] + speed: 1058.33 + value: {index: 17, type: Single} + - key: [64, 1024, 768] + speed: 1183.72 + value: {index: 18, type: Single} + - key: [64, 1024, 1792] + speed: 1225.2 + value: {index: 17, type: Single} + - key: [64, 1024, 3328] + speed: 1285.73 + value: {index: 17, type: Single} + - key: [64, 1408, 256] + speed: 1348.14 + value: {index: 17, type: Single} + - key: [64, 1408, 768] + speed: 1428.53 + value: {index: 17, type: Single} + - key: [64, 1408, 1792] + speed: 1640.28 + value: {index: 17, type: Single} + - key: [64, 1408, 3328] + speed: 1727.89 + value: {index: 17, type: Single} + - key: [64, 1856, 256] + speed: 1584.98 + value: {index: 18, type: Single} + - key: [64, 1856, 768] + speed: 1819.08 + value: {index: 18, type: Single} + - key: [64, 1856, 1792] + speed: 2114.46 + value: {index: 17, type: Single} + - key: [64, 1856, 3328] + speed: 2162.31 + value: {index: 17, type: Single} + - key: [64, 2368, 256] + speed: 1706.01 + value: {index: 17, type: Single} + - key: [64, 2368, 768] + speed: 2035.3 + value: {index: 17, type: Single} + - key: [64, 2368, 1792] + speed: 2365.32 + value: {index: 18, type: Single} + - key: [64, 2368, 3328] + speed: 2411.03 + value: {index: 18, type: Single} + - key: [64, 2944, 256] + speed: 2009.69 + value: {index: 17, type: Single} + - key: [64, 2944, 768] + speed: 2438.75 + value: {index: 18, type: Single} + - key: [64, 2944, 1792] + speed: 2745.8 + value: {index: 17, type: Single} + - key: [64, 2944, 3328] + speed: 2975.38 + value: {index: 18, type: Single} + - key: [64, 3584, 256] + speed: 2258.38 + value: {index: 17, type: Single} + - key: [64, 3584, 768] + speed: 2758.83 + value: {index: 17, type: Single} + - key: [64, 3584, 1792] + speed: 3310.81 + value: {index: 17, type: Single} + - key: [64, 3584, 3328] + speed: 3470.91 + value: {index: 17, type: Single} + - key: [64, 4288, 256] + speed: 1827.38 + value: {index: 17, type: Single} + - key: [64, 4288, 768] + speed: 2226.31 + value: {index: 17, type: Single} + - key: [64, 4288, 1792] + speed: 2436.29 + value: {index: 18, type: Single} + - key: [64, 4288, 3328] + speed: 2485.76 + value: {index: 17, type: Single} + - key: [64, 5056, 256] + speed: 1876.32 + value: {index: 14, type: Single} + - key: [64, 5056, 768] + speed: 2558.99 + value: {index: 14, type: Single} + - key: [64, 5056, 1792] + speed: 2795.69 + value: {index: 14, type: Single} + - key: [64, 5056, 3328] + speed: 2990.66 + value: {index: 17, type: Single} + - key: [64, 5888, 256] + speed: 2063.86 + value: {index: 14, type: Single} + - key: [64, 5888, 768] + speed: 2924.32 + value: {index: 14, type: Single} + - key: [64, 5888, 1792] + speed: 3170.81 + value: {index: 17, type: Single} + - key: [64, 5888, 3328] + speed: 3356.44 + value: {index: 17, type: Single} + - key: [64, 6784, 256] + speed: 2143.54 + value: {index: 17, type: Single} + - key: [64, 6784, 768] + speed: 2902.27 + value: {index: 18, type: Single} + - key: [64, 6784, 1792] + speed: 3264.94 + value: {index: 17, type: Single} + - key: [64, 6784, 3328] + speed: 3620.3 + value: {index: 14, type: Single} + - key: [128, 4, 256] + speed: 11.7186 + value: {index: 20, type: Single} + - key: [128, 4, 768] + speed: 15.5213 + value: {index: 20, type: Single} + - key: [128, 4, 1792] + speed: 14.1554 + value: {index: 20, type: Single} + - key: [128, 4, 3328] + speed: 13.9494 + value: {index: 20, type: Single} + - key: [128, 64, 256] + speed: 146.69 + value: {index: 17, type: Single} + - key: [128, 64, 768] + speed: 193.467 + value: {index: 18, type: Single} + - key: [128, 64, 1792] + speed: 209.488 + value: {index: 18, type: Single} + - key: [128, 64, 3328] + speed: 192.995 + value: {index: 17, type: Single} + - key: [128, 128, 256] + speed: 287.419 + value: {index: 18, type: Single} + - key: [128, 128, 768] + speed: 382.576 + value: {index: 18, type: Single} + - key: [128, 128, 1792] + speed: 413.296 + value: {index: 17, type: Single} + - key: [128, 128, 3328] + speed: 341.094 + value: {index: 17, type: Single} + - key: [128, 256, 256] + speed: 569.066 + value: {index: 17, type: Single} + - key: [128, 256, 768] + speed: 732.182 + value: {index: 18, type: Single} + - key: [128, 256, 1792] + speed: 717.378 + value: {index: 18, type: Single} + - key: [128, 256, 3328] + speed: 738.296 + value: {index: 17, type: Single} + - key: [128, 448, 256] + speed: 939.224 + value: {index: 18, type: Single} + - key: [128, 448, 768] + speed: 1298.11 + value: {index: 17, type: Single} + - key: [128, 448, 1792] + speed: 1141.76 + value: {index: 18, type: Single} + - key: [128, 448, 3328] + speed: 1187.78 + value: {index: 17, type: Single} + - key: [128, 704, 256] + speed: 1384.09 + value: {index: 17, type: Single} + - key: [128, 704, 768] + speed: 1793.18 + value: {index: 17, type: Single} + - key: [128, 704, 1792] + speed: 1644.0 + value: {index: 17, type: Single} + - key: [128, 704, 3328] + speed: 1684.04 + value: {index: 18, type: Single} + - key: [128, 1024, 256] + speed: 2058.93 + value: {index: 17, type: Single} + - key: [128, 1024, 768] + speed: 2283.88 + value: {index: 17, type: Single} + - key: [128, 1024, 1792] + speed: 2208.09 + value: {index: 17, type: Single} + - key: [128, 1024, 3328] + speed: 2595.27 + value: {index: 18, type: Single} + - key: [128, 1408, 256] + speed: 2133.03 + value: {index: 18, type: Single} + - key: [128, 1408, 768] + speed: 2624.32 + value: {index: 18, type: Single} + - key: [128, 1408, 1792] + speed: 2785.86 + value: {index: 18, type: Single} + - key: [128, 1408, 3328] + speed: 2890.71 + value: {index: 18, type: Single} + - key: [128, 1856, 256] + speed: 2648.44 + value: {index: 17, type: Single} + - key: [128, 1856, 768] + speed: 3314.99 + value: {index: 17, type: Single} + - key: [128, 1856, 1792] + speed: 3619.08 + value: {index: 17, type: Single} + - key: [128, 1856, 3328] + speed: 3795.59 + value: {index: 17, type: Single} + - key: [128, 2368, 256] + speed: 2476.33 + value: {index: 17, type: Single} + - key: [128, 2368, 768] + speed: 2734.98 + value: {index: 17, type: Single} + - key: [128, 2368, 1792] + speed: 2901.66 + value: {index: 17, type: Single} + - key: [128, 2368, 3328] + speed: 2893.65 + value: {index: 18, type: Single} + - key: [128, 2944, 256] + speed: 2913.41 + value: {index: 17, type: Single} + - key: [128, 2944, 768] + speed: 3291.39 + value: {index: 17, type: Single} + - key: [128, 2944, 1792] + speed: 3526.53 + value: {index: 17, type: Single} + - key: [128, 2944, 3328] + speed: 3682.81 + value: {index: 17, type: Single} + - key: [128, 3584, 256] + speed: 2903.67 + value: {index: 17, type: Single} + - key: [128, 3584, 768] + speed: 3739.17 + value: {index: 14, type: Single} + - key: [128, 3584, 1792] + speed: 4098.17 + value: {index: 14, type: Single} + - key: [128, 3584, 3328] + speed: 4204.46 + value: {index: 14, type: Single} + - key: [128, 4288, 256] + speed: 2891.49 + value: {index: 17, type: Single} + - key: [128, 4288, 768] + speed: 3287.39 + value: {index: 17, type: Single} + - key: [128, 4288, 1792] + speed: 3486.81 + value: {index: 17, type: Single} + - key: [128, 4288, 3328] + speed: 3565.46 + value: {index: 17, type: Single} + - key: [128, 5056, 256] + speed: 3227.36 + value: {index: 17, type: Single} + - key: [128, 5056, 768] + speed: 3733.82 + value: {index: 17, type: Single} + - key: [128, 5056, 1792] + speed: 4021.54 + value: {index: 17, type: Single} + - key: [128, 5056, 3328] + speed: 4046.09 + value: {index: 18, type: Single} + - key: [128, 5888, 256] + speed: 3301.13 + value: {index: 17, type: Single} + - key: [128, 5888, 768] + speed: 3837.82 + value: {index: 17, type: Single} + - key: [128, 5888, 1792] + speed: 4076.05 + value: {index: 17, type: Single} + - key: [128, 5888, 3328] + speed: 4185.9 + value: {index: 17, type: Single} + - key: [128, 6784, 256] + speed: 3572.55 + value: {index: 14, type: Single} + - key: [128, 6784, 768] + speed: 4297.3 + value: {index: 14, type: Single} + - key: [128, 6784, 1792] + speed: 4565.67 + value: {index: 14, type: Single} + - key: [128, 6784, 3328] + speed: 4647.64 + value: {index: 14, type: Single} + - key: [256, 4, 256] + speed: 21.9809 + value: {index: 20, type: Single} + - key: [256, 4, 768] + speed: 30.075 + value: {index: 20, type: Single} + - key: [256, 4, 1792] + speed: 27.1622 + value: {index: 20, type: Single} + - key: [256, 4, 3328] + speed: 28.3108 + value: {index: 20, type: Single} + - key: [256, 64, 256] + speed: 285.969 + value: {index: 17, type: Single} + - key: [256, 64, 768] + speed: 375.806 + value: {index: 18, type: Single} + - key: [256, 64, 1792] + speed: 401.572 + value: {index: 17, type: Single} + - key: [256, 64, 3328] + speed: 335.498 + value: {index: 18, type: Single} + - key: [256, 128, 256] + speed: 560.623 + value: {index: 17, type: Single} + - key: [256, 128, 768] + speed: 749.954 + value: {index: 18, type: Single} + - key: [256, 128, 1792] + speed: 748.539 + value: {index: 18, type: Single} + - key: [256, 128, 3328] + speed: 692.457 + value: {index: 18, type: Single} + - key: [256, 256, 256] + speed: 1083.66 + value: {index: 18, type: Single} + - key: [256, 256, 768] + speed: 1464.36 + value: {index: 17, type: Single} + - key: [256, 256, 1792] + speed: 1382.22 + value: {index: 18, type: Single} + - key: [256, 256, 3328] + speed: 1448.27 + value: {index: 17, type: Single} + - key: [256, 448, 256] + speed: 1761.57 + value: {index: 17, type: Single} + - key: [256, 448, 768] + speed: 2461.82 + value: {index: 18, type: Single} + - key: [256, 448, 1792] + speed: 2292.94 + value: {index: 18, type: Single} + - key: [256, 448, 3328] + speed: 2424.74 + value: {index: 17, type: Single} + - key: [256, 704, 256] + speed: 2155.15 + value: {index: 18, type: Single} + - key: [256, 704, 768] + speed: 2616.98 + value: {index: 18, type: Single} + - key: [256, 704, 1792] + speed: 2840.31 + value: {index: 17, type: Single} + - key: [256, 704, 3328] + speed: 2942.19 + value: {index: 18, type: Single} + - key: [256, 1024, 256] + speed: 2970.27 + value: {index: 18, type: Single} + - key: [256, 1024, 768] + speed: 3759.14 + value: {index: 18, type: Single} + - key: [256, 1024, 1792] + speed: 4083.47 + value: {index: 18, type: Single} + - key: [256, 1024, 3328] + speed: 4170.43 + value: {index: 17, type: Single} + - key: [256, 1408, 256] + speed: 2917.32 + value: {index: 18, type: Single} + - key: [256, 1408, 768] + speed: 3406.6 + value: {index: 17, type: Single} + - key: [256, 1408, 1792] + speed: 3483.71 + value: {index: 17, type: Single} + - key: [256, 1408, 3328] + speed: 3530.37 + value: {index: 17, type: Single} + - key: [256, 1856, 256] + speed: 3264.49 + value: {index: 19, type: Single} + - key: [256, 1856, 768] + speed: 4014.73 + value: {index: 14, type: Single} + - key: [256, 1856, 1792] + speed: 4268.18 + value: {index: 14, type: Single} + - key: [256, 1856, 3328] + speed: 4365.31 + value: {index: 14, type: Single} + - key: [256, 2368, 256] + speed: 3218.12 + value: {index: 18, type: Single} + - key: [256, 2368, 768] + speed: 3688.38 + value: {index: 17, type: Single} + - key: [256, 2368, 1792] + speed: 3840.01 + value: {index: 17, type: Single} + - key: [256, 2368, 3328] + speed: 3887.94 + value: {index: 18, type: Single} + - key: [256, 2944, 256] + speed: 3482.07 + value: {index: 14, type: Single} + - key: [256, 2944, 768] + speed: 3970.42 + value: {index: 18, type: Single} + - key: [256, 2944, 1792] + speed: 4138.03 + value: {index: 18, type: Single} + - key: [256, 2944, 3328] + speed: 4238.83 + value: {index: 18, type: Single} + - key: [256, 3584, 256] + speed: 4210.88 + value: {index: 14, type: Single} + - key: [256, 3584, 768] + speed: 4702.15 + value: {index: 14, type: Single} + - key: [256, 3584, 1792] + speed: 4890.0 + value: {index: 14, type: Single} + - key: [256, 3584, 3328] + speed: 4946.07 + value: {index: 14, type: Single} + - key: [256, 4288, 256] + speed: 3502.9 + value: {index: 13, type: Single} + - key: [256, 4288, 768] + speed: 3966.87 + value: {index: 18, type: Single} + - key: [256, 4288, 1792] + speed: 4090.51 + value: {index: 18, type: Single} + - key: [256, 4288, 3328] + speed: 4173.78 + value: {index: 18, type: Single} + - key: [256, 5056, 256] + speed: 3983.17 + value: {index: 13, type: Single} + - key: [256, 5056, 768] + speed: 4477.58 + value: {index: 14, type: Single} + - key: [256, 5056, 1792] + speed: 4643.59 + value: {index: 14, type: Single} + - key: [256, 5056, 3328] + speed: 4718.85 + value: {index: 14, type: Single} + - key: [256, 5888, 256] + speed: 4605.82 + value: {index: 14, type: Single} + - key: [256, 5888, 768] + speed: 5230.11 + value: {index: 14, type: Single} + - key: [256, 5888, 1792] + speed: 5428.66 + value: {index: 14, type: Single} + - key: [256, 5888, 3328] + speed: 5498.94 + value: {index: 14, type: Single} + - key: [256, 6784, 256] + speed: 4235.62 + value: {index: 14, type: Single} + - key: [256, 6784, 768] + speed: 4658.65 + value: {index: 14, type: Single} + - key: [256, 6784, 1792] + speed: 4935.77 + value: {index: 15, type: Single} + - key: [256, 6784, 3328] + speed: 5004.79 + value: {index: 15, type: Single} + - key: [448, 4, 256] + speed: 33.1169 + value: {index: 20, type: Single} + - key: [448, 4, 768] + speed: 47.8839 + value: {index: 20, type: Single} + - key: [448, 4, 1792] + speed: 50.3495 + value: {index: 20, type: Single} + - key: [448, 4, 3328] + speed: 50.0678 + value: {index: 20, type: Single} + - key: [448, 64, 256] + speed: 516.068 + value: {index: 17, type: Single} + - key: [448, 64, 768] + speed: 669.507 + value: {index: 17, type: Single} + - key: [448, 64, 1792] + speed: 583.359 + value: {index: 17, type: Single} + - key: [448, 64, 3328] + speed: 606.185 + value: {index: 18, type: Single} + - key: [448, 128, 256] + speed: 888.677 + value: {index: 17, type: Single} + - key: [448, 128, 768] + speed: 1300.92 + value: {index: 18, type: Single} + - key: [448, 128, 1792] + speed: 1214.73 + value: {index: 18, type: Single} + - key: [448, 128, 3328] + speed: 1212.37 + value: {index: 18, type: Single} + - key: [448, 256, 256] + speed: 1777.35 + value: {index: 17, type: Single} + - key: [448, 256, 768] + speed: 2456.74 + value: {index: 18, type: Single} + - key: [448, 256, 1792] + speed: 2389.71 + value: {index: 18, type: Single} + - key: [448, 256, 3328] + speed: 2497.61 + value: {index: 18, type: Single} + - key: [448, 448, 256] + speed: 2375.41 + value: {index: 17, type: Single} + - key: [448, 448, 768] + speed: 3020.12 + value: {index: 17, type: Single} + - key: [448, 448, 1792] + speed: 3223.99 + value: {index: 18, type: Single} + - key: [448, 448, 3328] + speed: 3292.07 + value: {index: 18, type: Single} + - key: [448, 704, 256] + speed: 2691.3 + value: {index: 18, type: Single} + - key: [448, 704, 768] + speed: 3076.12 + value: {index: 18, type: Single} + - key: [448, 704, 1792] + speed: 3135.95 + value: {index: 17, type: Single} + - key: [448, 704, 3328] + speed: 3204.33 + value: {index: 18, type: Single} + - key: [448, 1024, 256] + speed: 3248.79 + value: {index: 14, type: Single} + - key: [448, 1024, 768] + speed: 3869.99 + value: {index: 14, type: Single} + - key: [448, 1024, 1792] + speed: 4128.67 + value: {index: 14, type: Single} + - key: [448, 1024, 3328] + speed: 4219.96 + value: {index: 14, type: Single} + - key: [448, 1408, 256] + speed: 3400.81 + value: {index: 17, type: Single} + - key: [448, 1408, 768] + speed: 3937.27 + value: {index: 18, type: Single} + - key: [448, 1408, 1792] + speed: 4034.78 + value: {index: 18, type: Single} + - key: [448, 1408, 3328] + speed: 4079.34 + value: {index: 18, type: Single} + - key: [448, 1856, 256] + speed: 3674.61 + value: {index: 14, type: Single} + - key: [448, 1856, 768] + speed: 4227.88 + value: {index: 14, type: Single} + - key: [448, 1856, 1792] + speed: 4410.18 + value: {index: 19, type: Single} + - key: [448, 1856, 3328] + speed: 4482.92 + value: {index: 19, type: Single} + - key: [448, 2368, 256] + speed: 3515.11 + value: {index: 13, type: Single} + - key: [448, 2368, 768] + speed: 3896.11 + value: {index: 18, type: Single} + - key: [448, 2368, 1792] + speed: 4011.22 + value: {index: 17, type: Single} + - key: [448, 2368, 3328] + speed: 4054.21 + value: {index: 19, type: Single} + - key: [448, 2944, 256] + speed: 4283.89 + value: {index: 14, type: Single} + - key: [448, 2944, 768] + speed: 4743.02 + value: {index: 19, type: Single} + - key: [448, 2944, 1792] + speed: 4935.23 + value: {index: 19, type: Single} + - key: [448, 2944, 3328] + speed: 4978.12 + value: {index: 19, type: Single} + - key: [448, 3584, 256] + speed: 4107.28 + value: {index: 14, type: Single} + - key: [448, 3584, 768] + speed: 4448.62 + value: {index: 19, type: Single} + - key: [448, 3584, 1792] + speed: 4619.15 + value: {index: 19, type: Single} + - key: [448, 3584, 3328] + speed: 4635.39 + value: {index: 15, type: Single} + - key: [448, 4288, 256] + speed: 4708.44 + value: {index: 14, type: Single} + - key: [448, 4288, 768] + speed: 5101.61 + value: {index: 14, type: Single} + - key: [448, 4288, 1792] + speed: 5289.95 + value: {index: 14, type: Single} + - key: [448, 4288, 3328] + speed: 5350.95 + value: {index: 14, type: Single} + - key: [448, 5056, 256] + speed: 4575.07 + value: {index: 14, type: Single} + - key: [448, 5056, 768] + speed: 5105.18 + value: {index: 19, type: Single} + - key: [448, 5056, 1792] + speed: 5300.41 + value: {index: 19, type: Single} + - key: [448, 5056, 3328] + speed: 5094.27 + value: {index: 14, type: Single} + - key: [448, 5888, 256] + speed: 4636.88 + value: {index: 14, type: Single} + - key: [448, 5888, 768] + speed: 4988.74 + value: {index: 19, type: Single} + - key: [448, 5888, 1792] + speed: 5128.39 + value: {index: 15, type: Single} + - key: [448, 5888, 3328] + speed: 5198.68 + value: {index: 15, type: Single} + - key: [448, 6784, 256] + speed: 5272.74 + value: {index: 14, type: Single} + - key: [448, 6784, 768] + speed: 5548.5 + value: {index: 14, type: Single} + - key: [448, 6784, 1792] + speed: 5691.09 + value: {index: 14, type: Single} + - key: [448, 6784, 3328] + speed: 5741.2 + value: {index: 14, type: Single} + - key: [704, 4, 256] + speed: 48.9041 + value: {index: 20, type: Single} + - key: [704, 4, 768] + speed: 66.5043 + value: {index: 20, type: Single} + - key: [704, 4, 1792] + speed: 75.7761 + value: {index: 20, type: Single} + - key: [704, 4, 3328] + speed: 78.4826 + value: {index: 20, type: Single} + - key: [704, 64, 256] + speed: 794.458 + value: {index: 18, type: Single} + - key: [704, 64, 768] + speed: 957.233 + value: {index: 18, type: Single} + - key: [704, 64, 1792] + speed: 902.293 + value: {index: 18, type: Single} + - key: [704, 64, 3328] + speed: 918.016 + value: {index: 17, type: Single} + - key: [704, 128, 256] + speed: 1428.53 + value: {index: 17, type: Single} + - key: [704, 128, 768] + speed: 1930.27 + value: {index: 18, type: Single} + - key: [704, 128, 1792] + speed: 1860.01 + value: {index: 17, type: Single} + - key: [704, 128, 3328] + speed: 2010.15 + value: {index: 18, type: Single} + - key: [704, 256, 256] + speed: 2133.03 + value: {index: 18, type: Single} + - key: [704, 256, 768] + speed: 2684.64 + value: {index: 18, type: Single} + - key: [704, 256, 1792] + speed: 2889.25 + value: {index: 18, type: Single} + - key: [704, 256, 3328] + speed: 2992.19 + value: {index: 18, type: Single} + - key: [704, 448, 256] + speed: 2678.04 + value: {index: 18, type: Single} + - key: [704, 448, 768] + speed: 3010.96 + value: {index: 18, type: Single} + - key: [704, 448, 1792] + speed: 3143.7 + value: {index: 17, type: Single} + - key: [704, 448, 3328] + speed: 3167.09 + value: {index: 18, type: Single} + - key: [704, 704, 256] + speed: 3300.2 + value: {index: 18, type: Single} + - key: [704, 704, 768] + speed: 3910.51 + value: {index: 18, type: Single} + - key: [704, 704, 1792] + speed: 4068.42 + value: {index: 18, type: Single} + - key: [704, 704, 3328] + speed: 4172.09 + value: {index: 18, type: Single} + - key: [704, 1024, 256] + speed: 3408.15 + value: {index: 14, type: Single} + - key: [704, 1024, 768] + speed: 3909.03 + value: {index: 18, type: Single} + - key: [704, 1024, 1792] + speed: 4089.93 + value: {index: 18, type: Single} + - key: [704, 1024, 3328] + speed: 4105.41 + value: {index: 18, type: Single} + - key: [704, 1408, 256] + speed: 4673.43 + value: {index: 14, type: Single} + - key: [704, 1408, 768] + speed: 5123.04 + value: {index: 14, type: Single} + - key: [704, 1408, 1792] + speed: 5282.94 + value: {index: 14, type: Single} + - key: [704, 1408, 3328] + speed: 5333.92 + value: {index: 14, type: Single} + - key: [704, 1856, 256] + speed: 4379.82 + value: {index: 19, type: Single} + - key: [704, 1856, 768] + speed: 4807.21 + value: {index: 19, type: Single} + - key: [704, 1856, 1792] + speed: 4830.97 + value: {index: 19, type: Single} + - key: [704, 1856, 3328] + speed: 4755.55 + value: {index: 14, type: Single} + - key: [704, 2368, 256] + speed: 4208.35 + value: {index: 19, type: Single} + - key: [704, 2368, 768] + speed: 4647.42 + value: {index: 19, type: Single} + - key: [704, 2368, 1792] + speed: 4762.49 + value: {index: 19, type: Single} + - key: [704, 2368, 3328] + speed: 4815.55 + value: {index: 15, type: Single} + - key: [704, 2944, 256] + speed: 5243.5 + value: {index: 14, type: Single} + - key: [704, 2944, 768] + speed: 5600.19 + value: {index: 14, type: Single} + - key: [704, 2944, 1792] + speed: 5728.8 + value: {index: 14, type: Single} + - key: [704, 2944, 3328] + speed: 5785.29 + value: {index: 14, type: Single} + - key: [704, 3584, 256] + speed: 5177.99 + value: {index: 14, type: Single} + - key: [704, 3584, 768] + speed: 5499.1 + value: {index: 14, type: Single} + - key: [704, 3584, 1792] + speed: 5631.34 + value: {index: 14, type: Single} + - key: [704, 3584, 3328] + speed: 5681.46 + value: {index: 14, type: Single} + - key: [704, 4288, 256] + speed: 5144.25 + value: {index: 14, type: Single} + - key: [704, 4288, 768] + speed: 5528.63 + value: {index: 14, type: Single} + - key: [704, 4288, 1792] + speed: 5637.9 + value: {index: 14, type: Single} + - key: [704, 4288, 3328] + speed: 5692.7 + value: {index: 14, type: Single} + - key: [704, 5056, 256] + speed: 5256.85 + value: {index: 14, type: Single} + - key: [704, 5056, 768] + speed: 5619.48 + value: {index: 14, type: Single} + - key: [704, 5056, 1792] + speed: 5718.38 + value: {index: 14, type: Single} + - key: [704, 5056, 3328] + speed: 5783.08 + value: {index: 14, type: Single} + - key: [704, 5888, 256] + speed: 5478.12 + value: {index: 14, type: Single} + - key: [704, 5888, 768] + speed: 5784.87 + value: {index: 14, type: Single} + - key: [704, 5888, 1792] + speed: 5872.73 + value: {index: 14, type: Single} + - key: [704, 5888, 3328] + speed: 5919.54 + value: {index: 14, type: Single} + - key: [704, 6784, 256] + speed: 5151.44 + value: {index: 14, type: Single} + - key: [704, 6784, 768] + speed: 5591.83 + value: {index: 15, type: Single} + - key: [704, 6784, 1792] + speed: 5756.68 + value: {index: 15, type: Single} + - key: [704, 6784, 3328] + speed: 5764.85 + value: {index: 15, type: Single} + - key: [1024, 4, 256] + speed: 71.1333 + value: {index: 20, type: Single} + - key: [1024, 4, 768] + speed: 95.2154 + value: {index: 20, type: Single} + - key: [1024, 4, 1792] + speed: 106.431 + value: {index: 20, type: Single} + - key: [1024, 4, 3328] + speed: 112.276 + value: {index: 20, type: Single} + - key: [1024, 64, 256] + speed: 1038.93 + value: {index: 17, type: Single} + - key: [1024, 64, 768] + speed: 1260.59 + value: {index: 18, type: Single} + - key: [1024, 64, 1792] + speed: 1295.27 + value: {index: 17, type: Single} + - key: [1024, 64, 3328] + speed: 1339.55 + value: {index: 17, type: Single} + - key: [1024, 128, 256] + speed: 2059.0 + value: {index: 17, type: Single} + - key: [1024, 128, 768] + speed: 2535.31 + value: {index: 18, type: Single} + - key: [1024, 128, 1792] + speed: 2644.55 + value: {index: 18, type: Single} + - key: [1024, 128, 3328] + speed: 2858.57 + value: {index: 18, type: Single} + - key: [1024, 256, 256] + speed: 2913.03 + value: {index: 18, type: Single} + - key: [1024, 256, 768] + speed: 3743.59 + value: {index: 18, type: Single} + - key: [1024, 256, 1792] + speed: 4107.26 + value: {index: 17, type: Single} + - key: [1024, 256, 3328] + speed: 4260.96 + value: {index: 18, type: Single} + - key: [1024, 448, 256] + speed: 3202.85 + value: {index: 19, type: Single} + - key: [1024, 448, 768] + speed: 3686.99 + value: {index: 14, type: Single} + - key: [1024, 448, 1792] + speed: 4057.72 + value: {index: 14, type: Single} + - key: [1024, 448, 3328] + speed: 4188.23 + value: {index: 14, type: Single} + - key: [1024, 704, 256] + speed: 3312.95 + value: {index: 19, type: Single} + - key: [1024, 704, 768] + speed: 3759.59 + value: {index: 18, type: Single} + - key: [1024, 704, 1792] + speed: 4031.32 + value: {index: 18, type: Single} + - key: [1024, 704, 3328] + speed: 4068.79 + value: {index: 18, type: Single} + - key: [1024, 1024, 256] + speed: 4616.3 + value: {index: 19, type: Single} + - key: [1024, 1024, 768] + speed: 5194.15 + value: {index: 19, type: Single} + - key: [1024, 1024, 1792] + speed: 5469.27 + value: {index: 14, type: Single} + - key: [1024, 1024, 3328] + speed: 5594.92 + value: {index: 14, type: Single} + - key: [1024, 1408, 256] + speed: 4749.94 + value: {index: 19, type: Single} + - key: [1024, 1408, 768] + speed: 5101.75 + value: {index: 19, type: Single} + - key: [1024, 1408, 1792] + speed: 5186.46 + value: {index: 19, type: Single} + - key: [1024, 1408, 3328] + speed: 5420.06 + value: {index: 19, type: Single} + - key: [1024, 1856, 256] + speed: 4628.7 + value: {index: 15, type: Single} + - key: [1024, 1856, 768] + speed: 5211.4 + value: {index: 15, type: Single} + - key: [1024, 1856, 1792] + speed: 5413.49 + value: {index: 15, type: Single} + - key: [1024, 1856, 3328] + speed: 5495.97 + value: {index: 15, type: Single} + - key: [1024, 2368, 256] + speed: 4997.04 + value: {index: 19, type: Single} + - key: [1024, 2368, 768] + speed: 5445.09 + value: {index: 19, type: Single} + - key: [1024, 2368, 1792] + speed: 5619.85 + value: {index: 19, type: Single} + - key: [1024, 2368, 3328] + speed: 5429.36 + value: {index: 14, type: Single} + - key: [1024, 2944, 256] + speed: 5243.27 + value: {index: 19, type: Single} + - key: [1024, 2944, 768] + speed: 5709.75 + value: {index: 15, type: Single} + - key: [1024, 2944, 1792] + speed: 5853.96 + value: {index: 15, type: Single} + - key: [1024, 2944, 3328] + speed: 5960.96 + value: {index: 15, type: Single} + - key: [1024, 3584, 256] + speed: 5540.95 + value: {index: 19, type: Single} + - key: [1024, 3584, 768] + speed: 6010.07 + value: {index: 19, type: Single} + - key: [1024, 3584, 1792] + speed: 5845.58 + value: {index: 14, type: Single} + - key: [1024, 3584, 3328] + speed: 5935.28 + value: {index: 14, type: Single} + - key: [1024, 4288, 256] + speed: 5283.64 + value: {index: 19, type: Single} + - key: [1024, 4288, 768] + speed: 5353.85 + value: {index: 14, type: Single} + - key: [1024, 4288, 1792] + speed: 5616.06 + value: {index: 19, type: Single} + - key: [1024, 4288, 3328] + speed: 5561.6 + value: {index: 14, type: Single} + - key: [1024, 5056, 256] + speed: 5561.84 + value: {index: 19, type: Single} + - key: [1024, 5056, 768] + speed: 6030.49 + value: {index: 19, type: Single} + - key: [1024, 5056, 1792] + speed: 6212.04 + value: {index: 15, type: Single} + - key: [1024, 5056, 3328] + speed: 6292.14 + value: {index: 15, type: Single} + - key: [1024, 5888, 256] + speed: 5414.96 + value: {index: 15, type: Single} + - key: [1024, 5888, 768] + speed: 5904.44 + value: {index: 15, type: Single} + - key: [1024, 5888, 1792] + speed: 6012.75 + value: {index: 15, type: Single} + - key: [1024, 5888, 3328] + speed: 6134.84 + value: {index: 15, type: Single} + - key: [1024, 6784, 256] + speed: 5437.73 + value: {index: 15, type: Single} + - key: [1024, 6784, 768] + speed: 5876.53 + value: {index: 15, type: Single} + - key: [1024, 6784, 1792] + speed: 6042.25 + value: {index: 15, type: Single} + - key: [1024, 6784, 3328] + speed: 6111.19 + value: {index: 15, type: Single} + - key: [1408, 4, 256] + speed: 97.8083 + value: {index: 20, type: Single} + - key: [1408, 4, 768] + speed: 132.406 + value: {index: 20, type: Single} + - key: [1408, 4, 1792] + speed: 148.578 + value: {index: 20, type: Single} + - key: [1408, 4, 3328] + speed: 154.662 + value: {index: 20, type: Single} + - key: [1408, 64, 256] + speed: 1428.53 + value: {index: 17, type: Single} + - key: [1408, 64, 768] + speed: 1730.11 + value: {index: 18, type: Single} + - key: [1408, 64, 1792] + speed: 1831.88 + value: {index: 18, type: Single} + - key: [1408, 64, 3328] + speed: 1890.03 + value: {index: 18, type: Single} + - key: [1408, 128, 256] + speed: 2177.73 + value: {index: 18, type: Single} + - key: [1408, 128, 768] + speed: 2676.96 + value: {index: 18, type: Single} + - key: [1408, 128, 1792] + speed: 2939.9 + value: {index: 18, type: Single} + - key: [1408, 128, 3328] + speed: 2947.54 + value: {index: 18, type: Single} + - key: [1408, 256, 256] + speed: 2903.68 + value: {index: 18, type: Single} + - key: [1408, 256, 768] + speed: 3425.32 + value: {index: 17, type: Single} + - key: [1408, 256, 1792] + speed: 3556.17 + value: {index: 18, type: Single} + - key: [1408, 256, 3328] + speed: 3673.73 + value: {index: 17, type: Single} + - key: [1408, 448, 256] + speed: 3364.11 + value: {index: 18, type: Single} + - key: [1408, 448, 768] + speed: 3911.36 + value: {index: 17, type: Single} + - key: [1408, 448, 1792] + speed: 4108.65 + value: {index: 18, type: Single} + - key: [1408, 448, 3328] + speed: 4110.69 + value: {index: 17, type: Single} + - key: [1408, 704, 256] + speed: 4573.57 + value: {index: 19, type: Single} + - key: [1408, 704, 768] + speed: 5107.78 + value: {index: 19, type: Single} + - key: [1408, 704, 1792] + speed: 5120.49 + value: {index: 19, type: Single} + - key: [1408, 704, 3328] + speed: 5328.18 + value: {index: 19, type: Single} + - key: [1408, 1024, 256] + speed: 4880.19 + value: {index: 19, type: Single} + - key: [1408, 1024, 768] + speed: 5100.0 + value: {index: 14, type: Single} + - key: [1408, 1024, 1792] + speed: 5351.17 + value: {index: 19, type: Single} + - key: [1408, 1024, 3328] + speed: 5259.01 + value: {index: 14, type: Single} + - key: [1408, 1408, 256] + speed: 5174.62 + value: {index: 19, type: Single} + - key: [1408, 1408, 768] + speed: 5600.45 + value: {index: 19, type: Single} + - key: [1408, 1408, 1792] + speed: 5676.91 + value: {index: 15, type: Single} + - key: [1408, 1408, 3328] + speed: 5742.85 + value: {index: 15, type: Single} + - key: [1408, 1856, 256] + speed: 5547.4 + value: {index: 19, type: Single} + - key: [1408, 1856, 768] + speed: 5132.3 + value: {index: 19, type: Single} + - key: [1408, 1856, 1792] + speed: 5802.48 + value: {index: 19, type: Single} + - key: [1408, 1856, 3328] + speed: 5170.2 + value: {index: 15, type: Single} + - key: [1408, 2368, 256] + speed: 5292.83 + value: {index: 19, type: Single} + - key: [1408, 2368, 768] + speed: 5581.71 + value: {index: 19, type: Single} + - key: [1408, 2368, 1792] + speed: 5622.3 + value: {index: 19, type: Single} + - key: [1408, 2368, 3328] + speed: 5419.8 + value: {index: 14, type: Single} + - key: [1408, 2944, 256] + speed: 5732.4 + value: {index: 19, type: Single} + - key: [1408, 2944, 768] + speed: 5995.51 + value: {index: 15, type: Single} + - key: [1408, 2944, 1792] + speed: 6184.59 + value: {index: 15, type: Single} + - key: [1408, 2944, 3328] + speed: 6253.27 + value: {index: 15, type: Single} + - key: [1408, 3584, 256] + speed: 5680.61 + value: {index: 19, type: Single} + - key: [1408, 3584, 768] + speed: 5950.0 + value: {index: 19, type: Single} + - key: [1408, 3584, 1792] + speed: 6089.2 + value: {index: 15, type: Single} + - key: [1408, 3584, 3328] + speed: 6149.32 + value: {index: 15, type: Single} + - key: [1408, 4288, 256] + speed: 5716.46 + value: {index: 19, type: Single} + - key: [1408, 4288, 768] + speed: 6046.09 + value: {index: 19, type: Single} + - key: [1408, 4288, 1792] + speed: 6100.13 + value: {index: 15, type: Single} + - key: [1408, 4288, 3328] + speed: 6159.92 + value: {index: 15, type: Single} + - key: [1408, 5056, 256] + speed: 5820.22 + value: {index: 19, type: Single} + - key: [1408, 5056, 768] + speed: 6008.82 + value: {index: 15, type: Single} + - key: [1408, 5056, 1792] + speed: 6190.77 + value: {index: 15, type: Single} + - key: [1408, 5056, 3328] + speed: 6256.77 + value: {index: 15, type: Single} + - key: [1408, 5888, 256] + speed: 5917.1 + value: {index: 19, type: Single} + - key: [1408, 5888, 768] + speed: 6201.42 + value: {index: 15, type: Single} + - key: [1408, 5888, 1792] + speed: 6354.89 + value: {index: 15, type: Single} + - key: [1408, 5888, 3328] + speed: 6374.74 + value: {index: 15, type: Single} + - key: [1408, 6784, 256] + speed: 5841.52 + value: {index: 19, type: Single} + - key: [1408, 6784, 768] + speed: 6038.86 + value: {index: 19, type: Single} + - key: [1408, 6784, 1792] + speed: 5911.15 + value: {index: 15, type: Single} + - key: [1408, 6784, 3328] + speed: 5948.56 + value: {index: 15, type: Single} + - key: [1856, 4, 256] + speed: 127.012 + value: {index: 20, type: Single} + - key: [1856, 4, 768] + speed: 169.165 + value: {index: 20, type: Single} + - key: [1856, 4, 1792] + speed: 179.06 + value: {index: 20, type: Single} + - key: [1856, 4, 3328] + speed: 194.369 + value: {index: 20, type: Single} + - key: [1856, 64, 256] + speed: 1849.12 + value: {index: 17, type: Single} + - key: [1856, 64, 768] + speed: 2156.8 + value: {index: 17, type: Single} + - key: [1856, 64, 1792] + speed: 2326.76 + value: {index: 17, type: Single} + - key: [1856, 64, 3328] + speed: 2377.1 + value: {index: 17, type: Single} + - key: [1856, 128, 256] + speed: 2755.04 + value: {index: 17, type: Single} + - key: [1856, 128, 768] + speed: 3488.7 + value: {index: 18, type: Single} + - key: [1856, 128, 1792] + speed: 3793.47 + value: {index: 18, type: Single} + - key: [1856, 128, 3328] + speed: 3840.66 + value: {index: 17, type: Single} + - key: [1856, 256, 256] + speed: 3303.9 + value: {index: 14, type: Single} + - key: [1856, 256, 768] + speed: 4004.94 + value: {index: 14, type: Single} + - key: [1856, 256, 1792] + speed: 4287.27 + value: {index: 14, type: Single} + - key: [1856, 256, 3328] + speed: 4369.78 + value: {index: 14, type: Single} + - key: [1856, 448, 256] + speed: 3806.02 + value: {index: 19, type: Single} + - key: [1856, 448, 768] + speed: 4257.11 + value: {index: 19, type: Single} + - key: [1856, 448, 1792] + speed: 4425.71 + value: {index: 19, type: Single} + - key: [1856, 448, 3328] + speed: 4505.08 + value: {index: 19, type: Single} + - key: [1856, 704, 256] + speed: 4379.79 + value: {index: 19, type: Single} + - key: [1856, 704, 768] + speed: 4733.31 + value: {index: 19, type: Single} + - key: [1856, 704, 1792] + speed: 4704.43 + value: {index: 14, type: Single} + - key: [1856, 704, 3328] + speed: 4760.56 + value: {index: 14, type: Single} + - key: [1856, 1024, 256] + speed: 4894.27 + value: {index: 19, type: Single} + - key: [1856, 1024, 768] + speed: 5351.54 + value: {index: 19, type: Single} + - key: [1856, 1024, 1792] + speed: 5417.95 + value: {index: 15, type: Single} + - key: [1856, 1024, 3328] + speed: 5500.93 + value: {index: 15, type: Single} + - key: [1856, 1408, 256] + speed: 5407.87 + value: {index: 14, type: Single} + - key: [1856, 1408, 768] + speed: 5741.36 + value: {index: 14, type: Single} + - key: [1856, 1408, 1792] + speed: 5841.08 + value: {index: 14, type: Single} + - key: [1856, 1408, 3328] + speed: 5877.01 + value: {index: 14, type: Single} + - key: [1856, 1856, 256] + speed: 5377.0 + value: {index: 19, type: Single} + - key: [1856, 1856, 768] + speed: 5655.44 + value: {index: 19, type: Single} + - key: [1856, 1856, 1792] + speed: 5554.42 + value: {index: 14, type: Single} + - key: [1856, 1856, 3328] + speed: 5595.93 + value: {index: 14, type: Single} + - key: [1856, 2368, 256] + speed: 5457.69 + value: {index: 19, type: Single} + - key: [1856, 2368, 768] + speed: 5628.25 + value: {index: 19, type: Single} + - key: [1856, 2368, 1792] + speed: 5556.38 + value: {index: 14, type: Single} + - key: [1856, 2368, 3328] + speed: 5589.32 + value: {index: 14, type: Single} + - key: [1856, 2944, 256] + speed: 5611.69 + value: {index: 19, type: Single} + - key: [1856, 2944, 768] + speed: 5946.88 + value: {index: 19, type: Single} + - key: [1856, 2944, 1792] + speed: 5691.67 + value: {index: 14, type: Single} + - key: [1856, 2944, 3328] + speed: 5718.08 + value: {index: 14, type: Single} + - key: [1856, 3584, 256] + speed: 5556.77 + value: {index: 14, type: Single} + - key: [1856, 3584, 768] + speed: 5822.79 + value: {index: 14, type: Single} + - key: [1856, 3584, 1792] + speed: 5880.67 + value: {index: 14, type: Single} + - key: [1856, 3584, 3328] + speed: 5903.89 + value: {index: 14, type: Single} + - key: [1856, 4288, 256] + speed: 5744.35 + value: {index: 19, type: Single} + - key: [1856, 4288, 768] + speed: 5958.83 + value: {index: 15, type: Single} + - key: [1856, 4288, 1792] + speed: 6100.37 + value: {index: 15, type: Single} + - key: [1856, 4288, 3328] + speed: 6153.65 + value: {index: 15, type: Single} + - key: [1856, 5056, 256] + speed: 5743.9 + value: {index: 19, type: Single} + - key: [1856, 5056, 768] + speed: 5957.39 + value: {index: 19, type: Single} + - key: [1856, 5056, 1792] + speed: 5795.51 + value: {index: 15, type: Single} + - key: [1856, 5056, 3328] + speed: 5837.83 + value: {index: 15, type: Single} + - key: [1856, 5888, 256] + speed: 5776.48 + value: {index: 19, type: Single} + - key: [1856, 5888, 768] + speed: 6018.59 + value: {index: 15, type: Single} + - key: [1856, 5888, 1792] + speed: 6140.49 + value: {index: 15, type: Single} + - key: [1856, 5888, 3328] + speed: 6191.25 + value: {index: 15, type: Single} + - key: [1856, 6784, 256] + speed: 5869.92 + value: {index: 19, type: Single} + - key: [1856, 6784, 768] + speed: 5944.5 + value: {index: 19, type: Single} + - key: [1856, 6784, 1792] + speed: 6010.54 + value: {index: 15, type: Single} + - key: [1856, 6784, 3328] + speed: 6057.68 + value: {index: 15, type: Single} + - key: [2368, 4, 256] + speed: 139.294 + value: {index: 20, type: Single} + - key: [2368, 4, 768] + speed: 188.127 + value: {index: 20, type: Single} + - key: [2368, 4, 1792] + speed: 208.31 + value: {index: 20, type: Single} + - key: [2368, 4, 3328] + speed: 201.109 + value: {index: 20, type: Single} + - key: [2368, 64, 256] + speed: 1844.2 + value: {index: 18, type: Single} + - key: [2368, 64, 768] + speed: 2283.78 + value: {index: 18, type: Single} + - key: [2368, 64, 1792] + speed: 2458.92 + value: {index: 18, type: Single} + - key: [2368, 64, 3328] + speed: 2469.62 + value: {index: 18, type: Single} + - key: [2368, 128, 256] + speed: 2413.59 + value: {index: 18, type: Single} + - key: [2368, 128, 768] + speed: 2869.85 + value: {index: 18, type: Single} + - key: [2368, 128, 1792] + speed: 3016.24 + value: {index: 18, type: Single} + - key: [2368, 128, 3328] + speed: 2982.37 + value: {index: 18, type: Single} + - key: [2368, 256, 256] + speed: 3203.34 + value: {index: 18, type: Single} + - key: [2368, 256, 768] + speed: 3799.89 + value: {index: 18, type: Single} + - key: [2368, 256, 1792] + speed: 3907.54 + value: {index: 18, type: Single} + - key: [2368, 256, 3328] + speed: 4010.46 + value: {index: 18, type: Single} + - key: [2368, 448, 256] + speed: 3604.96 + value: {index: 19, type: Single} + - key: [2368, 448, 768] + speed: 3922.51 + value: {index: 19, type: Single} + - key: [2368, 448, 1792] + speed: 4047.27 + value: {index: 18, type: Single} + - key: [2368, 448, 3328] + speed: 4056.62 + value: {index: 19, type: Single} + - key: [2368, 704, 256] + speed: 4387.84 + value: {index: 19, type: Single} + - key: [2368, 704, 768] + speed: 4736.58 + value: {index: 19, type: Single} + - key: [2368, 704, 1792] + speed: 4797.05 + value: {index: 19, type: Single} + - key: [2368, 704, 3328] + speed: 4816.79 + value: {index: 15, type: Single} + - key: [2368, 1024, 256] + speed: 5185.64 + value: {index: 19, type: Single} + - key: [2368, 1024, 768] + speed: 5329.66 + value: {index: 14, type: Single} + - key: [2368, 1024, 1792] + speed: 5649.62 + value: {index: 19, type: Single} + - key: [2368, 1024, 3328] + speed: 5463.39 + value: {index: 14, type: Single} + - key: [2368, 1408, 256] + speed: 5283.12 + value: {index: 19, type: Single} + - key: [2368, 1408, 768] + speed: 5586.21 + value: {index: 19, type: Single} + - key: [2368, 1408, 1792] + speed: 5404.91 + value: {index: 14, type: Single} + - key: [2368, 1408, 3328] + speed: 5428.45 + value: {index: 14, type: Single} + - key: [2368, 1856, 256] + speed: 5451.81 + value: {index: 19, type: Single} + - key: [2368, 1856, 768] + speed: 5718.66 + value: {index: 19, type: Single} + - key: [2368, 1856, 1792] + speed: 5558.7 + value: {index: 14, type: Single} + - key: [2368, 1856, 3328] + speed: 5585.84 + value: {index: 14, type: Single} + - key: [2368, 2368, 256] + speed: 5750.37 + value: {index: 19, type: Single} + - key: [2368, 2368, 768] + speed: 5741.29 + value: {index: 14, type: Single} + - key: [2368, 2368, 1792] + speed: 5827.95 + value: {index: 14, type: Single} + - key: [2368, 2368, 3328] + speed: 5856.11 + value: {index: 14, type: Single} + - key: [2368, 2944, 256] + speed: 5729.49 + value: {index: 19, type: Single} + - key: [2368, 2944, 768] + speed: 5954.65 + value: {index: 19, type: Single} + - key: [2368, 2944, 1792] + speed: 6084.63 + value: {index: 15, type: Single} + - key: [2368, 2944, 3328] + speed: 6139.54 + value: {index: 15, type: Single} + - key: [2368, 3584, 256] + speed: 5789.58 + value: {index: 19, type: Single} + - key: [2368, 3584, 768] + speed: 5977.18 + value: {index: 19, type: Single} + - key: [2368, 3584, 1792] + speed: 5815.66 + value: {index: 15, type: Single} + - key: [2368, 3584, 3328] + speed: 5855.28 + value: {index: 15, type: Single} + - key: [2368, 4288, 256] + speed: 5833.96 + value: {index: 19, type: Single} + - key: [2368, 4288, 768] + speed: 5801.49 + value: {index: 14, type: Single} + - key: [2368, 4288, 1792] + speed: 5862.36 + value: {index: 14, type: Single} + - key: [2368, 4288, 3328] + speed: 5886.88 + value: {index: 14, type: Single} + - key: [2368, 5056, 256] + speed: 5827.64 + value: {index: 19, type: Single} + - key: [2368, 5056, 768] + speed: 6030.64 + value: {index: 15, type: Single} + - key: [2368, 5056, 1792] + speed: 6158.75 + value: {index: 15, type: Single} + - key: [2368, 5056, 3328] + speed: 6222.28 + value: {index: 15, type: Single} + - key: [2368, 5888, 256] + speed: 5848.4 + value: {index: 19, type: Single} + - key: [2368, 5888, 768] + speed: 6067.37 + value: {index: 15, type: Single} + - key: [2368, 5888, 1792] + speed: 6191.4 + value: {index: 15, type: Single} + - key: [2368, 5888, 3328] + speed: 6231.65 + value: {index: 15, type: Single} + - key: [2368, 6784, 256] + speed: 5879.23 + value: {index: 19, type: Single} + - key: [2368, 6784, 768] + speed: 6139.51 + value: {index: 15, type: Single} + - key: [2368, 6784, 1792] + speed: 6248.35 + value: {index: 15, type: Single} + - key: [2368, 6784, 3328] + speed: 6281.46 + value: {index: 15, type: Single} + - key: [2944, 4, 256] + speed: 166.791 + value: {index: 20, type: Single} + - key: [2944, 4, 768] + speed: 227.355 + value: {index: 20, type: Single} + - key: [2944, 4, 1792] + speed: 223.608 + value: {index: 20, type: Single} + - key: [2944, 4, 3328] + speed: 222.48 + value: {index: 20, type: Single} + - key: [2944, 64, 256] + speed: 2253.11 + value: {index: 18, type: Single} + - key: [2944, 64, 768] + speed: 2790.62 + value: {index: 18, type: Single} + - key: [2944, 64, 1792] + speed: 2986.92 + value: {index: 18, type: Single} + - key: [2944, 64, 3328] + speed: 3068.12 + value: {index: 18, type: Single} + - key: [2944, 128, 256] + speed: 2718.78 + value: {index: 19, type: Single} + - key: [2944, 128, 768] + speed: 3488.3 + value: {index: 18, type: Single} + - key: [2944, 128, 1792] + speed: 3678.81 + value: {index: 18, type: Single} + - key: [2944, 128, 3328] + speed: 3852.95 + value: {index: 18, type: Single} + - key: [2944, 256, 256] + speed: 3597.51 + value: {index: 14, type: Single} + - key: [2944, 256, 768] + speed: 4065.45 + value: {index: 18, type: Single} + - key: [2944, 256, 1792] + speed: 4245.96 + value: {index: 18, type: Single} + - key: [2944, 256, 3328] + speed: 4292.02 + value: {index: 18, type: Single} + - key: [2944, 448, 256] + speed: 4438.21 + value: {index: 19, type: Single} + - key: [2944, 448, 768] + speed: 4861.05 + value: {index: 19, type: Single} + - key: [2944, 448, 1792] + speed: 4951.32 + value: {index: 19, type: Single} + - key: [2944, 448, 3328] + speed: 4802.99 + value: {index: 14, type: Single} + - key: [2944, 704, 256] + speed: 5270.51 + value: {index: 19, type: Single} + - key: [2944, 704, 768] + speed: 5863.01 + value: {index: 19, type: Single} + - key: [2944, 704, 1792] + speed: 5807.1 + value: {index: 19, type: Single} + - key: [2944, 704, 3328] + speed: 4646.19 + value: {index: 14, type: Single} + - key: [2944, 1024, 256] + speed: 5291.21 + value: {index: 19, type: Single} + - key: [2944, 1024, 768] + speed: 5845.37 + value: {index: 19, type: Single} + - key: [2944, 1024, 1792] + speed: 5897.52 + value: {index: 15, type: Single} + - key: [2944, 1024, 3328] + speed: 5960.7 + value: {index: 15, type: Single} + - key: [2944, 1408, 256] + speed: 5680.13 + value: {index: 19, type: Single} + - key: [2944, 1408, 768] + speed: 5998.02 + value: {index: 15, type: Single} + - key: [2944, 1408, 1792] + speed: 6171.27 + value: {index: 15, type: Single} + - key: [2944, 1408, 3328] + speed: 6244.25 + value: {index: 15, type: Single} + - key: [2944, 1856, 256] + speed: 5581.83 + value: {index: 19, type: Single} + - key: [2944, 1856, 768] + speed: 5912.74 + value: {index: 19, type: Single} + - key: [2944, 1856, 1792] + speed: 5677.49 + value: {index: 14, type: Single} + - key: [2944, 1856, 3328] + speed: 5707.45 + value: {index: 14, type: Single} + - key: [2944, 2368, 256] + speed: 5668.82 + value: {index: 19, type: Single} + - key: [2944, 2368, 768] + speed: 5967.93 + value: {index: 19, type: Single} + - key: [2944, 2368, 1792] + speed: 6072.36 + value: {index: 15, type: Single} + - key: [2944, 2368, 3328] + speed: 6128.61 + value: {index: 15, type: Single} + - key: [2944, 2944, 256] + speed: 5795.83 + value: {index: 19, type: Single} + - key: [2944, 2944, 768] + speed: 5920.71 + value: {index: 19, type: Single} + - key: [2944, 2944, 1792] + speed: 5921.37 + value: {index: 15, type: Single} + - key: [2944, 2944, 3328] + speed: 5978.7 + value: {index: 15, type: Single} + - key: [2944, 3584, 256] + speed: 5789.83 + value: {index: 19, type: Single} + - key: [2944, 3584, 768] + speed: 5973.52 + value: {index: 19, type: Single} + - key: [2944, 3584, 1792] + speed: 5946.56 + value: {index: 15, type: Single} + - key: [2944, 3584, 3328] + speed: 5986.14 + value: {index: 15, type: Single} + - key: [2944, 4288, 256] + speed: 5814.61 + value: {index: 19, type: Single} + - key: [2944, 4288, 768] + speed: 5971.39 + value: {index: 19, type: Single} + - key: [2944, 4288, 1792] + speed: 6009.46 + value: {index: 15, type: Single} + - key: [2944, 4288, 3328] + speed: 6065.46 + value: {index: 15, type: Single} + - key: [2944, 5056, 256] + speed: 5915.45 + value: {index: 19, type: Single} + - key: [2944, 5056, 768] + speed: 6047.59 + value: {index: 19, type: Single} + - key: [2944, 5056, 1792] + speed: 6162.15 + value: {index: 15, type: Single} + - key: [2944, 5056, 3328] + speed: 6213.81 + value: {index: 15, type: Single} + - key: [2944, 5888, 256] + speed: 5937.13 + value: {index: 19, type: Single} + - key: [2944, 5888, 768] + speed: 6244.08 + value: {index: 15, type: Single} + - key: [2944, 5888, 1792] + speed: 6353.35 + value: {index: 15, type: Single} + - key: [2944, 5888, 3328] + speed: 6395.98 + value: {index: 15, type: Single} + - key: [2944, 6784, 256] + speed: 5999.28 + value: {index: 19, type: Single} + - key: [2944, 6784, 768] + speed: 6166.33 + value: {index: 15, type: Single} + - key: [2944, 6784, 1792] + speed: 6253.02 + value: {index: 15, type: Single} + - key: [2944, 6784, 3328] + speed: 6294.07 + value: {index: 15, type: Single} + - key: [3584, 4, 256] + speed: 195.053 + value: {index: 20, type: Single} + - key: [3584, 4, 768] + speed: 258.941 + value: {index: 20, type: Single} + - key: [3584, 4, 1792] + speed: 200.005 + value: {index: 20, type: Single} + - key: [3584, 4, 3328] + speed: 170.705 + value: {index: 20, type: Single} + - key: [3584, 64, 256] + speed: 2696.25 + value: {index: 18, type: Single} + - key: [3584, 64, 768] + speed: 3378.0 + value: {index: 18, type: Single} + - key: [3584, 64, 1792] + speed: 3596.18 + value: {index: 18, type: Single} + - key: [3584, 64, 3328] + speed: 3743.23 + value: {index: 18, type: Single} + - key: [3584, 128, 256] + speed: 3268.86 + value: {index: 19, type: Single} + - key: [3584, 128, 768] + speed: 3895.36 + value: {index: 14, type: Single} + - key: [3584, 128, 1792] + speed: 4150.27 + value: {index: 14, type: Single} + - key: [3584, 128, 3328] + speed: 4234.7 + value: {index: 14, type: Single} + - key: [3584, 256, 256] + speed: 4302.32 + value: {index: 14, type: Single} + - key: [3584, 256, 768] + speed: 4741.99 + value: {index: 19, type: Single} + - key: [3584, 256, 1792] + speed: 4885.68 + value: {index: 14, type: Single} + - key: [3584, 256, 3328] + speed: 4938.96 + value: {index: 14, type: Single} + - key: [3584, 448, 256] + speed: 4229.35 + value: {index: 19, type: Single} + - key: [3584, 448, 768] + speed: 4493.05 + value: {index: 19, type: Single} + - key: [3584, 448, 1792] + speed: 4588.6 + value: {index: 19, type: Single} + - key: [3584, 448, 3328] + speed: 4620.55 + value: {index: 15, type: Single} + - key: [3584, 704, 256] + speed: 5271.89 + value: {index: 19, type: Single} + - key: [3584, 704, 768] + speed: 5773.38 + value: {index: 19, type: Single} + - key: [3584, 704, 1792] + speed: 5642.28 + value: {index: 19, type: Single} + - key: [3584, 704, 3328] + speed: 4978.98 + value: {index: 15, type: Single} + - key: [3584, 1024, 256] + speed: 5644.53 + value: {index: 19, type: Single} + - key: [3584, 1024, 768] + speed: 6130.14 + value: {index: 19, type: Single} + - key: [3584, 1024, 1792] + speed: 5912.54 + value: {index: 14, type: Single} + - key: [3584, 1024, 3328] + speed: 5952.0 + value: {index: 14, type: Single} + - key: [3584, 1408, 256] + speed: 5557.51 + value: {index: 19, type: Single} + - key: [3584, 1408, 768] + speed: 5989.51 + value: {index: 19, type: Single} + - key: [3584, 1408, 1792] + speed: 6074.96 + value: {index: 15, type: Single} + - key: [3584, 1408, 3328] + speed: 6145.32 + value: {index: 15, type: Single} + - key: [3584, 1856, 256] + speed: 5690.19 + value: {index: 19, type: Single} + - key: [3584, 1856, 768] + speed: 6036.34 + value: {index: 19, type: Single} + - key: [3584, 1856, 1792] + speed: 5784.91 + value: {index: 15, type: Single} + - key: [3584, 1856, 3328] + speed: 5843.51 + value: {index: 15, type: Single} + - key: [3584, 2368, 256] + speed: 5615.55 + value: {index: 19, type: Single} + - key: [3584, 2368, 768] + speed: 6019.3 + value: {index: 19, type: Single} + - key: [3584, 2368, 1792] + speed: 5779.31 + value: {index: 15, type: Single} + - key: [3584, 2368, 3328] + speed: 5841.65 + value: {index: 15, type: Single} + - key: [3584, 2944, 256] + speed: 5663.92 + value: {index: 19, type: Single} + - key: [3584, 2944, 768] + speed: 6082.8 + value: {index: 19, type: Single} + - key: [3584, 2944, 1792] + speed: 5918.99 + value: {index: 15, type: Single} + - key: [3584, 2944, 3328] + speed: 5974.67 + value: {index: 15, type: Single} + - key: [3584, 3584, 256] + speed: 5810.39 + value: {index: 19, type: Single} + - key: [3584, 3584, 768] + speed: 5990.74 + value: {index: 19, type: Single} + - key: [3584, 3584, 1792] + speed: 6115.12 + value: {index: 15, type: Single} + - key: [3584, 3584, 3328] + speed: 6162.77 + value: {index: 15, type: Single} + - key: [3584, 4288, 256] + speed: 5824.22 + value: {index: 19, type: Single} + - key: [3584, 4288, 768] + speed: 6180.98 + value: {index: 19, type: Single} + - key: [3584, 4288, 1792] + speed: 6339.57 + value: {index: 15, type: Single} + - key: [3584, 4288, 3328] + speed: 6397.39 + value: {index: 15, type: Single} + - key: [3584, 5056, 256] + speed: 5882.91 + value: {index: 19, type: Single} + - key: [3584, 5056, 768] + speed: 6085.47 + value: {index: 15, type: Single} + - key: [3584, 5056, 1792] + speed: 6241.05 + value: {index: 15, type: Single} + - key: [3584, 5056, 3328] + speed: 6305.96 + value: {index: 15, type: Single} + - key: [3584, 5888, 256] + speed: 5853.49 + value: {index: 19, type: Single} + - key: [3584, 5888, 768] + speed: 6124.86 + value: {index: 15, type: Single} + - key: [3584, 5888, 1792] + speed: 6257.9 + value: {index: 15, type: Single} + - key: [3584, 5888, 3328] + speed: 6310.93 + value: {index: 15, type: Single} + - key: [3584, 6784, 256] + speed: 5904.47 + value: {index: 19, type: Single} + - key: [3584, 6784, 768] + speed: 6186.46 + value: {index: 15, type: Single} + - key: [3584, 6784, 1792] + speed: 6317.04 + value: {index: 15, type: Single} + - key: [3584, 6784, 3328] + speed: 6369.06 + value: {index: 15, type: Single} + - key: [4288, 4, 256] + speed: 238.054 + value: {index: 20, type: Single} + - key: [4288, 4, 768] + speed: 280.928 + value: {index: 20, type: Single} + - key: [4288, 4, 1792] + speed: 165.245 + value: {index: 20, type: Single} + - key: [4288, 4, 3328] + speed: 158.72 + value: {index: 20, type: Single} + - key: [4288, 64, 256] + speed: 2126.48 + value: {index: 18, type: Single} + - key: [4288, 64, 768] + speed: 2476.27 + value: {index: 18, type: Single} + - key: [4288, 64, 1792] + speed: 2621.99 + value: {index: 18, type: Single} + - key: [4288, 64, 3328] + speed: 2820.07 + value: {index: 18, type: Single} + - key: [4288, 128, 256] + speed: 2860.97 + value: {index: 17, type: Single} + - key: [4288, 128, 768] + speed: 3365.17 + value: {index: 18, type: Single} + - key: [4288, 128, 1792] + speed: 3584.71 + value: {index: 18, type: Single} + - key: [4288, 128, 3328] + speed: 3651.52 + value: {index: 18, type: Single} + - key: [4288, 256, 256] + speed: 3704.73 + value: {index: 19, type: Single} + - key: [4288, 256, 768] + speed: 4037.23 + value: {index: 19, type: Single} + - key: [4288, 256, 1792] + speed: 4126.74 + value: {index: 18, type: Single} + - key: [4288, 256, 3328] + speed: 4194.73 + value: {index: 17, type: Single} + - key: [4288, 448, 256] + speed: 4932.3 + value: {index: 19, type: Single} + - key: [4288, 448, 768] + speed: 5423.93 + value: {index: 19, type: Single} + - key: [4288, 448, 1792] + speed: 5411.3 + value: {index: 19, type: Single} + - key: [4288, 448, 3328] + speed: 4492.97 + value: {index: 18, type: Single} + - key: [4288, 704, 256] + speed: 5462.07 + value: {index: 19, type: Single} + - key: [4288, 704, 768] + speed: 5789.42 + value: {index: 19, type: Single} + - key: [4288, 704, 1792] + speed: 5438.07 + value: {index: 19, type: Single} + - key: [4288, 704, 3328] + speed: 4901.8 + value: {index: 14, type: Single} + - key: [4288, 1024, 256] + speed: 5450.65 + value: {index: 19, type: Single} + - key: [4288, 1024, 768] + speed: 5646.71 + value: {index: 19, type: Single} + - key: [4288, 1024, 1792] + speed: 5567.2 + value: {index: 14, type: Single} + - key: [4288, 1024, 3328] + speed: 5596.31 + value: {index: 14, type: Single} + - key: [4288, 1408, 256] + speed: 5676.03 + value: {index: 19, type: Single} + - key: [4288, 1408, 768] + speed: 5935.45 + value: {index: 15, type: Single} + - key: [4288, 1408, 1792] + speed: 6114.95 + value: {index: 15, type: Single} + - key: [4288, 1408, 3328] + speed: 6170.29 + value: {index: 15, type: Single} + - key: [4288, 1856, 256] + speed: 5737.16 + value: {index: 19, type: Single} + - key: [4288, 1856, 768] + speed: 5924.6 + value: {index: 15, type: Single} + - key: [4288, 1856, 1792] + speed: 6078.99 + value: {index: 15, type: Single} + - key: [4288, 1856, 3328] + speed: 6145.93 + value: {index: 15, type: Single} + - key: [4288, 2368, 256] + speed: 5811.73 + value: {index: 19, type: Single} + - key: [4288, 2368, 768] + speed: 5944.98 + value: {index: 19, type: Single} + - key: [4288, 2368, 1792] + speed: 5852.31 + value: {index: 14, type: Single} + - key: [4288, 2368, 3328] + speed: 5886.96 + value: {index: 14, type: Single} + - key: [4288, 2944, 256] + speed: 5866.21 + value: {index: 19, type: Single} + - key: [4288, 2944, 768] + speed: 5934.3 + value: {index: 15, type: Single} + - key: [4288, 2944, 1792] + speed: 6027.13 + value: {index: 15, type: Single} + - key: [4288, 2944, 3328] + speed: 6074.23 + value: {index: 15, type: Single} + - key: [4288, 3584, 256] + speed: 5914.38 + value: {index: 19, type: Single} + - key: [4288, 3584, 768] + speed: 6248.12 + value: {index: 15, type: Single} + - key: [4288, 3584, 1792] + speed: 6367.58 + value: {index: 15, type: Single} + - key: [4288, 3584, 3328] + speed: 6415.87 + value: {index: 15, type: Single} + - key: [4288, 4288, 256] + speed: 5962.06 + value: {index: 19, type: Single} + - key: [4288, 4288, 768] + speed: 5986.59 + value: {index: 19, type: Single} + - key: [4288, 4288, 1792] + speed: 6045.52 + value: {index: 15, type: Single} + - key: [4288, 4288, 3328] + speed: 6086.38 + value: {index: 15, type: Single} + - key: [4288, 5056, 256] + speed: 6005.98 + value: {index: 19, type: Single} + - key: [4288, 5056, 768] + speed: 6077.92 + value: {index: 15, type: Single} + - key: [4288, 5056, 1792] + speed: 6167.07 + value: {index: 15, type: Single} + - key: [4288, 5056, 3328] + speed: 6203.12 + value: {index: 15, type: Single} + - key: [4288, 5888, 256] + speed: 5947.37 + value: {index: 15, type: Single} + - key: [4288, 5888, 768] + speed: 6243.72 + value: {index: 15, type: Single} + - key: [4288, 5888, 1792] + speed: 6329.5 + value: {index: 15, type: Single} + - key: [4288, 5888, 3328] + speed: 6367.76 + value: {index: 15, type: Single} + - key: [4288, 6784, 256] + speed: 5986.53 + value: {index: 19, type: Single} + - key: [4288, 6784, 768] + speed: 6225.13 + value: {index: 15, type: Single} + - key: [4288, 6784, 1792] + speed: 6305.7 + value: {index: 15, type: Single} + - key: [4288, 6784, 3328] + speed: 6333.35 + value: {index: 15, type: Single} + - key: [5056, 4, 256] + speed: 258.854 + value: {index: 20, type: Single} + - key: [5056, 4, 768] + speed: 318.661 + value: {index: 20, type: Single} + - key: [5056, 4, 1792] + speed: 167.493 + value: {index: 20, type: Single} + - key: [5056, 4, 3328] + speed: 165.17 + value: {index: 20, type: Single} + - key: [5056, 64, 256] + speed: 2410.1 + value: {index: 17, type: Single} + - key: [5056, 64, 768] + speed: 2879.69 + value: {index: 17, type: Single} + - key: [5056, 64, 1792] + speed: 3090.38 + value: {index: 17, type: Single} + - key: [5056, 64, 3328] + speed: 3119.65 + value: {index: 17, type: Single} + - key: [5056, 128, 256] + speed: 3284.21 + value: {index: 17, type: Single} + - key: [5056, 128, 768] + speed: 3914.63 + value: {index: 18, type: Single} + - key: [5056, 128, 1792] + speed: 3933.64 + value: {index: 17, type: Single} + - key: [5056, 128, 3328] + speed: 4196.77 + value: {index: 17, type: Single} + - key: [5056, 256, 256] + speed: 4359.75 + value: {index: 19, type: Single} + - key: [5056, 256, 768] + speed: 4760.32 + value: {index: 19, type: Single} + - key: [5056, 256, 1792] + speed: 4826.85 + value: {index: 19, type: Single} + - key: [5056, 256, 3328] + speed: 4832.58 + value: {index: 19, type: Single} + - key: [5056, 448, 256] + speed: 4901.65 + value: {index: 19, type: Single} + - key: [5056, 448, 768] + speed: 5204.75 + value: {index: 19, type: Single} + - key: [5056, 448, 1792] + speed: 5245.1 + value: {index: 19, type: Single} + - key: [5056, 448, 3328] + speed: 5082.57 + value: {index: 14, type: Single} + - key: [5056, 704, 256] + speed: 5558.53 + value: {index: 19, type: Single} + - key: [5056, 704, 768] + speed: 5881.91 + value: {index: 19, type: Single} + - key: [5056, 704, 1792] + speed: 5646.76 + value: {index: 19, type: Single} + - key: [5056, 704, 3328] + speed: 5365.15 + value: {index: 15, type: Single} + - key: [5056, 1024, 256] + speed: 5710.95 + value: {index: 19, type: Single} + - key: [5056, 1024, 768] + speed: 6133.85 + value: {index: 19, type: Single} + - key: [5056, 1024, 1792] + speed: 6230.27 + value: {index: 15, type: Single} + - key: [5056, 1024, 3328] + speed: 6299.98 + value: {index: 15, type: Single} + - key: [5056, 1408, 256] + speed: 5737.44 + value: {index: 19, type: Single} + - key: [5056, 1408, 768] + speed: 6044.25 + value: {index: 15, type: Single} + - key: [5056, 1408, 1792] + speed: 6202.58 + value: {index: 15, type: Single} + - key: [5056, 1408, 3328] + speed: 6266.33 + value: {index: 15, type: Single} + - key: [5056, 1856, 256] + speed: 5662.66 + value: {index: 19, type: Single} + - key: [5056, 1856, 768] + speed: 5860.15 + value: {index: 19, type: Single} + - key: [5056, 1856, 1792] + speed: 5788.13 + value: {index: 15, type: Single} + - key: [5056, 1856, 3328] + speed: 5839.53 + value: {index: 15, type: Single} + - key: [5056, 2368, 256] + speed: 5740.33 + value: {index: 19, type: Single} + - key: [5056, 2368, 768] + speed: 6055.65 + value: {index: 15, type: Single} + - key: [5056, 2368, 1792] + speed: 6175.43 + value: {index: 15, type: Single} + - key: [5056, 2368, 3328] + speed: 6224.15 + value: {index: 15, type: Single} + - key: [5056, 2944, 256] + speed: 5920.9 + value: {index: 19, type: Single} + - key: [5056, 2944, 768] + speed: 6074.96 + value: {index: 15, type: Single} + - key: [5056, 2944, 1792] + speed: 6180.77 + value: {index: 15, type: Single} + - key: [5056, 2944, 3328] + speed: 6221.96 + value: {index: 15, type: Single} + - key: [5056, 3584, 256] + speed: 5981.24 + value: {index: 19, type: Single} + - key: [5056, 3584, 768] + speed: 6188.1 + value: {index: 15, type: Single} + - key: [5056, 3584, 1792] + speed: 6283.27 + value: {index: 15, type: Single} + - key: [5056, 3584, 3328] + speed: 6326.0 + value: {index: 15, type: Single} + - key: [5056, 4288, 256] + speed: 5968.66 + value: {index: 19, type: Single} + - key: [5056, 4288, 768] + speed: 6084.01 + value: {index: 15, type: Single} + - key: [5056, 4288, 1792] + speed: 6165.33 + value: {index: 15, type: Single} + - key: [5056, 4288, 3328] + speed: 6207.11 + value: {index: 15, type: Single} + - key: [5056, 5056, 256] + speed: 5945.48 + value: {index: 19, type: Single} + - key: [5056, 5056, 768] + speed: 6263.3 + value: {index: 15, type: Single} + - key: [5056, 5056, 1792] + speed: 6389.1 + value: {index: 15, type: Single} + - key: [5056, 5056, 3328] + speed: 6437.04 + value: {index: 15, type: Single} + - key: [5056, 5888, 256] + speed: 6062.91 + value: {index: 15, type: Single} + - key: [5056, 5888, 768] + speed: 6347.71 + value: {index: 15, type: Single} + - key: [5056, 5888, 1792] + speed: 6415.66 + value: {index: 15, type: Single} + - key: [5056, 5888, 3328] + speed: 6478.15 + value: {index: 15, type: Single} + - key: [5056, 6784, 256] + speed: 6005.24 + value: {index: 15, type: Single} + - key: [5056, 6784, 768] + speed: 6270.83 + value: {index: 15, type: Single} + - key: [5056, 6784, 1792] + speed: 6342.39 + value: {index: 15, type: Single} + - key: [5056, 6784, 3328] + speed: 6383.72 + value: {index: 15, type: Single} + - key: [5888, 4, 256] + speed: 290.688 + value: {index: 20, type: Single} + - key: [5888, 4, 768] + speed: 278.111 + value: {index: 20, type: Single} + - key: [5888, 4, 1792] + speed: 167.329 + value: {index: 20, type: Single} + - key: [5888, 4, 3328] + speed: 165.305 + value: {index: 20, type: Single} + - key: [5888, 64, 256] + speed: 2718.78 + value: {index: 17, type: Single} + - key: [5888, 64, 768] + speed: 3310.93 + value: {index: 17, type: Single} + - key: [5888, 64, 1792] + speed: 3522.45 + value: {index: 17, type: Single} + - key: [5888, 64, 3328] + speed: 3552.96 + value: {index: 18, type: Single} + - key: [5888, 128, 256] + speed: 3500.8 + value: {index: 19, type: Single} + - key: [5888, 128, 768] + speed: 4007.07 + value: {index: 18, type: Single} + - key: [5888, 128, 1792] + speed: 4239.05 + value: {index: 18, type: Single} + - key: [5888, 128, 3328] + speed: 4209.31 + value: {index: 18, type: Single} + - key: [5888, 256, 256] + speed: 4994.45 + value: {index: 19, type: Single} + - key: [5888, 256, 768] + speed: 5462.27 + value: {index: 19, type: Single} + - key: [5888, 256, 1792] + speed: 5471.02 + value: {index: 19, type: Single} + - key: [5888, 256, 3328] + speed: 5498.5 + value: {index: 14, type: Single} + - key: [5888, 448, 256] + speed: 4743.02 + value: {index: 19, type: Single} + - key: [5888, 448, 768] + speed: 5115.65 + value: {index: 19, type: Single} + - key: [5888, 448, 1792] + speed: 5185.06 + value: {index: 19, type: Single} + - key: [5888, 448, 3328] + speed: 5185.71 + value: {index: 15, type: Single} + - key: [5888, 704, 256] + speed: 5497.04 + value: {index: 19, type: Single} + - key: [5888, 704, 768] + speed: 5763.93 + value: {index: 19, type: Single} + - key: [5888, 704, 1792] + speed: 5691.73 + value: {index: 19, type: Single} + - key: [5888, 704, 3328] + speed: 5260.4 + value: {index: 14, type: Single} + - key: [5888, 1024, 256] + speed: 5534.32 + value: {index: 15, type: Single} + - key: [5888, 1024, 768] + speed: 5952.79 + value: {index: 15, type: Single} + - key: [5888, 1024, 1792] + speed: 6101.8 + value: {index: 15, type: Single} + - key: [5888, 1024, 3328] + speed: 6149.33 + value: {index: 15, type: Single} + - key: [5888, 1408, 256] + speed: 5757.75 + value: {index: 15, type: Single} + - key: [5888, 1408, 768] + speed: 6192.93 + value: {index: 15, type: Single} + - key: [5888, 1408, 1792] + speed: 6338.62 + value: {index: 15, type: Single} + - key: [5888, 1408, 3328] + speed: 6402.69 + value: {index: 15, type: Single} + - key: [5888, 1856, 256] + speed: 5725.68 + value: {index: 19, type: Single} + - key: [5888, 1856, 768] + speed: 5987.73 + value: {index: 15, type: Single} + - key: [5888, 1856, 1792] + speed: 6133.65 + value: {index: 15, type: Single} + - key: [5888, 1856, 3328] + speed: 6177.23 + value: {index: 15, type: Single} + - key: [5888, 2368, 256] + speed: 5743.15 + value: {index: 19, type: Single} + - key: [5888, 2368, 768] + speed: 6028.4 + value: {index: 15, type: Single} + - key: [5888, 2368, 1792] + speed: 6169.88 + value: {index: 15, type: Single} + - key: [5888, 2368, 3328] + speed: 6222.12 + value: {index: 15, type: Single} + - key: [5888, 2944, 256] + speed: 5884.07 + value: {index: 15, type: Single} + - key: [5888, 2944, 768] + speed: 6214.28 + value: {index: 15, type: Single} + - key: [5888, 2944, 1792] + speed: 6347.19 + value: {index: 15, type: Single} + - key: [5888, 2944, 3328] + speed: 6392.14 + value: {index: 15, type: Single} + - key: [5888, 3584, 256] + speed: 5873.77 + value: {index: 15, type: Single} + - key: [5888, 3584, 768] + speed: 6192.47 + value: {index: 15, type: Single} + - key: [5888, 3584, 1792] + speed: 6281.23 + value: {index: 15, type: Single} + - key: [5888, 3584, 3328] + speed: 6321.79 + value: {index: 15, type: Single} + - key: [5888, 4288, 256] + speed: 5840.68 + value: {index: 15, type: Single} + - key: [5888, 4288, 768] + speed: 6188.66 + value: {index: 15, type: Single} + - key: [5888, 4288, 1792] + speed: 6307.34 + value: {index: 15, type: Single} + - key: [5888, 4288, 3328] + speed: 6352.14 + value: {index: 15, type: Single} + - key: [5888, 5056, 256] + speed: 5913.07 + value: {index: 15, type: Single} + - key: [5888, 5056, 768] + speed: 6273.39 + value: {index: 15, type: Single} + - key: [5888, 5056, 1792] + speed: 6413.6 + value: {index: 15, type: Single} + - key: [5888, 5056, 3328] + speed: 6462.47 + value: {index: 15, type: Single} + - key: [5888, 5888, 256] + speed: 6059.65 + value: {index: 15, type: Single} + - key: [5888, 5888, 768] + speed: 6295.37 + value: {index: 15, type: Single} + - key: [5888, 5888, 1792] + speed: 6400.52 + value: {index: 15, type: Single} + - key: [5888, 5888, 3328] + speed: 6438.44 + value: {index: 15, type: Single} + - key: [5888, 6784, 256] + speed: 6065.45 + value: {index: 15, type: Single} + - key: [5888, 6784, 768] + speed: 6352.77 + value: {index: 15, type: Single} + - key: [5888, 6784, 1792] + speed: 6445.6 + value: {index: 15, type: Single} + - key: [5888, 6784, 3328] + speed: 6474.61 + value: {index: 15, type: Single} + - key: [6784, 4, 256] + speed: 310.528 + value: {index: 20, type: Single} + - key: [6784, 4, 768] + speed: 305.803 + value: {index: 20, type: Single} + - key: [6784, 4, 1792] + speed: 166.318 + value: {index: 20, type: Single} + - key: [6784, 4, 3328] + speed: 160.857 + value: {index: 20, type: Single} + - key: [6784, 64, 256] + speed: 2896.66 + value: {index: 19, type: Single} + - key: [6784, 64, 768] + speed: 3326.99 + value: {index: 17, type: Single} + - key: [6784, 64, 1792] + speed: 3489.47 + value: {index: 17, type: Single} + - key: [6784, 64, 3328] + speed: 3615.6 + value: {index: 18, type: Single} + - key: [6784, 128, 256] + speed: 4011.99 + value: {index: 19, type: Single} + - key: [6784, 128, 768] + speed: 4483.49 + value: {index: 14, type: Single} + - key: [6784, 128, 1792] + speed: 4642.35 + value: {index: 14, type: Single} + - key: [6784, 128, 3328] + speed: 4687.29 + value: {index: 14, type: Single} + - key: [6784, 256, 256] + speed: 4519.52 + value: {index: 19, type: Single} + - key: [6784, 256, 768] + speed: 4876.95 + value: {index: 19, type: Single} + - key: [6784, 256, 1792] + speed: 5008.14 + value: {index: 15, type: Single} + - key: [6784, 256, 3328] + speed: 5050.14 + value: {index: 15, type: Single} + - key: [6784, 448, 256] + speed: 5342.49 + value: {index: 19, type: Single} + - key: [6784, 448, 768] + speed: 5810.43 + value: {index: 19, type: Single} + - key: [6784, 448, 1792] + speed: 5466.0 + value: {index: 19, type: Single} + - key: [6784, 448, 3328] + speed: 4984.23 + value: {index: 19, type: Single} + - key: [6784, 704, 256] + speed: 5236.42 + value: {index: 19, type: Single} + - key: [6784, 704, 768] + speed: 5615.29 + value: {index: 19, type: Single} + - key: [6784, 704, 1792] + speed: 5737.52 + value: {index: 15, type: Single} + - key: [6784, 704, 3328] + speed: 5809.33 + value: {index: 15, type: Single} + - key: [6784, 1024, 256] + speed: 5515.18 + value: {index: 15, type: Single} + - key: [6784, 1024, 768] + speed: 5930.24 + value: {index: 15, type: Single} + - key: [6784, 1024, 1792] + speed: 6059.24 + value: {index: 15, type: Single} + - key: [6784, 1024, 3328] + speed: 6120.54 + value: {index: 15, type: Single} + - key: [6784, 1408, 256] + speed: 5566.68 + value: {index: 19, type: Single} + - key: [6784, 1408, 768] + speed: 5869.91 + value: {index: 19, type: Single} + - key: [6784, 1408, 1792] + speed: 5907.38 + value: {index: 15, type: Single} + - key: [6784, 1408, 3328] + speed: 5948.07 + value: {index: 15, type: Single} + - key: [6784, 1856, 256] + speed: 5637.22 + value: {index: 19, type: Single} + - key: [6784, 1856, 768] + speed: 5928.57 + value: {index: 19, type: Single} + - key: [6784, 1856, 1792] + speed: 6004.62 + value: {index: 15, type: Single} + - key: [6784, 1856, 3328] + speed: 6051.92 + value: {index: 15, type: Single} + - key: [6784, 2368, 256] + speed: 5753.7 + value: {index: 19, type: Single} + - key: [6784, 2368, 768] + speed: 6105.97 + value: {index: 15, type: Single} + - key: [6784, 2368, 1792] + speed: 6235.22 + value: {index: 15, type: Single} + - key: [6784, 2368, 3328] + speed: 6263.04 + value: {index: 15, type: Single} + - key: [6784, 2944, 256] + speed: 5931.75 + value: {index: 15, type: Single} + - key: [6784, 2944, 768] + speed: 6177.55 + value: {index: 15, type: Single} + - key: [6784, 2944, 1792] + speed: 6249.06 + value: {index: 15, type: Single} + - key: [6784, 2944, 3328] + speed: 6290.24 + value: {index: 15, type: Single} + - key: [6784, 3584, 256] + speed: 5981.39 + value: {index: 15, type: Single} + - key: [6784, 3584, 768] + speed: 6278.91 + value: {index: 15, type: Single} + - key: [6784, 3584, 1792] + speed: 6353.41 + value: {index: 15, type: Single} + - key: [6784, 3584, 3328] + speed: 6388.02 + value: {index: 15, type: Single} + - key: [6784, 4288, 256] + speed: 5879.74 + value: {index: 15, type: Single} + - key: [6784, 4288, 768] + speed: 6220.77 + value: {index: 15, type: Single} + - key: [6784, 4288, 1792] + speed: 6297.74 + value: {index: 15, type: Single} + - key: [6784, 4288, 3328] + speed: 6332.74 + value: {index: 15, type: Single} + - key: [6784, 5056, 256] + speed: 5923.61 + value: {index: 15, type: Single} + - key: [6784, 5056, 768] + speed: 6264.31 + value: {index: 15, type: Single} + - key: [6784, 5056, 1792] + speed: 6333.38 + value: {index: 15, type: Single} + - key: [6784, 5056, 3328] + speed: 6374.82 + value: {index: 15, type: Single} + - key: [6784, 5888, 256] + speed: 6122.21 + value: {index: 15, type: Single} + - key: [6784, 5888, 768] + speed: 6383.12 + value: {index: 15, type: Single} + - key: [6784, 5888, 1792] + speed: 6449.64 + value: {index: 15, type: Single} + - key: [6784, 5888, 3328] + speed: 6483.07 + value: {index: 15, type: Single} + - key: [6784, 6784, 256] + speed: 6189.21 + value: {index: 15, type: Single} + - key: [6784, 6784, 768] + speed: 6390.3 + value: {index: 15, type: Single} + - key: [6784, 6784, 1792] + speed: 6517.92 + value: {index: 15, type: Single} + - key: [6784, 6784, 3328] + speed: 6616.49 + value: {index: 15, type: Single} + type: Matching + predicate: + type: And + value: + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + type: Problem + property: {type: OperationIdentifier} + type: ProblemMap + predicate: + type: AMDGPU + value: {type: Processor, value: gfx1010} + type: Hardware +solutions: +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 0 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '2', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'False', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '128', + LSCB: '8', LSPA: '8', LSPB: '64', LVCA: '32', LVCB: '4', LVPA: '2', LVPB: '32', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '3584', LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '512', + LdsOffsetA: '0', LdsOffsetA_Blk: '2048', LdsOffsetB: '1024', LdsOffsetB_Blk: '3072', + LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', + LocalReadVectorWidth: '4', LocalSplitU: '1', LocalWrite2A: 'True', LocalWrite2B: 'True', + LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', + LoopIters: '8', LoopTail: 'True', LoopUnroll: '8', MacroTile0: '128', MacroTile1: '64', + MacroTileA: '128', MacroTileB: '64', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '32', NumGlobalWriteVectorsPerThread: '8', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '0', SolutionNameMin: Cijk_Ailk_Bljk_SB_MT128x64x8_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '16', SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', + SuppressNoLoadLoop: 'True', ThreadTile: '[8, 4]', ThreadTile0: '8', ThreadTile1: '4', + ThreadTileA: '8', ThreadTileB: '4', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[16, 16, 1]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Ailk_Bljk_SB_MT128x64x8_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT8_4_USFGRO0_VW4_WG16_16_1_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 4} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 8, DUorMT1: 64, ShiftPtrElemA: 4, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 64} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [128, 64, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [8, 4] + workGroup: [16, 16, 1] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 1 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '16', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'False', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '64', + LSCB: '16', LSPA: '16', LSPB: '64', LVCA: '16', LVCB: '4', LVPA: '4', LVPB: '16', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '16384', LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '1024', + LdsOffsetA: '0', LdsOffsetA_Blk: '2048', LdsOffsetB: '1024', LdsOffsetB_Blk: '3072', + LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', + LocalReadVectorWidth: '4', LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', + LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', + LoopIters: '4', LoopTail: 'True', LoopUnroll: '4', MacroTile0: '64', MacroTile1: '64', + MacroTileA: '64', MacroTileB: '64', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '16', NumGlobalWriteVectorsPerThread: '4', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '1', SolutionNameMin: Cijk_Ailk_Bljk_SB_MT64x64x16_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '8', SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', + SuppressNoLoadLoop: 'True', ThreadTile: '[8, 8]', ThreadTile0: '8', ThreadTile1: '8', + ThreadTileA: '8', ThreadTileB: '8', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[8, 8, 4]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '2'} + name: Cijk_Ailk_Bljk_SB_MT64x64x16_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT8_8_USFGRO0_VW4_WG8_8_4_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 4} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 16, DUorMT1: 64, ShiftPtrElemA: 4, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 64} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 16 + globalAccumulation: false + globalSplitU: 1 + macroTile: [64, 64, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 2 + staggerU: 32 + threadTile: [8, 8] + workGroup: [8, 8, 4] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 2 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'False', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '128', + LSCB: '8', LSPA: '8', LSPB: '128', LVCA: '32', LVCB: '2', LVPA: '2', LVPB: '32', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '4096', LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '1024', + LdsOffsetA: '0', LdsOffsetA_Blk: '2048', LdsOffsetB: '1024', LdsOffsetB_Blk: '3072', + LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', + LocalReadVectorWidth: '4', LocalSplitU: '1', LocalWrite2A: 'True', LocalWrite2B: 'True', + LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', + LoopIters: '8', LoopTail: 'True', LoopUnroll: '8', MacroTile0: '128', MacroTile1: '128', + MacroTileA: '128', MacroTileB: '128', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '64', NumGlobalWriteVectorsPerThread: '16', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '2', SolutionNameMin: Cijk_Ailk_Bljk_SB_MT128x128x8_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '16', SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', + SuppressNoLoadLoop: 'True', ThreadTile: '[8, 8]', ThreadTile0: '8', ThreadTile1: '8', + ThreadTileA: '8', ThreadTileB: '8', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[16, 16, 1]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Ailk_Bljk_SB_MT128x128x8_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT8_8_USFGRO0_VW4_WG16_16_1_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 4} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 8, DUorMT1: 128, ShiftPtrElemA: 4, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 128} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [128, 128, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [8, 8] + workGroup: [16, 16, 1] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 3 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '16', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '2', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'False', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '64', + LSCB: '16', LSPA: '16', LSPB: '32', LVCA: '16', LVCB: '8', LVPA: '4', LVPB: '16', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '8192', LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '512', + LdsOffsetA: '0', LdsOffsetA_Blk: '2048', LdsOffsetB: '1024', LdsOffsetB_Blk: '3072', + LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', + LocalReadVectorWidth: '4', LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', + LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', + LoopIters: '4', LoopTail: 'True', LoopUnroll: '4', MacroTile0: '64', MacroTile1: '32', + MacroTileA: '64', MacroTileB: '32', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '8', NumGlobalWriteVectorsPerThread: '2', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '3', SolutionNameMin: Cijk_Ailk_Bljk_SB_MT64x32x16_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '8', SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', + SuppressNoLoadLoop: 'True', ThreadTile: '[8, 4]', ThreadTile0: '8', ThreadTile1: '4', + ThreadTileA: '8', ThreadTileB: '4', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[8, 8, 4]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '2'} + name: Cijk_Ailk_Bljk_SB_MT64x32x16_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT8_4_USFGRO0_VW4_WG8_8_4_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 4} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 16, DUorMT1: 32, ShiftPtrElemA: 4, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 32} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 16 + globalAccumulation: false + globalSplitU: 1 + macroTile: [64, 32, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 2 + staggerU: 32 + threadTile: [8, 4] + workGroup: [8, 8, 4] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 4 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '1', GlobalLoadVectorWidthB: '1', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '32', LSCB: '8', LSPA: '8', + LSPB: '32', LVCA: '32', LVCB: '8', LVPA: '8', LVPB: '32', LdcEqualsLdd: 'True', + LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '4096', + LdsNumElementsAlignedA: '256', LdsNumElementsAlignedB: '256', LdsOffsetA: '0', + LdsOffsetA_Blk: '512', LdsOffsetB: '256', LdsOffsetB_Blk: '768', LdsPadA: '0', + LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '2', LoopTail: 'True', + LoopUnroll: '2', MacroTile0: '32', MacroTile1: '32', MacroTileA: '32', MacroTileB: '32', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '4', + NumGlobalWriteVectorsPerThread: '1', NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'True', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '4', + SolutionNameMin: Cijk_Ailk_Bljk_SB_MT32x32x8_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '8', + SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', SuppressNoLoadLoop: 'True', ThreadTile: '[4, + 4]', ThreadTile0: '4', ThreadTile1: '4', ThreadTileA: '4', ThreadTileB: '4', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: '1', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[8, 8, + 4]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Ailk_Bljk_SB_MT32x32x8_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT4_4_USFGRO1_VW4_WG8_8_4_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 8, DUorMT1: 32, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 32} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [32, 32, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [4, 4] + workGroup: [8, 8, 4] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 5 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '16', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'False', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '64', + LSCB: '16', LSPA: '16', LSPB: '64', LVCA: '16', LVCB: '4', LVPA: '4', LVPB: '16', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '7168', LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '2048', + LdsOffsetA: '0', LdsOffsetA_Blk: '4096', LdsOffsetB: '1024', LdsOffsetB_Blk: '5120', + LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', + LocalReadVectorWidth: '4', LocalSplitU: '1', LocalWrite2A: 'True', LocalWrite2B: 'True', + LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', + LoopIters: '16', LoopTail: 'True', LoopUnroll: '16', MacroTile0: '64', MacroTile1: '128', + MacroTileA: '64', MacroTileB: '128', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '32', NumGlobalWriteVectorsPerThread: '8', + NumLoadsA: '1', NumLoadsB: '2', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '2', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '5', SolutionNameMin: Cijk_Ailk_Bljk_SB_MT64x128x16_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '16', SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', + SuppressNoLoadLoop: 'True', ThreadTile: '[4, 8]', ThreadTile0: '4', ThreadTile1: '8', + ThreadTileA: '4', ThreadTileB: '8', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[16, 16, 1]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '2'} + name: Cijk_Ailk_Bljk_SB_MT64x128x16_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT4_8_USFGRO0_VW4_WG16_16_1_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 4} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 16, DUorMT1: 128, ShiftPtrElemA: 4, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 128} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 16 + globalAccumulation: false + globalSplitU: 1 + macroTile: [64, 128, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 2 + staggerU: 32 + threadTile: [4, 8] + workGroup: [16, 16, 1] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 6 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BufferLoad: 'True', BufferStore: 'True', + CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '1', GlobalLoadVectorWidthB: '1', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '2', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '1', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '8', LSCB: '8', LSPA: '8', + LSPB: '8', LVCA: '8', LVCB: '8', LVPA: '8', LVPB: '8', LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', + LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '409', LdsNumElementsAlignedA: '64', + LdsNumElementsAlignedB: '64', LdsOffsetA: '0', LdsOffsetA_Blk: '128', LdsOffsetB: '64', + LdsOffsetB_Blk: '192', LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', + LocalRead2B: 'True', LocalReadVectorWidth: '2', LocalSplitU: '4', LocalWrite2A: 'True', + LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', + LoopDoWhile: 'False', LoopIters: '2', LoopTail: 'True', LoopUnroll: '2', MacroTile0: '8', + MacroTile1: '8', MacroTileA: '8', MacroTileB: '8', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '1', NumGlobalWriteVectorsPerThread: '1', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '64', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '6', SolutionNameMin: Cijk_Ailk_Bljk_SB_MT8x8x8_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '4', SubGroup1: '4', SubGroupA: '4', SubGroupB: '4', + SuppressNoLoadLoop: 'True', ThreadTile: '[2, 2]', ThreadTile0: '2', ThreadTile1: '2', + ThreadTileA: '2', ThreadTileB: '2', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: '1', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '2', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[4, 4, 4]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Ailk_Bljk_SB_MT8x8x8_SE_EPS1_GRVW2_K1_LRVW2_PGR1_SNLL1_TT2_2_USFGRO1_VW2_WG4_4_4_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 8, DUorMT1: 8, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 8} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [8, 8, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [2, 2] + workGroup: [4, 4, 4] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 7 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '16', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '2', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'False', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '16', + LSCB: '64', LSPA: '32', LSPB: '16', LVCA: '8', LVCB: '16', LVPA: '16', LVPB: '4', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '8192', LdsNumElementsAlignedA: '512', LdsNumElementsAlignedB: '1024', + LdsOffsetA: '0', LdsOffsetA_Blk: '2048', LdsOffsetB: '512', LdsOffsetB_Blk: '2560', + LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', + LocalReadVectorWidth: '4', LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', + LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', + LoopIters: '4', LoopTail: 'True', LoopUnroll: '4', MacroTile0: '32', MacroTile1: '64', + MacroTileA: '32', MacroTileB: '64', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '8', NumGlobalWriteVectorsPerThread: '2', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '0', SolutionNameMin: Cijk_Alik_Bjlk_SB_MT32x64x16_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '8', SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', + SuppressNoLoadLoop: 'True', ThreadTile: '[4, 8]', ThreadTile0: '4', ThreadTile1: '8', + ThreadTileA: '4', ThreadTileB: '8', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[8, 8, 4]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '2'} + name: Cijk_Alik_Bjlk_SB_MT32x64x16_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT4_8_USFGRO0_VW4_WG8_8_4_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 4} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 32, DUorMT1: 16, ShiftPtrElemA: 0, ShiftPtrElemB: 4} + - {type: BufferStoreOffsetLimitCheck, value: 64} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 16 + globalAccumulation: false + globalSplitU: 1 + macroTile: [32, 64, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 2 + staggerU: 32 + threadTile: [4, 8] + workGroup: [8, 8, 4] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 8 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'False', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '8', LSCB: '128', + LSPA: '128', LSPB: '8', LVCA: '2', LVCB: '32', LVPA: '32', LVPB: '2', LdcEqualsLdd: 'True', + LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '4096', + LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '1024', LdsOffsetA: '0', + LdsOffsetA_Blk: '2048', LdsOffsetB: '1024', LdsOffsetB_Blk: '3072', LdsPadA: '0', + LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '1', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '8', LoopTail: 'True', + LoopUnroll: '8', MacroTile0: '128', MacroTile1: '128', MacroTileA: '128', MacroTileB: '128', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '64', + NumGlobalWriteVectorsPerThread: '16', NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'True', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '1', + SolutionNameMin: Cijk_Alik_Bjlk_SB_MT128x128x8_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '16', + SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', SuppressNoLoadLoop: 'True', + ThreadTile: '[8, 8]', ThreadTile0: '8', ThreadTile1: '8', ThreadTileA: '8', ThreadTileB: '8', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[16, 16, + 1]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Alik_Bjlk_SB_MT128x128x8_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT8_8_USFGRO0_VW4_WG16_16_1_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 4} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 128, DUorMT1: 8, ShiftPtrElemA: 0, ShiftPtrElemB: 4} + - {type: BufferStoreOffsetLimitCheck, value: 128} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [128, 128, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [8, 8] + workGroup: [16, 16, 1] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 9 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '16', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'False', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '16', + LSCB: '64', LSPA: '64', LSPB: '16', LVCA: '4', LVCB: '16', LVPA: '16', LVPB: '4', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '16384', LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '1024', + LdsOffsetA: '0', LdsOffsetA_Blk: '2048', LdsOffsetB: '1024', LdsOffsetB_Blk: '3072', + LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', + LocalReadVectorWidth: '4', LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', + LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', + LoopIters: '4', LoopTail: 'True', LoopUnroll: '4', MacroTile0: '64', MacroTile1: '64', + MacroTileA: '64', MacroTileB: '64', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '16', NumGlobalWriteVectorsPerThread: '4', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '2', SolutionNameMin: Cijk_Alik_Bjlk_SB_MT64x64x16_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '8', SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', + SuppressNoLoadLoop: 'True', ThreadTile: '[8, 8]', ThreadTile0: '8', ThreadTile1: '8', + ThreadTileA: '8', ThreadTileB: '8', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[8, 8, 4]', WorkGroupMapping: '4', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '2'} + name: Cijk_Alik_Bjlk_SB_MT64x64x16_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT8_8_USFGRO0_VW4_WG8_8_4_WGM4 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 4} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 64, DUorMT1: 16, ShiftPtrElemA: 0, ShiftPtrElemB: 4} + - {type: BufferStoreOffsetLimitCheck, value: 64} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 16 + globalAccumulation: false + globalSplitU: 1 + macroTile: [64, 64, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 2 + staggerU: 32 + threadTile: [8, 8] + workGroup: [8, 8, 4] + workGroupMapping: 4 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 10 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '16', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'False', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '16', + LSCB: '128', LSPA: '64', LSPB: '8', LVCA: '4', LVCB: '32', LVPA: '16', LVPB: '2', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '7168', LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '2048', + LdsOffsetA: '0', LdsOffsetA_Blk: '4096', LdsOffsetB: '1024', LdsOffsetB_Blk: '5120', + LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', + LocalReadVectorWidth: '4', LocalSplitU: '1', LocalWrite2A: 'True', LocalWrite2B: 'True', + LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', + LoopIters: '16', LoopTail: 'True', LoopUnroll: '16', MacroTile0: '64', MacroTile1: '128', + MacroTileA: '64', MacroTileB: '128', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '32', NumGlobalWriteVectorsPerThread: '8', + NumLoadsA: '1', NumLoadsB: '2', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '2', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '3', SolutionNameMin: Cijk_Alik_Bjlk_SB_MT64x128x16_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '16', SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', + SuppressNoLoadLoop: 'True', ThreadTile: '[4, 8]', ThreadTile0: '4', ThreadTile1: '8', + ThreadTileA: '4', ThreadTileB: '8', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[16, 16, 1]', WorkGroupMapping: '4', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '2'} + name: Cijk_Alik_Bjlk_SB_MT64x128x16_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT4_8_USFGRO0_VW4_WG16_16_1_WGM4 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 4} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 64, DUorMT1: 16, ShiftPtrElemA: 0, ShiftPtrElemB: 4} + - {type: BufferStoreOffsetLimitCheck, value: 128} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 16 + globalAccumulation: false + globalSplitU: 1 + macroTile: [64, 128, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 2 + staggerU: 32 + threadTile: [4, 8] + workGroup: [16, 16, 1] + workGroupMapping: 4 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 11 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '1', GlobalLoadVectorWidthB: '1', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '8', LSCB: '32', LSPA: '32', + LSPB: '8', LVCA: '8', LVCB: '32', LVPA: '32', LVPB: '8', LdcEqualsLdd: 'True', + LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '4096', + LdsNumElementsAlignedA: '256', LdsNumElementsAlignedB: '256', LdsOffsetA: '0', + LdsOffsetA_Blk: '512', LdsOffsetB: '256', LdsOffsetB_Blk: '768', LdsPadA: '0', + LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '2', LoopTail: 'True', + LoopUnroll: '2', MacroTile0: '32', MacroTile1: '32', MacroTileA: '32', MacroTileB: '32', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '4', + NumGlobalWriteVectorsPerThread: '1', NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'True', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '4', + SolutionNameMin: Cijk_Alik_Bjlk_SB_MT32x32x8_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '8', + SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', SuppressNoLoadLoop: 'True', ThreadTile: '[4, + 4]', ThreadTile0: '4', ThreadTile1: '4', ThreadTileA: '4', ThreadTileB: '4', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: '1', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[8, 8, + 4]', WorkGroupMapping: '4', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Alik_Bjlk_SB_MT32x32x8_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT4_4_USFGRO1_VW4_WG8_8_4_WGM4 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 32, DUorMT1: 8, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 32} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [32, 32, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [4, 4] + workGroup: [8, 8, 4] + workGroupMapping: 4 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 12 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BufferLoad: 'True', BufferStore: 'True', + CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '1', GlobalLoadVectorWidthB: '1', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '2', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '1', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '8', LSCB: '8', LSPA: '8', + LSPB: '8', LVCA: '8', LVCB: '8', LVPA: '8', LVPB: '8', LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', + LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '409', LdsNumElementsAlignedA: '64', + LdsNumElementsAlignedB: '64', LdsOffsetA: '0', LdsOffsetA_Blk: '128', LdsOffsetB: '64', + LdsOffsetB_Blk: '192', LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', + LocalRead2B: 'True', LocalReadVectorWidth: '2', LocalSplitU: '4', LocalWrite2A: 'True', + LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', + LoopDoWhile: 'False', LoopIters: '2', LoopTail: 'True', LoopUnroll: '2', MacroTile0: '8', + MacroTile1: '8', MacroTileA: '8', MacroTileB: '8', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '1', NumGlobalWriteVectorsPerThread: '1', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '64', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '5', SolutionNameMin: Cijk_Alik_Bjlk_SB_MT8x8x8_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '4', SubGroup1: '4', SubGroupA: '4', SubGroupB: '4', + SuppressNoLoadLoop: 'True', ThreadTile: '[2, 2]', ThreadTile0: '2', ThreadTile1: '2', + ThreadTileA: '2', ThreadTileB: '2', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: '1', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '2', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[4, 4, 4]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Alik_Bjlk_SB_MT8x8x8_SE_EPS1_GRVW2_K1_LRVW2_PGR1_SNLL1_TT2_2_USFGRO1_VW2_WG4_4_4_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 8, DUorMT1: 8, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 8} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [8, 8, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [2, 2] + workGroup: [4, 4, 4] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 13 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'False', + FractionalLoad: '0', GlobalLoadVectorWidthA: '2', GlobalLoadVectorWidthB: '2', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '8', LSCB: '8', LSPA: '64', + LSPB: '64', LVCA: '4', LVCB: '4', LVPA: '32', LVPB: '32', LdcEqualsLdd: 'True', + LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '1024', + LdsOffsetA: '0', LdsOffsetB: '512', LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', + LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', LocalSplitU: '1', + LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', + LoopDoWhile: 'False', LoopIters: '8', LoopTail: 'True', LoopUnroll: '8', MacroTile0: '64', + MacroTile1: '64', MacroTileA: '64', MacroTileB: '64', MacroTileShapeMax: '64', + MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', + MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', + NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '16', NumGlobalWriteVectorsPerThread: '4', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'False', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '0', SolutionNameMin: Cijk_Alik_Bljk_SB_MT64x64x8_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '16', SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', + SuppressNoLoadLoop: 'False', ThreadTile: '[4, 4]', ThreadTile0: '4', ThreadTile1: '4', + ThreadTileA: '4', ThreadTileB: '4', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: '1', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[16, 16, 1]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Alik_Bljk_SB_MT64x64x8_SE_EPS0_GRVW4_K1_LRVW4_PGR0_SNLL0_TT4_4_USFGRO1_VW4_WG16_16_1_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 64, DUorMT1: 64, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 64} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [64, 64, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [4, 4] + workGroup: [16, 16, 1] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 14 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '16', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '16', LSCB: '16', LSPA: '64', + LSPB: '64', LVCA: '4', LVCB: '4', LVPA: '16', LVPB: '16', LdcEqualsLdd: 'True', + LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '7168', + LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '2048', LdsOffsetA: '0', + LdsOffsetA_Blk: '4096', LdsOffsetB: '1024', LdsOffsetB_Blk: '5120', LdsPadA: '0', + LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '1', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '16', LoopTail: 'True', + LoopUnroll: '16', MacroTile0: '64', MacroTile1: '128', MacroTileA: '64', MacroTileB: '128', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '32', + NumGlobalWriteVectorsPerThread: '8', NumLoadsA: '1', NumLoadsB: '2', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '2', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'True', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '1', + SolutionNameMin: Cijk_Alik_Bljk_SB_MT64x128x16_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '16', + SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', SuppressNoLoadLoop: 'True', + ThreadTile: '[4, 8]', ThreadTile0: '4', ThreadTile1: '8', ThreadTileA: '4', ThreadTileB: '8', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: '1', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[16, 16, + 1]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '2'} + name: Cijk_Alik_Bljk_SB_MT64x128x16_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT4_8_USFGRO1_VW4_WG16_16_1_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 64, DUorMT1: 128, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 128} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 16 + globalAccumulation: false + globalSplitU: 1 + macroTile: [64, 128, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 2 + staggerU: 32 + threadTile: [4, 8] + workGroup: [16, 16, 1] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 15 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '8', LSCB: '8', LSPA: '128', + LSPB: '128', LVCA: '2', LVCB: '2', LVPA: '32', LVPB: '32', LdcEqualsLdd: 'True', + LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '4096', + LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '1024', LdsOffsetA: '0', + LdsOffsetA_Blk: '2048', LdsOffsetB: '1024', LdsOffsetB_Blk: '3072', LdsPadA: '0', + LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '1', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '8', LoopTail: 'True', + LoopUnroll: '8', MacroTile0: '128', MacroTile1: '128', MacroTileA: '128', MacroTileB: '128', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '64', + NumGlobalWriteVectorsPerThread: '16', NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'True', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '2', + SolutionNameMin: Cijk_Alik_Bljk_SB_MT128x128x8_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '16', + SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', SuppressNoLoadLoop: 'True', + ThreadTile: '[8, 8]', ThreadTile0: '8', ThreadTile1: '8', ThreadTileA: '8', ThreadTileB: '8', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: '1', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[16, 16, + 1]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Alik_Bljk_SB_MT128x128x8_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT8_8_USFGRO1_VW4_WG16_16_1_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 128, DUorMT1: 128, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 128} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [128, 128, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [8, 8] + workGroup: [16, 16, 1] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 16 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '1', GlobalLoadVectorWidthB: '1', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '8', LSCB: '8', LSPA: '32', + LSPB: '32', LVCA: '8', LVCB: '8', LVPA: '32', LVPB: '32', LdcEqualsLdd: 'True', + LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '4096', + LdsNumElementsAlignedA: '256', LdsNumElementsAlignedB: '256', LdsOffsetA: '0', + LdsOffsetA_Blk: '512', LdsOffsetB: '256', LdsOffsetB_Blk: '768', LdsPadA: '0', + LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '2', LoopTail: 'True', + LoopUnroll: '2', MacroTile0: '32', MacroTile1: '32', MacroTileA: '32', MacroTileB: '32', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '4', + NumGlobalWriteVectorsPerThread: '1', NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'True', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '3', + SolutionNameMin: Cijk_Alik_Bljk_SB_MT32x32x8_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '8', + SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', SuppressNoLoadLoop: 'True', ThreadTile: '[4, + 4]', ThreadTile0: '4', ThreadTile1: '4', ThreadTileA: '4', ThreadTileB: '4', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: '1', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[8, 8, + 4]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Alik_Bljk_SB_MT32x32x8_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT4_4_USFGRO1_VW4_WG8_8_4_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 32, DUorMT1: 32, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 32} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [32, 32, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [4, 4] + workGroup: [8, 8, 4] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 17 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '16', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '2', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '16', LSCB: '16', LSPA: '32', + LSPB: '64', LVCA: '8', LVCB: '4', LVPA: '16', LVPB: '16', LdcEqualsLdd: 'True', + LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '8192', + LdsNumElementsAlignedA: '512', LdsNumElementsAlignedB: '1024', LdsOffsetA: '0', + LdsOffsetA_Blk: '2048', LdsOffsetB: '512', LdsOffsetB_Blk: '2560', LdsPadA: '0', + LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '4', LoopTail: 'True', + LoopUnroll: '4', MacroTile0: '32', MacroTile1: '64', MacroTileA: '32', MacroTileB: '64', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '8', + NumGlobalWriteVectorsPerThread: '2', NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'True', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '4', + SolutionNameMin: Cijk_Alik_Bljk_SB_MT32x64x16_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '8', + SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', SuppressNoLoadLoop: 'True', ThreadTile: '[4, + 8]', ThreadTile0: '4', ThreadTile1: '8', ThreadTileA: '4', ThreadTileB: '8', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: '1', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[8, 8, + 4]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '2'} + name: Cijk_Alik_Bljk_SB_MT32x64x16_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT4_8_USFGRO1_VW4_WG8_8_4_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 32, DUorMT1: 64, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 64} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 16 + globalAccumulation: false + globalSplitU: 1 + macroTile: [32, 64, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 2 + staggerU: 32 + threadTile: [4, 8] + workGroup: [8, 8, 4] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 18 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '16', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '2', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '16', LSCB: '16', LSPA: '64', + LSPB: '32', LVCA: '4', LVCB: '8', LVPA: '16', LVPB: '16', LdcEqualsLdd: 'True', + LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '8192', + LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '512', LdsOffsetA: '0', + LdsOffsetA_Blk: '2048', LdsOffsetB: '1024', LdsOffsetB_Blk: '3072', LdsPadA: '0', + LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '4', LoopTail: 'True', + LoopUnroll: '4', MacroTile0: '64', MacroTile1: '32', MacroTileA: '64', MacroTileB: '32', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '8', + NumGlobalWriteVectorsPerThread: '2', NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'True', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '5', + SolutionNameMin: Cijk_Alik_Bljk_SB_MT64x32x16_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '8', + SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', SuppressNoLoadLoop: 'True', ThreadTile: '[8, + 4]', ThreadTile0: '8', ThreadTile1: '4', ThreadTileA: '8', ThreadTileB: '4', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: '1', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[8, 8, + 4]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '2'} + name: Cijk_Alik_Bljk_SB_MT64x32x16_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT8_4_USFGRO1_VW4_WG8_8_4_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 64, DUorMT1: 32, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 32} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 16 + globalAccumulation: false + globalSplitU: 1 + macroTile: [64, 32, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 2 + staggerU: 32 + threadTile: [8, 4] + workGroup: [8, 8, 4] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 19 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '2', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '8', LSCB: '8', LSPA: '128', + LSPB: '64', LVCA: '2', LVCB: '4', LVPA: '32', LVPB: '32', LdcEqualsLdd: 'True', + LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '3584', + LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '512', LdsOffsetA: '0', + LdsOffsetA_Blk: '2048', LdsOffsetB: '1024', LdsOffsetB_Blk: '3072', LdsPadA: '0', + LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '1', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '8', LoopTail: 'True', + LoopUnroll: '8', MacroTile0: '128', MacroTile1: '64', MacroTileA: '128', MacroTileB: '64', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '32', + NumGlobalWriteVectorsPerThread: '8', NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'True', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '6', + SolutionNameMin: Cijk_Alik_Bljk_SB_MT128x64x8_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '16', + SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', SuppressNoLoadLoop: 'True', + ThreadTile: '[8, 4]', ThreadTile0: '8', ThreadTile1: '4', ThreadTileA: '8', ThreadTileB: '4', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: '1', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[16, 16, + 1]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Alik_Bljk_SB_MT128x64x8_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT8_4_USFGRO1_VW4_WG16_16_1_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 128, DUorMT1: 64, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 64} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [128, 64, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [8, 4] + workGroup: [16, 16, 1] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 20 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BufferLoad: 'True', BufferStore: 'True', + CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '1', GlobalLoadVectorWidthB: '1', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '2', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '1', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '8', LSCB: '8', LSPA: '8', + LSPB: '8', LVCA: '8', LVCB: '8', LVPA: '8', LVPB: '8', LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', + LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '409', LdsNumElementsAlignedA: '64', + LdsNumElementsAlignedB: '64', LdsOffsetA: '0', LdsOffsetA_Blk: '128', LdsOffsetB: '64', + LdsOffsetB_Blk: '192', LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', + LocalRead2B: 'True', LocalReadVectorWidth: '2', LocalSplitU: '4', LocalWrite2A: 'True', + LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', + LoopDoWhile: 'False', LoopIters: '2', LoopTail: 'True', LoopUnroll: '2', MacroTile0: '8', + MacroTile1: '8', MacroTileA: '8', MacroTileB: '8', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '1', NumGlobalWriteVectorsPerThread: '1', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '64', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '7', SolutionNameMin: Cijk_Alik_Bljk_SB_MT8x8x8_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '4', SubGroup1: '4', SubGroupA: '4', SubGroupB: '4', + SuppressNoLoadLoop: 'True', ThreadTile: '[2, 2]', ThreadTile0: '2', ThreadTile1: '2', + ThreadTileA: '2', ThreadTileB: '2', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: '1', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '2', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[4, 4, 4]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Alik_Bljk_SB_MT8x8x8_SE_EPS1_GRVW2_K1_LRVW2_PGR1_SNLL1_TT2_2_USFGRO1_VW2_WG4_4_4_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Alik_Bljk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 8, DUorMT1: 8, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 8} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Alik_Bljk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [8, 8, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [2, 2] + workGroup: [4, 4, 4] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 21 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '16', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'False', GuaranteeNoPartialB: 'False', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '64', + LSCB: '128', LSPA: '16', LSPB: '8', LVCA: '16', LVCB: '32', LVPA: '4', LVPB: '2', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '7168', LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '2048', + LdsOffsetA: '0', LdsOffsetA_Blk: '4096', LdsOffsetB: '1024', LdsOffsetB_Blk: '5120', + LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', + LocalReadVectorWidth: '4', LocalSplitU: '1', LocalWrite2A: 'True', LocalWrite2B: 'True', + LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', + LoopIters: '16', LoopTail: 'True', LoopUnroll: '16', MacroTile0: '64', MacroTile1: '128', + MacroTileA: '64', MacroTileB: '128', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '32', NumGlobalWriteVectorsPerThread: '8', + NumLoadsA: '1', NumLoadsB: '2', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '2', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '0', SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT64x128x16_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '16', SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', + SuppressNoLoadLoop: 'True', ThreadTile: '[4, 8]', ThreadTile0: '4', ThreadTile1: '8', + ThreadTileA: '4', ThreadTileB: '8', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[16, 16, 1]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '2'} + name: Cijk_Ailk_Bjlk_SB_MT64x128x16_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT4_8_USFGRO0_VW4_WG16_16_1_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 4} + - {type: LeadingFree1SizesGreaterOrEqual, value: 4} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 16, DUorMT1: 16, ShiftPtrElemA: 4, ShiftPtrElemB: 4} + - {type: BufferStoreOffsetLimitCheck, value: 128} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 16 + globalAccumulation: false + globalSplitU: 1 + macroTile: [64, 128, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 2 + staggerU: 32 + threadTile: [4, 8] + workGroup: [16, 16, 1] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 22 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '32', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'False', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'False', GuaranteeNoPartialB: 'False', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '64', + LSCB: '32', LSPA: '16', LSPB: '32', LVCA: '16', LVCB: '8', LVPA: '4', LVPB: '8', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '8192', LdsOffsetA: '0', LdsOffsetB: '2048', LdsPadA: '0', LdsPadB: '0', + LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '8', LoopTail: 'True', + LoopUnroll: '8', MacroTile0: '64', MacroTile1: '32', MacroTileA: '64', MacroTileB: '32', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '8', + NumGlobalWriteVectorsPerThread: '2', NumLoadsA: '2', NumLoadsB: '1', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '2', NumLoadsPerpendicularB: '1', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'False', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '1', + SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT64x32x32_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '8', + SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', SuppressNoLoadLoop: 'False', ThreadTile: '[8, + 4]', ThreadTile0: '8', ThreadTile1: '4', ThreadTileA: '8', ThreadTileB: '4', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[8, 8, + 4]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '1'} + name: Cijk_Ailk_Bjlk_SB_MT64x32x32_SE_EPS0_GRVW4_K1_LRVW4_PGR0_SNLL0_TT8_4_USFGRO0_VW4_WG8_8_4_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 4} + - {type: LeadingFree1SizesGreaterOrEqual, value: 4} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 32, DUorMT1: 32, ShiftPtrElemA: 4, ShiftPtrElemB: 4} + - {type: BufferStoreOffsetLimitCheck, value: 32} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 32 + globalAccumulation: false + globalSplitU: 1 + macroTile: [64, 32, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 1 + staggerU: 32 + threadTile: [8, 4] + workGroup: [8, 8, 4] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 23 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '32', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'False', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'False', GuaranteeNoPartialB: 'False', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '32', + LSCB: '64', LSPA: '32', LSPB: '16', LVCA: '8', LVCB: '16', LVPA: '8', LVPB: '4', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '8192', LdsOffsetA: '0', LdsOffsetB: '1024', LdsPadA: '0', LdsPadB: '0', + LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '8', LoopTail: 'True', + LoopUnroll: '8', MacroTile0: '32', MacroTile1: '64', MacroTileA: '32', MacroTileB: '64', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '8', + NumGlobalWriteVectorsPerThread: '2', NumLoadsA: '1', NumLoadsB: '2', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '2', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'False', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '2', + SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT32x64x32_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '8', + SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', SuppressNoLoadLoop: 'False', ThreadTile: '[4, + 8]', ThreadTile0: '4', ThreadTile1: '8', ThreadTileA: '4', ThreadTileB: '8', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[8, 8, + 4]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '1'} + name: Cijk_Ailk_Bjlk_SB_MT32x64x32_SE_EPS0_GRVW4_K1_LRVW4_PGR0_SNLL0_TT4_8_USFGRO0_VW4_WG8_8_4_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 4} + - {type: LeadingFree1SizesGreaterOrEqual, value: 4} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 32, DUorMT1: 32, ShiftPtrElemA: 4, ShiftPtrElemB: 4} + - {type: BufferStoreOffsetLimitCheck, value: 64} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 32 + globalAccumulation: false + globalSplitU: 1 + macroTile: [32, 64, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 1 + staggerU: 32 + threadTile: [4, 8] + workGroup: [8, 8, 4] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 24 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '16', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'False', GuaranteeNoPartialB: 'False', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '64', + LSCB: '64', LSPA: '16', LSPB: '16', LVCA: '16', LVCB: '16', LVPA: '4', LVPB: '4', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '4096', LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '1024', + LdsOffsetA: '0', LdsOffsetA_Blk: '2048', LdsOffsetB: '1024', LdsOffsetB_Blk: '3072', + LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', + LocalReadVectorWidth: '4', LocalSplitU: '1', LocalWrite2A: 'True', LocalWrite2B: 'True', + LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', + LoopIters: '16', LoopTail: 'True', LoopUnroll: '16', MacroTile0: '64', MacroTile1: '64', + MacroTileA: '64', MacroTileB: '64', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '16', NumGlobalWriteVectorsPerThread: '4', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '3', SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT64x64x16_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '16', SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', + SuppressNoLoadLoop: 'True', ThreadTile: '[4, 4]', ThreadTile0: '4', ThreadTile1: '4', + ThreadTileA: '4', ThreadTileB: '4', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[16, 16, 1]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '2'} + name: Cijk_Ailk_Bjlk_SB_MT64x64x16_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT4_4_USFGRO0_VW4_WG16_16_1_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 4} + - {type: LeadingFree1SizesGreaterOrEqual, value: 4} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 16, DUorMT1: 16, ShiftPtrElemA: 4, ShiftPtrElemB: 4} + - {type: BufferStoreOffsetLimitCheck, value: 64} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 16 + globalAccumulation: false + globalSplitU: 1 + macroTile: [64, 64, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 2 + staggerU: 32 + threadTile: [4, 4] + workGroup: [16, 16, 1] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 25 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '4', GlobalLoadVectorWidthB: '4', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'False', GuaranteeNoPartialB: 'False', ISA: '(10, 1, 0)', + InnerUnroll: '1', InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '128', + LSCB: '128', LSPA: '8', LSPB: '8', LVCA: '32', LVCB: '32', LVPA: '2', LVPB: '2', + LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', + LdsNumElements: '4096', LdsNumElementsAlignedA: '1024', LdsNumElementsAlignedB: '1024', + LdsOffsetA: '0', LdsOffsetA_Blk: '2048', LdsOffsetB: '1024', LdsOffsetB_Blk: '3072', + LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', + LocalReadVectorWidth: '4', LocalSplitU: '1', LocalWrite2A: 'True', LocalWrite2B: 'True', + LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', + LoopIters: '8', LoopTail: 'True', LoopUnroll: '8', MacroTile0: '128', MacroTile1: '128', + MacroTileA: '128', MacroTileB: '128', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '64', NumGlobalWriteVectorsPerThread: '16', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '256', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '4', SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT128x128x8_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '16', SubGroup1: '16', SubGroupA: '16', SubGroupB: '16', + SuppressNoLoadLoop: 'True', ThreadTile: '[8, 8]', ThreadTile0: '8', ThreadTile1: '8', + ThreadTileA: '8', ThreadTileB: '8', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: 'False', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '4', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[16, 16, 1]', WorkGroupMapping: '8', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: 'False', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Ailk_Bjlk_SB_MT128x128x8_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT8_8_USFGRO0_VW4_WG16_16_1_WGM8 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 4} + - {type: LeadingFree1SizesGreaterOrEqual, value: 4} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 8, DUorMT1: 8, ShiftPtrElemA: 4, ShiftPtrElemB: 4} + - {type: BufferStoreOffsetLimitCheck, value: 128} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [128, 128, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [8, 8] + workGroup: [16, 16, 1] + workGroupMapping: 8 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 26 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BenchmarkFork: '0', BufferLoad: 'True', + BufferStore: 'True', CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '1', GlobalLoadVectorWidthB: '1', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '4', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '4', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '32', LSCB: '32', LSPA: '8', + LSPB: '8', LVCA: '32', LVCB: '32', LVPA: '8', LVPB: '8', LdcEqualsLdd: 'True', + LdsBlockSizePerPad: '0', LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '4096', + LdsNumElementsAlignedA: '256', LdsNumElementsAlignedB: '256', LdsOffsetA: '0', + LdsOffsetA_Blk: '512', LdsOffsetB: '256', LdsOffsetB_Blk: '768', LdsPadA: '0', + LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', LocalRead2B: 'True', LocalReadVectorWidth: '4', + LocalSplitU: '4', LocalWrite2A: 'True', LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', + LocalWriteUseSgprB: 'False', LoopDoWhile: 'False', LoopIters: '2', LoopTail: 'True', + LoopUnroll: '2', MacroTile0: '32', MacroTile1: '32', MacroTileA: '32', MacroTileB: '32', + MacroTileShapeMax: '64', MacroTileShapeMin: '1', MagicDivAlg: '2', MatrixInstruction: '[]', + MaxOccupancy: '40', MaxVgprNumber: '256', MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', + NonTemporalA: '0', NonTemporalB: '0', NonTemporalC: '0', NumElementsPerThread: '4', + NumGlobalWriteVectorsPerThread: '1', NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', + NumLoadsCoalescedB: '1', NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', + NumThreads: '256', OptNoLoadLoop: '1', PackBatchDims: '0', PackFreeDims: '1', + PackGranularity: '2', PackSummationDims: '0', PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', + PackedC0IndicesX: '[0]', PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', + PackedC1IndicesX: '[1]', PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', + PerformanceWaitLocation: '-1', PersistentKernel: '0', PrefetchAcrossPersistent: '0', + PrefetchGlobalRead: 'True', PrefetchLocalRead: 'True', ReplacementKernel: 'False', + ScheduleGlobalRead: '1', ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '5', + SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT32x32x8_SE_, StaggerU: '32', StaggerUMapping: '0', + StaggerUStride: '256', StoreRemapVectorWidth: '0', StoreVectorWidth: '4', SubGroup0: '8', + SubGroup1: '8', SubGroupA: '8', SubGroupB: '8', SuppressNoLoadLoop: 'True', ThreadTile: '[4, + 4]', ThreadTile0: '4', ThreadTile1: '4', ThreadTileA: '4', ThreadTileB: '4', + TransposeLDS: '0', UnrollIncIsDepthU: '0', UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', + UnrollMemFence: 'False', Use64bShadowLimit: '1', UseInstOffsetForGRO: '0', UseSgprForGRO: '1', + Valid: 'True', VectorAtomicWidth: '1', VectorStore: 'True', VectorWidth: '4', + WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', WorkGroup: '[8, 8, + 4]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Ailk_Bjlk_SB_MT32x32x8_SE_EPS1_GRVW4_K1_LRVW4_PGR1_SNLL1_TT4_4_USFGRO1_VW4_WG8_8_4_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 8, DUorMT1: 8, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 32} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [32, 32, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [4, 4] + workGroup: [8, 8, 4] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +- debugKernel: false + hardwarePredicate: {type: TruePred} + ideals: {} + index: 27 + info: {1LDSBuffer: '0', AggressivePerfMode: '1', AssertFree0ElementMultiple: '1', + AssertFree1ElementMultiple: '1', AssertMinApproxSize: '3', AssertSizeEqual: '{}', + AssertStrideAEqual: '{0: 1}', AssertStrideBEqual: '{0: 1}', AssertStrideCEqual: '{0: + 1}', AssertStrideDEqual: '{0: 1}', AssertSummationElementMultiple: '1', AssignedDerivedParameters: 'True', + AssignedProblemIndependentDerivedParameters: 'True', BufferLoad: 'True', BufferStore: 'True', + CheckDimOverflow: '0', CheckTensorDimAsserts: 'False', CodeObjectVersion: V3, + DepthU: '8', DirectToLds: 'False', DirectToLdsA: 'False', DirectToLdsB: 'False', + DisableAtomicFail: '0', DisableKernelPieces: '0', DisableVgprOverlapping: 'False', + EdgeType: ShiftPtr, EnableMatrixInstruction: 'False', ExpandPointerSwap: 'True', + FractionalLoad: '0', GlobalLoadVectorWidthA: '1', GlobalLoadVectorWidthB: '1', + GlobalRead2A: 'True', GlobalRead2B: 'True', GlobalReadCoalesceGroupA: 'True', + GlobalReadCoalesceGroupB: 'True', GlobalReadCoalesceVectorA: 'True', GlobalReadCoalesceVectorB: 'True', + GlobalReadVectorWidth: '2', GlobalSplitU: '1', GlobalSplitUSummationAssignmentRoundRobin: 'True', + GlobalSplitUWorkGroupMappingRoundRobin: 'False', GlobalWriteVectorWidth: '1', + GuaranteeNoPartialA: 'True', GuaranteeNoPartialB: 'True', ISA: '(10, 1, 0)', InnerUnroll: '1', + InterleaveAlpha: '0', KernelLanguage: Assembly, LSCA: '8', LSCB: '8', LSPA: '8', + LSPB: '8', LVCA: '8', LVCB: '8', LVPA: '8', LVPB: '8', LdcEqualsLdd: 'True', LdsBlockSizePerPad: '0', + LdsBlockSizePerPadA: '0', LdsBlockSizePerPadB: '0', LdsNumElements: '409', LdsNumElementsAlignedA: '64', + LdsNumElementsAlignedB: '64', LdsOffsetA: '0', LdsOffsetA_Blk: '128', LdsOffsetB: '64', + LdsOffsetB_Blk: '192', LdsPadA: '0', LdsPadB: '0', LocalDotLayout: '1', LocalRead2A: 'True', + LocalRead2B: 'True', LocalReadVectorWidth: '2', LocalSplitU: '4', LocalWrite2A: 'True', + LocalWrite2B: 'True', LocalWriteUseSgprA: 'False', LocalWriteUseSgprB: 'False', + LoopDoWhile: 'False', LoopIters: '2', LoopTail: 'True', LoopUnroll: '2', MacroTile0: '8', + MacroTile1: '8', MacroTileA: '8', MacroTileB: '8', MacroTileShapeMax: '64', MacroTileShapeMin: '1', + MagicDivAlg: '2', MatrixInstruction: '[]', MaxOccupancy: '40', MaxVgprNumber: '256', + MinGlobalWriteVectorWidth: '1', MinVgprNumber: '0', NonTemporalA: '0', NonTemporalB: '0', + NonTemporalC: '0', NumElementsPerThread: '1', NumGlobalWriteVectorsPerThread: '1', + NumLoadsA: '1', NumLoadsB: '1', NumLoadsCoalescedA: '1', NumLoadsCoalescedB: '1', + NumLoadsPerpendicularA: '1', NumLoadsPerpendicularB: '1', NumThreads: '64', OptNoLoadLoop: '1', + PackBatchDims: '0', PackFreeDims: '1', PackGranularity: '2', PackSummationDims: '0', + PackedC0IdxChars: '[''I'']', PackedC0Indices: '[''I'']', PackedC0IndicesX: '[0]', + PackedC1IdxChars: '[''J'']', PackedC1Indices: '[''J'']', PackedC1IndicesX: '[1]', + PerformanceSyncLocation: '-1', PerformanceWaitCount: '-1', PerformanceWaitLocation: '-1', + PersistentKernel: '0', PrefetchAcrossPersistent: '0', PrefetchGlobalRead: 'True', + PrefetchLocalRead: 'True', ReplacementKernel: 'False', ScheduleGlobalRead: '1', + ScheduleIterAlg: '1', ScheduleLocalWrite: '1', SolutionIndex: '6', SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT8x8x8_SE_, + StaggerU: '32', StaggerUMapping: '0', StaggerUStride: '256', StoreRemapVectorWidth: '0', + StoreVectorWidth: '4', SubGroup0: '4', SubGroup1: '4', SubGroupA: '4', SubGroupB: '4', + SuppressNoLoadLoop: 'True', ThreadTile: '[2, 2]', ThreadTile0: '2', ThreadTile1: '2', + ThreadTileA: '2', ThreadTileB: '2', TransposeLDS: '0', UnrollIncIsDepthU: '0', + UnrollMajorLDSA: '0', UnrollMajorLDSB: '0', UnrollMemFence: 'False', Use64bShadowLimit: '1', + UseInstOffsetForGRO: '0', UseSgprForGRO: '1', Valid: 'True', VectorAtomicWidth: '1', + VectorStore: 'True', VectorWidth: '2', WaveSeparateGlobalReadA: '0', WaveSeparateGlobalReadB: '0', + WorkGroup: '[4, 4, 4]', WorkGroupMapping: '1', WorkGroupMappingType: B, _GlobalAccumulation: 'False', + _UseSgprForGRO: '1', _VectorStore: 'True', _WorkspaceSizePerElemC: '0', _staggerStrideShift: '3'} + name: Cijk_Ailk_Bjlk_SB_MT8x8x8_SE_EPS1_GRVW2_K1_LRVW2_PGR1_SNLL1_TT2_2_USFGRO1_VW2_WG4_4_4_WGM1 + problemPredicate: + type: And + value: + - {index: 0, type: StrideAEqual, value: 1} + - {index: 0, type: StrideBEqual, value: 1} + - {index: 0, type: StrideCEqual, value: 1} + - {index: 0, type: StrideDEqual, value: 1} + - {index: 0, type: WorkspaceCheck, value: 0} + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bjlk_Cijk_Dijk} + - type: TypesEqual + value: [Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: LeadingFree0SizesGreaterOrEqual, value: 1} + - {type: LeadingFree1SizesGreaterOrEqual, value: 1} + - {type: CDStridesEqual, value: true} + - {type: KernelLanguageCompatible, value: Assembly} + - {type: ArithmeticUnitCompatible, value: VALU} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 8, DUorMT1: 8, ShiftPtrElemA: 0, ShiftPtrElemB: 0} + - {type: BufferStoreOffsetLimitCheck, value: 8} + problemType: {aType: Float, bType: Float, cType: Float, dType: Float, highPrecisionAccumulate: false, + operationIdentifier: Contraction_l_Ailk_Bjlk_Cijk_Dijk, useBeta: true, useInitialStridesAB: false, + useInitialStridesCD: false} + sizeMapping: + depthU: 8 + globalAccumulation: false + globalSplitU: 1 + macroTile: [8, 8, 1] + magicDivAlg: 2 + packBatchDims: 0 + packSummationDims: 0 + persistentKernel: 0 + sourceKernel: false + staggerStrideShift: 3 + staggerU: 32 + threadTile: [2, 2] + workGroup: [4, 4, 4] + workGroupMapping: 1 + workspaceSizePerElemC: 0 +... diff --git a/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Ailk_Bjlk_SB.yaml b/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Ailk_Bjlk_SB.yaml new file mode 100644 index 0000000000..b930d56889 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Ailk_Bjlk_SB.yaml @@ -0,0 +1,3238 @@ +- {MinimumRequiredVersion: 4.9.4} +- navi10 +- gfx1010 +- [NAVI] +- AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: true + UseBeta: true + UseInitialStrides: false +- - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 16 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: false + GuaranteeNoPartialB: false + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 64 + LSCB: 128 + LSPA: 16 + LSPB: 8 + LVCA: 16 + LVCB: 32 + LVPA: 4 + LVPB: 2 + LdcEqualsLdd: true + LdsNumElements: 7168 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 2048 + LdsOffsetA: 0 + LdsOffsetA_Blk: 4096 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 5120 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 16 + MacroTile0: 64 + MacroTile1: 128 + MacroTileA: 64 + MacroTileB: 128 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 32 + NumGlobalWriteVectorsPerThread: 8 + NumLoadsA: 1 + NumLoadsB: 2 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 2 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 0 + SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT64x128x16_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: true + ThreadTile: [4, 8] + ThreadTile0: 4 + ThreadTile1: 8 + ThreadTileA: 4 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 2 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 32 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: false + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: false + GuaranteeNoPartialB: false + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 64 + LSCB: 32 + LSPA: 16 + LSPB: 32 + LVCA: 16 + LVCB: 8 + LVPA: 4 + LVPB: 8 + LdcEqualsLdd: true + LdsNumElements: 8192 + LdsOffsetA: 0 + LdsOffsetB: 2048 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 8 + MacroTile0: 64 + MacroTile1: 32 + MacroTileA: 64 + MacroTileB: 32 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 8 + NumGlobalWriteVectorsPerThread: 2 + NumLoadsA: 2 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 2 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: false + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 1 + SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT64x32x32_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: false + ThreadTile: [8, 4] + ThreadTile0: 8 + ThreadTile1: 4 + ThreadTileA: 8 + ThreadTileB: 4 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 1 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 32 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: false + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: false + GuaranteeNoPartialB: false + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 32 + LSCB: 64 + LSPA: 32 + LSPB: 16 + LVCA: 8 + LVCB: 16 + LVPA: 8 + LVPB: 4 + LdcEqualsLdd: true + LdsNumElements: 8192 + LdsOffsetA: 0 + LdsOffsetB: 1024 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 8 + MacroTile0: 32 + MacroTile1: 64 + MacroTileA: 32 + MacroTileB: 64 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 8 + NumGlobalWriteVectorsPerThread: 2 + NumLoadsA: 1 + NumLoadsB: 2 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 2 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: false + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 2 + SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT32x64x32_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: false + ThreadTile: [4, 8] + ThreadTile0: 4 + ThreadTile1: 8 + ThreadTileA: 4 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 1 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 16 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: false + GuaranteeNoPartialB: false + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 64 + LSCB: 64 + LSPA: 16 + LSPB: 16 + LVCA: 16 + LVCB: 16 + LVPA: 4 + LVPB: 4 + LdcEqualsLdd: true + LdsNumElements: 4096 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 1024 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 3072 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 16 + MacroTile0: 64 + MacroTile1: 64 + MacroTileA: 64 + MacroTileB: 64 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 16 + NumGlobalWriteVectorsPerThread: 4 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 3 + SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT64x64x16_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: true + ThreadTile: [4, 4] + ThreadTile0: 4 + ThreadTile1: 4 + ThreadTileA: 4 + ThreadTileB: 4 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 2 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: false + GuaranteeNoPartialB: false + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 128 + LSCB: 128 + LSPA: 8 + LSPB: 8 + LVCA: 32 + LVCB: 32 + LVPA: 2 + LVPB: 2 + LdcEqualsLdd: true + LdsNumElements: 4096 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 1024 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 3072 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 8 + MacroTile0: 128 + MacroTile1: 128 + MacroTileA: 128 + MacroTileB: 128 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 64 + NumGlobalWriteVectorsPerThread: 16 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 4 + SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT128x128x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: true + ThreadTile: [8, 8] + ThreadTile0: 8 + ThreadTile1: 8 + ThreadTileA: 8 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 3 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 1 + GlobalLoadVectorWidthB: 1 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 32 + LSCB: 32 + LSPA: 8 + LSPB: 8 + LVCA: 32 + LVCB: 32 + LVPA: 8 + LVPB: 8 + LdcEqualsLdd: true + LdsNumElements: 4096 + LdsNumElementsAlignedA: 256 + LdsNumElementsAlignedB: 256 + LdsOffsetA: 0 + LdsOffsetA_Blk: 512 + LdsOffsetB: 256 + LdsOffsetB_Blk: 768 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 2 + MacroTile0: 32 + MacroTile1: 32 + MacroTileA: 32 + MacroTileB: 32 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 4 + NumGlobalWriteVectorsPerThread: 1 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 5 + SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT32x32x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: true + ThreadTile: [4, 4] + ThreadTile0: 4 + ThreadTile1: 4 + ThreadTileA: 4 + ThreadTileB: 4 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 3 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 1 + GlobalLoadVectorWidthB: 1 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 2 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 1 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 8 + LSCB: 8 + LSPA: 8 + LSPB: 8 + LVCA: 8 + LVCB: 8 + LVPA: 8 + LVPB: 8 + LdcEqualsLdd: true + LdsNumElements: 409 + LdsNumElementsAlignedA: 64 + LdsNumElementsAlignedB: 64 + LdsOffsetA: 0 + LdsOffsetA_Blk: 128 + LdsOffsetB: 64 + LdsOffsetB_Blk: 192 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 2 + MacroTile0: 8 + MacroTile1: 8 + MacroTileA: 8 + MacroTileB: 8 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 1 + NumGlobalWriteVectorsPerThread: 1 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 64 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 6 + SolutionNameMin: Cijk_Ailk_Bjlk_SB_MT8x8x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 4 + SubGroup1: 4 + SubGroupA: 4 + SubGroupB: 4 + SuppressNoLoadLoop: true + ThreadTile: [2, 2] + ThreadTile0: 2 + ThreadTile1: 2 + ThreadTileA: 2 + ThreadTileB: 2 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 2 + WorkGroup: [4, 4, 4] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 3 +- [2, 3, 0, 1] +- - - [256, 2944, 1, 256, 256, 256, 256, 2944] + - [3, 4537.64] + - - [4288, 1408, 1, 3328, 4288, 4288, 4288, 1408] + - [4, 6282.76] + - - [256, 2944, 1, 3328, 256, 256, 256, 2944] + - [3, 5592.95] + - - [1856, 256, 1, 768, 1856, 1856, 1856, 256] + - [3, 4709.45] + - - [128, 64, 1, 256, 128, 128, 128, 64] + - [5, 137.428] + - - [5056, 704, 1, 3328, 5056, 5056, 5056, 704] + - [3, 6115.47] + - - [3584, 1856, 1, 3328, 3584, 3584, 3584, 1856] + - [3, 6215.14] + - - [448, 1408, 1, 256, 448, 448, 448, 1408] + - [3, 3942.02] + - - [5056, 448, 1, 1792, 5056, 5056, 5056, 448] + - [3, 5941.83] + - - [2944, 1024, 1, 256, 2944, 2944, 2944, 1024] + - [0, 5409.33] + - - [6784, 128, 1, 768, 6784, 6784, 6784, 128] + - [3, 4723.42] + - - [5056, 256, 1, 768, 5056, 5056, 5056, 256] + - [3, 5761.82] + - - [1408, 448, 1, 3328, 1408, 1408, 1408, 448] + - [3, 4696.57] + - - [64, 1856, 1, 256, 64, 64, 64, 1856] + - [1, 1732.1] + - - [1408, 128, 1, 1792, 1408, 1408, 1408, 128] + - [1, 2949.84] + - - [4288, 6784, 1, 256, 4288, 4288, 4288, 6784] + - [3, 6262.87] + - - [4288, 64, 1, 1792, 4288, 4288, 4288, 64] + - [3, 2894.02] + - - [4, 6784, 1, 768, 4, 4, 4, 6784] + - [6, 346.478] + - - [4288, 5056, 1, 1792, 4288, 4288, 4288, 5056] + - [0, 6451.94] + - - [448, 1408, 1, 1792, 448, 448, 448, 1408] + - [3, 4624.1] + - - [5888, 1856, 1, 3328, 5888, 5888, 5888, 1856] + - [3, 6350.85] + - - [256, 6784, 1, 256, 256, 256, 256, 6784] + - [3, 5182.95] + - - [5056, 448, 1, 768, 5056, 5056, 5056, 448] + - [3, 5833.03] + - - [448, 1856, 1, 3328, 448, 448, 448, 1856] + - [3, 4752.68] + - - [4, 1408, 1, 256, 4, 4, 4, 1408] + - [6, 81.0975] + - - [256, 256, 1, 3328, 256, 256, 256, 256] + - [2, 1518.47] + - - [5888, 4288, 1, 256, 5888, 5888, 5888, 4288] + - [3, 6213.33] + - - [5888, 1856, 1, 256, 5888, 5888, 5888, 1856] + - [3, 6130.94] + - - [64, 64, 1, 768, 64, 64, 64, 64] + - [2, 85.1036] + - - [4, 4, 1, 1792, 4, 4, 4, 4] + - [6, 0.369687] + - - [1408, 6784, 1, 768, 1408, 1408, 1408, 6784] + - [3, 6232.29] + - - [64, 6784, 1, 256, 64, 64, 64, 6784] + - [3, 3624.33] + - - [3584, 128, 1, 3328, 3584, 3584, 3584, 128] + - [3, 4902.53] + - - [704, 1408, 1, 3328, 704, 704, 704, 1408] + - [3, 5662.16] + - - [4, 3584, 1, 3328, 4, 4, 4, 3584] + - [6, 287.789] + - - [2944, 1024, 1, 3328, 2944, 2944, 2944, 1024] + - [4, 6094.81] + - - [4288, 1856, 1, 3328, 4288, 4288, 4288, 1856] + - [4, 6251.77] + - - [448, 128, 1, 1792, 448, 448, 448, 128] + - [1, 1229.81] + - - [5056, 2944, 1, 1792, 5056, 5056, 5056, 2944] + - [0, 6410.17] + - - [1024, 2944, 1, 768, 1024, 1024, 1024, 2944] + - [3, 5865.1] + - - [5056, 4, 1, 1792, 5056, 5056, 5056, 4] + - [6, 354.783] + - - [6784, 6784, 1, 1792, 6784, 6784, 6784, 6784] + - [4, 6699.74] + - - [4, 1024, 1, 768, 4, 4, 4, 1024] + - [6, 81.1968] + - - [1024, 1856, 1, 768, 1024, 1024, 1024, 1856] + - [3, 5458.26] + - - [4, 5888, 1, 3328, 4, 4, 4, 5888] + - [6, 369.067] + - - [1856, 3584, 1, 768, 1856, 1856, 1856, 3584] + - [0, 6196.32] + - - [704, 128, 1, 3328, 704, 704, 704, 128] + - [1, 2022.2] + - - [3584, 5056, 1, 768, 3584, 3584, 3584, 5056] + - [0, 6288.34] + - - [4288, 128, 1, 256, 4288, 4288, 4288, 128] + - [3, 3375.11] + - - [4288, 1408, 1, 256, 4288, 4288, 4288, 1408] + - [3, 5860.97] + - - [1856, 128, 1, 256, 1856, 1856, 1856, 128] + - [3, 2614.68] + - - [128, 4, 1, 1792, 128, 128, 128, 4] + - [6, 14.5718] + - - [2368, 2368, 1, 1792, 2368, 2368, 2368, 2368] + - [0, 6221.38] + - - [1408, 64, 1, 768, 1408, 1408, 1408, 64] + - [2, 1762.76] + - - [3584, 1856, 1, 768, 3584, 3584, 3584, 1856] + - [3, 6138.96] + - - [64, 3584, 1, 1792, 64, 64, 64, 3584] + - [1, 3686.99] + - - [64, 704, 1, 1792, 64, 64, 64, 704] + - [1, 971.447] + - - [6784, 4, 1, 256, 6784, 6784, 6784, 4] + - [6, 294.906] + - - [4, 2944, 1, 256, 4, 4, 4, 2944] + - [6, 164.762] + - - [2944, 256, 1, 256, 2944, 2944, 2944, 256] + - [3, 4651.07] + - - [64, 128, 1, 1792, 64, 64, 64, 128] + - [1, 197.189] + - - [1856, 6784, 1, 3328, 1856, 1856, 1856, 6784] + - [0, 6324.24] + - - [704, 4, 1, 256, 704, 704, 704, 4] + - [6, 42.4969] + - - [4288, 4288, 1, 256, 4288, 4288, 4288, 4288] + - [3, 6167.48] + - - [448, 64, 1, 1792, 448, 448, 448, 64] + - [2, 638.101] + - - [2368, 128, 1, 768, 2368, 2368, 2368, 128] + - [3, 3004.28] + - - [1024, 64, 1, 3328, 1024, 1024, 1024, 64] + - [2, 1414.18] + - - [5888, 128, 1, 1792, 5888, 5888, 5888, 128] + - [3, 5501.56] + - - [704, 704, 1, 768, 704, 704, 704, 704] + - [3, 4917.14] + - - [3584, 1856, 1, 256, 3584, 3584, 3584, 1856] + - [3, 5946.29] + - - [3584, 3584, 1, 1792, 3584, 3584, 3584, 3584] + - [3, 6387.63] + - - [128, 448, 1, 256, 128, 128, 128, 448] + - [1, 865.391] + - - [1856, 128, 1, 768, 1856, 1856, 1856, 128] + - [2, 3478.89] + - - [64, 1024, 1, 256, 64, 64, 64, 1024] + - [5, 972.029] + - - [4288, 2368, 1, 3328, 4288, 4288, 4288, 2368] + - [3, 6350.47] + - - [6784, 5056, 1, 256, 6784, 6784, 6784, 5056] + - [3, 6265.51] + - - [1024, 5888, 1, 1792, 1024, 1024, 1024, 5888] + - [3, 6305.15] + - - [704, 704, 1, 256, 704, 704, 704, 704] + - [3, 4187.8] + - - [4, 704, 1, 256, 4, 4, 4, 704] + - [6, 40.7194] + - - [128, 704, 1, 768, 128, 128, 128, 704] + - [2, 1850.01] + - - [704, 2368, 1, 1792, 704, 704, 704, 2368] + - [3, 5556.45] + - - [128, 704, 1, 3328, 128, 128, 128, 704] + - [2, 2011.14] + - - [4, 2368, 1, 3328, 4, 4, 4, 2368] + - [6, 197.195] + - - [5888, 1408, 1, 1792, 5888, 5888, 5888, 1408] + - [4, 6485.79] + - - [2944, 3584, 1, 3328, 2944, 2944, 2944, 3584] + - [0, 6284.61] + - - [4288, 2368, 1, 1792, 4288, 4288, 4288, 2368] + - [3, 6334.48] + - - [5888, 1024, 1, 768, 5888, 5888, 5888, 1024] + - [4, 6115.86] + - - [704, 4288, 1, 1792, 704, 704, 704, 4288] + - [0, 6008.03] + - - [6784, 2944, 1, 3328, 6784, 6784, 6784, 2944] + - [0, 6445.99] + - - [5056, 2368, 1, 256, 5056, 5056, 5056, 2368] + - [3, 6150.77] + - - [128, 4288, 1, 3328, 128, 128, 128, 4288] + - [3, 4085.27] + - - [2368, 1024, 1, 256, 2368, 2368, 2368, 1024] + - [0, 5208.2] + - - [64, 448, 1, 1792, 64, 64, 64, 448] + - [1, 628.843] + - - [2944, 2368, 1, 768, 2944, 2944, 2944, 2368] + - [3, 6199.28] + - - [1024, 256, 1, 1792, 1024, 1024, 1024, 256] + - [2, 4191.42] + - - [4288, 2368, 1, 768, 4288, 4288, 4288, 2368] + - [3, 6288.0] + - - [448, 2944, 1, 768, 448, 448, 448, 2944] + - [3, 4974.22] + - - [4288, 448, 1, 1792, 4288, 4288, 4288, 448] + - [3, 5646.67] + - - [3584, 2944, 1, 256, 3584, 3584, 3584, 2944] + - [3, 6057.19] + - - [64, 2944, 1, 256, 64, 64, 64, 2944] + - [1, 2170.48] + - - [128, 5056, 1, 1792, 128, 128, 128, 5056] + - [3, 4734.16] + - - [3584, 1024, 1, 256, 3584, 3584, 3584, 1024] + - [0, 5592.26] + - - [64, 1856, 1, 1792, 64, 64, 64, 1856] + - [2, 2514.04] + - - [1408, 2944, 1, 3328, 1408, 1408, 1408, 2944] + - [4, 6380.96] + - - [2944, 448, 1, 256, 2944, 2944, 2944, 448] + - [3, 4674.89] + - - [5888, 704, 1, 768, 5888, 5888, 5888, 704] + - [3, 6142.91] + - - [2368, 704, 1, 768, 2368, 2368, 2368, 704] + - [3, 5450.57] + - - [128, 1024, 1, 256, 128, 128, 128, 1024] + - [5, 1797.53] + - - [2944, 5056, 1, 256, 2944, 2944, 2944, 5056] + - [3, 6187.24] + - - [448, 448, 1, 256, 448, 448, 448, 448] + - [5, 2259.37] + - - [5888, 256, 1, 1792, 5888, 5888, 5888, 256] + - [3, 5819.39] + - - [5056, 4288, 1, 768, 5056, 5056, 5056, 4288] + - [0, 6375.29] + - - [4288, 5888, 1, 256, 4288, 4288, 4288, 5888] + - [3, 6213.77] + - - [3584, 128, 1, 256, 3584, 3584, 3584, 128] + - [3, 3765.81] + - - [1408, 64, 1, 256, 1408, 1408, 1408, 64] + - [2, 1336.54] + - - [1408, 1856, 1, 768, 1408, 1408, 1408, 1856] + - [3, 6084.3] + - - [4, 256, 1, 256, 4, 4, 4, 256] + - [6, 20.3386] + - - [6784, 5056, 1, 3328, 6784, 6784, 6784, 5056] + - [4, 6474.48] + - - [2944, 1856, 1, 3328, 2944, 2944, 2944, 1856] + - [3, 6301.54] + - - [448, 6784, 1, 1792, 448, 448, 448, 6784] + - [0, 6053.27] + - - [2944, 4288, 1, 768, 2944, 2944, 2944, 4288] + - [3, 6242.52] + - - [1024, 1408, 1, 768, 1024, 1024, 1024, 1408] + - [3, 5377.01] + - - [1856, 1856, 1, 1792, 1856, 1856, 1856, 1856] + - [3, 5895.91] + - - [2368, 5056, 1, 768, 2368, 2368, 2368, 5056] + - [3, 6298.71] + - - [128, 2368, 1, 256, 128, 128, 128, 2368] + - [3, 2561.13] + - - [5888, 5888, 1, 768, 5888, 5888, 5888, 5888] + - [4, 6456.16] + - - [256, 1856, 1, 256, 256, 256, 256, 1856] + - [3, 3689.93] + - - [1856, 64, 1, 3328, 1856, 1856, 1856, 64] + - [1, 2615.98] + - - [1856, 4288, 1, 1792, 1856, 1856, 1856, 4288] + - [3, 6232.68] + - - [5056, 256, 1, 1792, 5056, 5056, 5056, 256] + - [3, 5917.92] + - - [128, 128, 1, 1792, 128, 128, 128, 128] + - [2, 388.961] + - - [2368, 4288, 1, 3328, 2368, 2368, 2368, 4288] + - [3, 6351.62] + - - [4, 128, 1, 1792, 4, 4, 4, 128] + - [6, 14.8691] + - - [1408, 704, 1, 768, 1408, 1408, 1408, 704] + - [3, 5408.83] + - - [256, 2368, 1, 768, 256, 256, 256, 2368] + - [3, 4255.26] + - - [5888, 704, 1, 1792, 5888, 5888, 5888, 704] + - [3, 6219.87] + - - [4288, 6784, 1, 3328, 4288, 4288, 4288, 6784] + - [0, 6550.96] + - - [1024, 6784, 1, 3328, 1024, 1024, 1024, 6784] + - [3, 6247.33] + - - [704, 1024, 1, 3328, 704, 704, 704, 1024] + - [3, 5353.31] + - - [5888, 1856, 1, 768, 5888, 5888, 5888, 1856] + - [3, 6288.14] + - - [2368, 64, 1, 1792, 2368, 2368, 2368, 64] + - [1, 2477.2] + - - [128, 2944, 1, 1792, 128, 128, 128, 2944] + - [3, 3961.78] + - - [448, 1856, 1, 768, 448, 448, 448, 1856] + - [3, 4527.64] + - - [4, 1856, 1, 256, 4, 4, 4, 1856] + - [6, 105.151] + - - [2944, 6784, 1, 1792, 2944, 2944, 2944, 6784] + - [0, 6414.65] + - - [128, 256, 1, 256, 128, 128, 128, 256] + - [2, 539.252] + - - [64, 256, 1, 3328, 64, 64, 64, 256] + - [2, 365.664] + - - [1856, 4, 1, 256, 1856, 1856, 1856, 4] + - [6, 108.256] + - - [704, 2368, 1, 256, 704, 704, 704, 2368] + - [3, 5062.61] + - - [1408, 3584, 1, 1792, 1408, 1408, 1408, 3584] + - [4, 6215.69] + - - [5056, 704, 1, 1792, 5056, 5056, 5056, 704] + - [3, 6086.61] + - - [5888, 256, 1, 768, 5888, 5888, 5888, 256] + - [3, 5666.27] + - - [3584, 2944, 1, 1792, 3584, 3584, 3584, 2944] + - [3, 6263.65] + - - [256, 704, 1, 3328, 256, 256, 256, 704] + - [1, 3017.84] + - - [1856, 5888, 1, 1792, 1856, 1856, 1856, 5888] + - [0, 6466.09] + - - [2368, 2944, 1, 3328, 2368, 2368, 2368, 2944] + - [3, 6276.97] + - - [64, 704, 1, 256, 64, 64, 64, 704] + - [1, 707.758] + - - [64, 5056, 1, 3328, 64, 64, 64, 5056] + - [3, 3464.63] + - - [1408, 1856, 1, 256, 1408, 1408, 1408, 1856] + - [3, 5737.71] + - - [4, 1024, 1, 1792, 4, 4, 4, 1024] + - [6, 82.1621] + - - [4288, 3584, 1, 1792, 4288, 4288, 4288, 3584] + - [4, 6492.79] + - - [2944, 1408, 1, 768, 2944, 2944, 2944, 1408] + - [4, 6171.14] + - - [4288, 4, 1, 1792, 4288, 4288, 4288, 4] + - [6, 311.275] + - - [1856, 64, 1, 256, 1856, 1856, 1856, 64] + - [5, 1754.28] + - - [5888, 3584, 1, 768, 5888, 5888, 5888, 3584] + - [0, 6360.6] + - - [1408, 128, 1, 768, 1408, 1408, 1408, 128] + - [5, 2743.77] + - - [2944, 128, 1, 1792, 2944, 2944, 2944, 128] + - [3, 3958.34] + - - [64, 448, 1, 256, 64, 64, 64, 448] + - [5, 460.869] + - - [1856, 5056, 1, 256, 1856, 1856, 1856, 5056] + - [3, 6125.8] + - - [5056, 3584, 1, 768, 5056, 5056, 5056, 3584] + - [0, 6399.29] + - - [5888, 4, 1, 768, 5888, 5888, 5888, 4] + - [6, 366.086] + - - [128, 6784, 1, 1792, 128, 128, 128, 6784] + - [3, 4904.64] + - - [704, 1024, 1, 768, 704, 704, 704, 1024] + - [3, 5056.89] + - - [448, 4, 1, 768, 448, 448, 448, 4] + - [6, 37.2323] + - - [6784, 4288, 1, 256, 6784, 6784, 6784, 4288] + - [3, 6262.48] + - - [128, 5888, 1, 3328, 128, 128, 128, 5888] + - [3, 5600.35] + - - [1024, 64, 1, 256, 1024, 1024, 1024, 64] + - [5, 967.879] + - - [3584, 4, 1, 256, 3584, 3584, 3584, 4] + - [6, 192.032] + - - [2368, 64, 1, 768, 2368, 2368, 2368, 64] + - [1, 2314.07] + - - [4288, 256, 1, 256, 4288, 4288, 4288, 256] + - [3, 4473.64] + - - [256, 2944, 1, 1792, 256, 256, 256, 2944] + - [3, 5493.26] + - - [2944, 5888, 1, 768, 2944, 2944, 2944, 5888] + - [4, 6399.49] + - - [704, 448, 1, 256, 704, 704, 704, 448] + - [3, 2658.47] + - - [448, 256, 1, 3328, 448, 448, 448, 256] + - [2, 2525.77] + - - [2944, 2944, 1, 256, 2944, 2944, 2944, 2944] + - [3, 5991.76] + - - [64, 4288, 1, 1792, 64, 64, 64, 4288] + - [3, 2895.28] + - - [256, 2368, 1, 256, 256, 256, 256, 2368] + - [3, 3754.47] + - - [1856, 5888, 1, 256, 1856, 1856, 1856, 5888] + - [3, 6137.92] + - - [448, 256, 1, 256, 448, 448, 448, 256] + - [5, 1644.59] + - - [256, 4, 1, 768, 256, 256, 256, 4] + - [6, 26.021] + - - [5056, 256, 1, 3328, 5056, 5056, 5056, 256] + - [3, 5956.2] + - - [5056, 2368, 1, 768, 5056, 5056, 5056, 2368] + - [3, 6298.71] + - - [5888, 5056, 1, 1792, 5888, 5888, 5888, 5056] + - [4, 6542.77] + - - [1408, 3584, 1, 256, 1408, 1408, 1408, 3584] + - [3, 5836.5] + - - [1856, 704, 1, 3328, 1856, 1856, 1856, 704] + - [3, 6027.56] + - - [128, 1856, 1, 3328, 128, 128, 128, 1856] + - [1, 3948.64] + - - [2368, 5888, 1, 768, 2368, 2368, 2368, 5888] + - [0, 6362.73] + - - [4, 1024, 1, 256, 4, 4, 4, 1024] + - [6, 57.0793] + - - [5056, 6784, 1, 1792, 5056, 5056, 5056, 6784] + - [0, 6532.78] + - - [128, 1024, 1, 1792, 128, 128, 128, 1024] + - [1, 2536.65] + - - [128, 2944, 1, 3328, 128, 128, 128, 2944] + - [3, 4041.5] + - - [448, 448, 1, 3328, 448, 448, 448, 448] + - [1, 3338.39] + - - [64, 4288, 1, 768, 64, 64, 64, 4288] + - [3, 2743.71] + - - [1408, 256, 1, 256, 1408, 1408, 1408, 256] + - [3, 3091.02] + - - [256, 2944, 1, 768, 256, 256, 256, 2944] + - [3, 5268.91] + - - [1024, 2368, 1, 1792, 1024, 1024, 1024, 2368] + - [3, 5729.63] + - - [64, 1024, 1, 3328, 64, 64, 64, 1024] + - [1, 1411.47] + - - [5056, 448, 1, 256, 5056, 5056, 5056, 448] + - [3, 5489.43] + - - [4, 2368, 1, 1792, 4, 4, 4, 2368] + - [6, 189.844] + - - [704, 3584, 1, 768, 704, 704, 704, 3584] + - [3, 5875.82] + - - [704, 4288, 1, 256, 704, 704, 704, 4288] + - [3, 5636.17] + - - [2368, 1408, 1, 256, 2368, 2368, 2368, 1408] + - [3, 5775.69] + - - [3584, 256, 1, 1792, 3584, 3584, 3584, 256] + - [0, 5084.92] + - - [1856, 1856, 1, 3328, 1856, 1856, 1856, 1856] + - [0, 5971.17] + - - [256, 128, 1, 768, 256, 256, 256, 128] + - [1, 721.29] + - - [4, 448, 1, 1792, 4, 4, 4, 448] + - [6, 43.2214] + - - [128, 6784, 1, 3328, 128, 128, 128, 6784] + - [3, 4966.55] + - - [704, 256, 1, 3328, 704, 704, 704, 256] + - [2, 3047.38] + - - [1408, 5056, 1, 1792, 1408, 1408, 1408, 5056] + - [4, 6315.63] + - - [4, 2944, 1, 768, 4, 4, 4, 2944] + - [6, 219.195] + - - [256, 64, 1, 3328, 256, 256, 256, 64] + - [2, 364.397] + - - [4, 3584, 1, 256, 4, 4, 4, 3584] + - [6, 198.176] + - - [2368, 1024, 1, 768, 2368, 2368, 2368, 1024] + - [0, 5610.35] + - - [1856, 704, 1, 256, 1856, 1856, 1856, 704] + - [3, 5299.95] + - - [2944, 704, 1, 1792, 2944, 2944, 2944, 704] + - [3, 6087.71] + - - [5056, 6784, 1, 256, 5056, 5056, 5056, 6784] + - [3, 6265.51] + - - [4288, 1856, 1, 768, 4288, 4288, 4288, 1856] + - [3, 6182.97] + - - [1024, 1856, 1, 256, 1024, 1024, 1024, 1856] + - [3, 5052.38] + - - [5888, 6784, 1, 256, 5888, 5888, 5888, 6784] + - [4, 6317.77] + - - [1408, 5888, 1, 768, 1408, 1408, 1408, 5888] + - [4, 6371.97] + - - [1856, 1024, 1, 768, 1856, 1856, 1856, 1024] + - [0, 5399.91] + - - [5056, 1408, 1, 256, 5056, 5056, 5056, 1408] + - [3, 5956.91] + - - [5888, 6784, 1, 768, 5888, 5888, 5888, 6784] + - [4, 6497.28] + - - [256, 448, 1, 256, 256, 256, 256, 448] + - [5, 1701.1] + - - [64, 1408, 1, 768, 64, 64, 64, 1408] + - [2, 1743.01] + - - [1024, 256, 1, 768, 1024, 1024, 1024, 256] + - [2, 3871.55] + - - [704, 3584, 1, 256, 704, 704, 704, 3584] + - [3, 5561.04] + - - [256, 3584, 1, 3328, 256, 256, 256, 3584] + - [3, 5229.04] + - - [4288, 5056, 1, 768, 4288, 4288, 4288, 5056] + - [0, 6356.53] + - - [256, 4288, 1, 3328, 256, 256, 256, 4288] + - [3, 5067.03] + - - [6784, 2944, 1, 768, 6784, 6784, 6784, 2944] + - [0, 6345.46] + - - [128, 5056, 1, 256, 128, 128, 128, 5056] + - [3, 3965.56] + - - [256, 2368, 1, 3328, 256, 256, 256, 2368] + - [3, 4503.13] + - - [5888, 64, 1, 768, 5888, 5888, 5888, 64] + - [3, 3727.93] + - - [6784, 2368, 1, 768, 6784, 6784, 6784, 2368] + - [3, 6285.94] + - - [6784, 6784, 1, 768, 6784, 6784, 6784, 6784] + - [4, 6627.17] + - - [704, 64, 1, 256, 704, 704, 704, 64] + - [5, 707.758] + - - [3584, 1024, 1, 1792, 3584, 3584, 3584, 1024] + - [0, 6269.96] + - - [2368, 5888, 1, 1792, 2368, 2368, 2368, 5888] + - [0, 6443.07] + - - [2944, 4, 1, 256, 2944, 2944, 2944, 4] + - [6, 164.099] + - - [256, 1408, 1, 768, 256, 256, 256, 1408] + - [3, 3579.52] + - - [1024, 5888, 1, 3328, 1024, 1024, 1024, 5888] + - [3, 6339.73] + - - [448, 2368, 1, 256, 448, 448, 448, 2368] + - [3, 4323.41] + - - [2944, 2944, 1, 1792, 2944, 2944, 2944, 2944] + - [0, 6314.21] + - - [4, 5056, 1, 1792, 4, 4, 4, 5056] + - [6, 357.636] + - - [1024, 64, 1, 768, 1024, 1024, 1024, 64] + - [2, 1237.62] + - - [5056, 4288, 1, 1792, 5056, 5056, 5056, 4288] + - [0, 6461.23] + - - [448, 1408, 1, 768, 448, 448, 448, 1408] + - [3, 4430.76] + - - [6784, 5888, 1, 3328, 6784, 6784, 6784, 5888] + - [4, 6583.77] + - - [128, 6784, 1, 768, 128, 128, 128, 6784] + - [3, 4745.84] + - - [6784, 4, 1, 768, 6784, 6784, 6784, 4] + - [6, 350.796] + - - [1856, 448, 1, 256, 1856, 1856, 1856, 448] + - [3, 4128.65] + - - [2944, 1024, 1, 768, 2944, 2944, 2944, 1024] + - [4, 5855.21] + - - [2944, 704, 1, 768, 2944, 2944, 2944, 704] + - [3, 5965.53] + - - [1408, 6784, 1, 256, 1408, 1408, 1408, 6784] + - [3, 6075.9] + - - [3584, 1856, 1, 1792, 3584, 3584, 3584, 1856] + - [3, 6194.89] + - - [3584, 4288, 1, 1792, 3584, 3584, 3584, 4288] + - [4, 6476.39] + - - [1024, 1024, 1, 768, 1024, 1024, 1024, 1024] + - [0, 5331.71] + - - [3584, 5056, 1, 1792, 3584, 3584, 3584, 5056] + - [0, 6424.99] + - - [5888, 704, 1, 3328, 5888, 5888, 5888, 704] + - [3, 6244.45] + - - [4, 4288, 1, 256, 4, 4, 4, 4288] + - [6, 228.866] + - - [1408, 704, 1, 3328, 1408, 1408, 1408, 704] + - [3, 5649.94] + - - [4288, 2944, 1, 768, 4288, 4288, 4288, 2944] + - [3, 6242.22] + - - [5888, 1408, 1, 256, 5888, 5888, 5888, 1408] + - [3, 6053.34] + - - [2368, 128, 1, 256, 2368, 2368, 2368, 128] + - [3, 2592.8] + - - [1408, 4288, 1, 3328, 1408, 1408, 1408, 4288] + - [4, 6284.8] + - - [256, 3584, 1, 768, 256, 256, 256, 3584] + - [3, 4998.66] + - - [64, 64, 1, 256, 64, 64, 64, 64] + - [1, 53.2151] + - - [6784, 6784, 1, 256, 6784, 6784, 6784, 6784] + - [4, 6436.66] + - - [256, 256, 1, 1792, 256, 256, 256, 256] + - [2, 1436.05] + - - [6784, 1024, 1, 768, 6784, 6784, 6784, 1024] + - [4, 6074.78] + - - [6784, 4288, 1, 1792, 6784, 6784, 6784, 4288] + - [3, 6414.73] + - - [128, 3584, 1, 256, 128, 128, 128, 3584] + - [3, 3774.77] + - - [64, 5888, 1, 768, 64, 64, 64, 5888] + - [3, 3735.08] + - - [4, 4, 1, 256, 4, 4, 4, 4] + - [6, 0.234298] + - - [1408, 448, 1, 1792, 1408, 1408, 1408, 448] + - [3, 4617.1] + - - [6784, 2368, 1, 1792, 6784, 6784, 6784, 2368] + - [4, 6365.22] + - - [448, 3584, 1, 1792, 448, 448, 448, 3584] + - [3, 5357.57] + - - [64, 64, 1, 3328, 64, 64, 64, 64] + - [1, 102.977] + - - [6784, 256, 1, 256, 6784, 6784, 6784, 256] + - [3, 5283.35] + - - [1856, 3584, 1, 3328, 1856, 1856, 1856, 3584] + - [0, 6342.83] + - - [448, 704, 1, 256, 448, 448, 448, 704] + - [3, 2658.47] + - - [5056, 1856, 1, 768, 5056, 5056, 5056, 1856] + - [3, 6288.13] + - - [3584, 4288, 1, 3328, 3584, 3584, 3584, 4288] + - [4, 6518.29] + - - [4288, 3584, 1, 256, 4288, 4288, 4288, 3584] + - [3, 6179.29] + - - [2944, 448, 1, 3328, 2944, 2944, 2944, 448] + - [3, 5148.09] + - - [4288, 3584, 1, 3328, 4288, 4288, 4288, 3584] + - [4, 6525.99] + - - [3584, 5888, 1, 1792, 3584, 3584, 3584, 5888] + - [0, 6430.77] + - - [2368, 256, 1, 768, 2368, 2368, 2368, 256] + - [3, 4278.43] + - - [1024, 448, 1, 1792, 1024, 1024, 1024, 448] + - [3, 4750.79] + - - [1408, 704, 1, 256, 1408, 1408, 1408, 704] + - [3, 4859.03] + - - [704, 5056, 1, 3328, 704, 704, 704, 5056] + - [0, 6177.6] + - - [4288, 256, 1, 1792, 4288, 4288, 4288, 256] + - [3, 5019.95] + - - [5888, 2368, 1, 256, 5888, 5888, 5888, 2368] + - [3, 6118.72] + - - [128, 1856, 1, 1792, 128, 128, 128, 1856] + - [3, 3828.85] + - - [3584, 4288, 1, 768, 3584, 3584, 3584, 4288] + - [4, 6356.03] + - - [4, 2944, 1, 1792, 4, 4, 4, 2944] + - [6, 230.111] + - - [2944, 5888, 1, 3328, 2944, 2944, 2944, 5888] + - [4, 6509.19] + - - [128, 5056, 1, 768, 128, 128, 128, 5056] + - [3, 4512.19] + - - [64, 6784, 1, 1792, 64, 64, 64, 6784] + - [3, 4558.73] + - - [2944, 704, 1, 3328, 2944, 2944, 2944, 704] + - [3, 6123.92] + - - [3584, 3584, 1, 3328, 3584, 3584, 3584, 3584] + - [0, 6425.64] + - - [448, 128, 1, 256, 448, 448, 448, 128] + - [1, 857.906] + - - [6784, 2368, 1, 256, 6784, 6784, 6784, 2368] + - [3, 6155.61] + - - [128, 448, 1, 768, 128, 128, 128, 448] + - [2, 1218.3] + - - [2368, 2368, 1, 256, 2368, 2368, 2368, 2368] + - [3, 5899.16] + - - [64, 2944, 1, 1792, 64, 64, 64, 2944] + - [1, 3073.52] + - - [256, 5056, 1, 256, 256, 256, 256, 5056] + - [3, 5083.07] + - - [6784, 3584, 1, 3328, 6784, 6784, 6784, 3584] + - [0, 6515.93] + - - [448, 704, 1, 768, 448, 448, 448, 704] + - [3, 3162.38] + - - [1856, 1856, 1, 768, 1856, 1856, 1856, 1856] + - [3, 5829.92] + - - [1856, 448, 1, 1792, 1856, 1856, 1856, 448] + - [3, 4693.14] + - - [256, 1024, 1, 256, 256, 256, 256, 1024] + - [3, 2950.94] + - - [64, 3584, 1, 256, 64, 64, 64, 3584] + - [3, 2557.11] + - - [1856, 128, 1, 3328, 1856, 1856, 1856, 128] + - [1, 3975.11] + - - [5888, 5056, 1, 768, 5888, 5888, 5888, 5056] + - [4, 6434.11] + - - [6784, 448, 1, 1792, 6784, 6784, 6784, 448] + - [3, 6033.9] + - - [3584, 448, 1, 256, 3584, 3584, 3584, 448] + - [3, 4880.31] + - - [448, 2368, 1, 1792, 448, 448, 448, 2368] + - [3, 4862.39] + - - [128, 2368, 1, 1792, 128, 128, 128, 2368] + - [3, 3185.27] + - - [1408, 3584, 1, 3328, 1408, 1408, 1408, 3584] + - [4, 6273.18] + - - [3584, 1408, 1, 1792, 3584, 3584, 3584, 1408] + - [4, 6213.47] + - - [5056, 128, 1, 1792, 5056, 5056, 5056, 128] + - [3, 4738.46] + - - [5888, 1024, 1, 256, 5888, 5888, 5888, 1024] + - [3, 5768.75] + - - [256, 704, 1, 1792, 256, 256, 256, 704] + - [2, 2912.4] + - - [4288, 128, 1, 768, 4288, 4288, 4288, 128] + - [3, 3868.43] + - - [6784, 2368, 1, 3328, 6784, 6784, 6784, 2368] + - [4, 6395.68] + - - [2368, 3584, 1, 1792, 2368, 2368, 2368, 3584] + - [3, 6245.35] + - - [2368, 704, 1, 3328, 2368, 2368, 2368, 704] + - [3, 5602.23] + - - [5056, 64, 1, 768, 5056, 5056, 5056, 64] + - [3, 3219.6] + - - [4, 128, 1, 3328, 4, 4, 4, 128] + - [6, 13.1069] + - - [64, 3584, 1, 768, 64, 64, 64, 3584] + - [3, 3344.74] + - - [4288, 128, 1, 1792, 4288, 4288, 4288, 128] + - [3, 4026.01] + - - [5888, 64, 1, 3328, 5888, 5888, 5888, 64] + - [3, 4034.76] + - - [3584, 4288, 1, 256, 3584, 3584, 3584, 4288] + - [3, 6165.66] + - - [4288, 448, 1, 3328, 4288, 4288, 4288, 448] + - [3, 5679.86] + - - [704, 256, 1, 768, 704, 704, 704, 256] + - [5, 2924.09] + - - [6784, 64, 1, 3328, 6784, 6784, 6784, 64] + - [3, 4643.21] + - - [128, 6784, 1, 256, 128, 128, 128, 6784] + - [3, 4256.63] + - - [128, 1024, 1, 768, 128, 128, 128, 1024] + - [1, 2257.35] + - - [448, 4288, 1, 256, 448, 448, 448, 4288] + - [3, 5211.05] + - - [1856, 4288, 1, 256, 1856, 1856, 1856, 4288] + - [3, 6010.48] + - - [4288, 2944, 1, 256, 4288, 4288, 4288, 2944] + - [3, 6105.08] + - - [256, 448, 1, 3328, 256, 256, 256, 448] + - [1, 2492.77] + - - [704, 704, 1, 3328, 704, 704, 704, 704] + - [3, 5305.32] + - - [704, 5888, 1, 256, 704, 704, 704, 5888] + - [3, 5900.01] + - - [4, 256, 1, 768, 4, 4, 4, 256] + - [6, 27.2216] + - - [1408, 64, 1, 3328, 1408, 1408, 1408, 64] + - [2, 1989.4] + - - [4288, 704, 1, 768, 4288, 4288, 4288, 704] + - [3, 5914.13] + - - [4288, 448, 1, 256, 4288, 4288, 4288, 448] + - [3, 5244.0] + - - [6784, 3584, 1, 1792, 6784, 6784, 6784, 3584] + - [0, 6487.31] + - - [1024, 128, 1, 1792, 1024, 1024, 1024, 128] + - [2, 2607.56] + - - [1024, 6784, 1, 768, 1024, 1024, 1024, 6784] + - [3, 6146.32] + - - [2944, 1856, 1, 256, 2944, 2944, 2944, 1856] + - [3, 5998.5] + - - [704, 64, 1, 768, 704, 704, 704, 64] + - [2, 973.184] + - - [6784, 64, 1, 1792, 6784, 6784, 6784, 64] + - [3, 4548.84] + - - [256, 704, 1, 768, 256, 256, 256, 704] + - [5, 2735.74] + - - [2368, 1856, 1, 256, 2368, 2368, 2368, 1856] + - [3, 5905.41] + - - [128, 128, 1, 256, 128, 128, 128, 128] + - [5, 274.865] + - - [1408, 2368, 1, 256, 1408, 1408, 1408, 2368] + - [3, 5761.23] + - - [4288, 2368, 1, 256, 4288, 4288, 4288, 2368] + - [3, 6125.19] + - - [6784, 256, 1, 768, 6784, 6784, 6784, 256] + - [3, 5672.86] + - - [704, 6784, 1, 768, 704, 704, 704, 6784] + - [3, 5998.98] + - - [2944, 64, 1, 256, 2944, 2944, 2944, 64] + - [1, 2253.11] + - - [6784, 1408, 1, 3328, 6784, 6784, 6784, 1408] + - [3, 6294.51] + - - [1024, 4, 1, 1792, 1024, 1024, 1024, 4] + - [6, 82.2989] + - - [256, 6784, 1, 768, 256, 256, 256, 6784] + - [3, 5642.63] + - - [6784, 1024, 1, 1792, 6784, 6784, 6784, 1024] + - [4, 6193.9] + - - [4288, 2944, 1, 1792, 4288, 4288, 4288, 2944] + - [0, 6303.3] + - - [3584, 704, 1, 256, 3584, 3584, 3584, 704] + - [3, 5543.38] + - - [64, 1024, 1, 1792, 64, 64, 64, 1024] + - [1, 1370.27] + - - [2944, 1408, 1, 256, 2944, 2944, 2944, 1408] + - [3, 5900.01] + - - [704, 64, 1, 3328, 704, 704, 704, 64] + - [2, 1011.1] + - - [2944, 128, 1, 3328, 2944, 2944, 2944, 128] + - [3, 4044.4] + - - [4, 5056, 1, 768, 4, 4, 4, 5056] + - [6, 332.82] + - - [1408, 1408, 1, 3328, 1408, 1408, 1408, 1408] + - [4, 5876.99] + - - [256, 3584, 1, 256, 256, 256, 256, 3584] + - [3, 4428.5] + - - [3584, 64, 1, 3328, 3584, 3584, 3584, 64] + - [2, 3869.74] + - - [1856, 1024, 1, 1792, 1856, 1856, 1856, 1024] + - [0, 5564.84] + - - [128, 64, 1, 1792, 128, 128, 128, 64] + - [2, 200.582] + - - [2944, 2944, 1, 768, 2944, 2944, 2944, 2944] + - [0, 6231.51] + - - [64, 128, 1, 3328, 64, 64, 64, 128] + - [1, 195.974] + - - [2944, 4288, 1, 1792, 2944, 2944, 2944, 4288] + - [0, 6290.7] + - - [128, 3584, 1, 768, 128, 128, 128, 3584] + - [3, 4465.93] + - - [5056, 6784, 1, 768, 5056, 5056, 5056, 6784] + - [0, 6464.96] + - - [4288, 4, 1, 768, 4288, 4288, 4288, 4] + - [6, 289.152] + - - [4288, 4, 1, 3328, 4288, 4288, 4288, 4] + - [6, 321.211] + - - [5056, 3584, 1, 1792, 5056, 5056, 5056, 3584] + - [0, 6474.52] + - - [4, 6784, 1, 3328, 4, 4, 4, 6784] + - [6, 348.523] + - - [1408, 256, 1, 1792, 1408, 1408, 1408, 256] + - [3, 3807.73] + - - [2368, 448, 1, 768, 2368, 2368, 2368, 448] + - [3, 4730.64] + - - [3584, 448, 1, 768, 3584, 3584, 3584, 448] + - [3, 5244.72] + - - [448, 64, 1, 256, 448, 448, 448, 64] + - [5, 454.533] + - - [704, 1408, 1, 768, 704, 704, 704, 1408] + - [3, 5408.83] + - - [704, 6784, 1, 256, 704, 704, 704, 6784] + - [3, 5785.21] + - - [2368, 64, 1, 256, 2368, 2368, 2368, 64] + - [5, 1883.96] + - - [1024, 3584, 1, 1792, 1024, 1024, 1024, 3584] + - [3, 6213.79] + - - [448, 3584, 1, 256, 448, 448, 448, 3584] + - [3, 4893.21] + - - [448, 2368, 1, 3328, 448, 448, 448, 2368] + - [3, 4904.93] + - - [3584, 64, 1, 768, 3584, 3584, 3584, 64] + - [2, 3378.0] + - - [128, 256, 1, 1792, 128, 128, 128, 256] + - [2, 747.129] + - - [2368, 448, 1, 256, 2368, 2368, 2368, 448] + - [3, 4369.77] + - - [704, 6784, 1, 3328, 704, 704, 704, 6784] + - [3, 6081.84] + - - [3584, 256, 1, 3328, 3584, 3584, 3584, 256] + - [0, 5146.77] + - - [2368, 2944, 1, 768, 2368, 2368, 2368, 2944] + - [3, 6198.22] + - - [4, 64, 1, 1792, 4, 4, 4, 64] + - [6, 6.14994] + - - [3584, 3584, 1, 768, 3584, 3584, 3584, 3584] + - [0, 6271.44] + - - [6784, 1856, 1, 1792, 6784, 6784, 6784, 1856] + - [3, 6266.89] + - - [64, 256, 1, 1792, 64, 64, 64, 256] + - [2, 386.684] + - - [1024, 6784, 1, 1792, 1024, 1024, 1024, 6784] + - [3, 6220.49] + - - [448, 1024, 1, 1792, 448, 448, 448, 1024] + - [3, 4771.21] + - - [64, 4, 1, 1792, 64, 64, 64, 4] + - [6, 5.9663] + - - [128, 4, 1, 256, 128, 128, 128, 4] + - [6, 10.2874] + - - [1856, 2944, 1, 1792, 1856, 1856, 1856, 2944] + - [3, 6282.16] + - - [256, 1856, 1, 1792, 256, 256, 256, 1856] + - [3, 4924.67] + - - [4288, 1024, 1, 1792, 4288, 4288, 4288, 1024] + - [0, 5942.16] + - - [64, 2368, 1, 1792, 64, 64, 64, 2368] + - [2, 2472.19] + - - [5888, 3584, 1, 1792, 5888, 5888, 5888, 3584] + - [0, 6440.34] + - - [1856, 448, 1, 768, 1856, 1856, 1856, 448] + - [3, 4546.74] + - - [3584, 1024, 1, 768, 3584, 3584, 3584, 1024] + - [0, 6120.26] + - - [6784, 4288, 1, 3328, 6784, 6784, 6784, 4288] + - [0, 6435.89] + - - [256, 64, 1, 768, 256, 256, 256, 64] + - [1, 364.516] + - - [3584, 704, 1, 3328, 3584, 3584, 3584, 704] + - [3, 6012.64] + - - [2944, 6784, 1, 768, 2944, 2944, 2944, 6784] + - [0, 6346.82] + - - [1856, 2944, 1, 256, 1856, 1856, 1856, 2944] + - [3, 6004.23] + - - [1408, 1024, 1, 1792, 1408, 1408, 1408, 1024] + - [0, 5448.12] + - - [6784, 1856, 1, 256, 6784, 6784, 6784, 1856] + - [3, 6096.91] + - - [64, 1856, 1, 3328, 64, 64, 64, 1856] + - [2, 2595.61] + - - [256, 5888, 1, 768, 256, 256, 256, 5888] + - [3, 5643.76] + - - [704, 1408, 1, 1792, 704, 704, 704, 1408] + - [3, 5597.39] + - - [4288, 256, 1, 768, 4288, 4288, 4288, 256] + - [3, 4888.72] + - - [6784, 128, 1, 256, 6784, 6784, 6784, 128] + - [3, 4250.65] + - - [704, 4, 1, 768, 704, 704, 704, 4] + - [6, 58.3911] + - - [64, 6784, 1, 768, 64, 64, 64, 6784] + - [3, 4315.83] + - - [4, 1856, 1, 768, 4, 4, 4, 1856] + - [6, 150.334] + - - [128, 3584, 1, 3328, 128, 128, 128, 3584] + - [3, 4888.58] + - - [704, 5056, 1, 1792, 704, 704, 704, 5056] + - [0, 6123.83] + - - [448, 256, 1, 1792, 448, 448, 448, 256] + - [2, 2408.39] + - - [5056, 64, 1, 1792, 5056, 5056, 5056, 64] + - [3, 3403.45] + - - [256, 5056, 1, 768, 256, 256, 256, 5056] + - [3, 5717.65] + - - [6784, 448, 1, 768, 6784, 6784, 6784, 448] + - [3, 5948.63] + - - [64, 2944, 1, 3328, 64, 64, 64, 2944] + - [2, 3135.16] + - - [704, 1856, 1, 3328, 704, 704, 704, 1856] + - [3, 6024.47] + - - [4, 64, 1, 256, 4, 4, 4, 64] + - [6, 5.26605] + - - [256, 2368, 1, 1792, 256, 256, 256, 2368] + - [3, 4430.51] + - - [1024, 5056, 1, 256, 1024, 1024, 1024, 5056] + - [3, 5858.68] + - - [448, 2944, 1, 3328, 448, 448, 448, 2944] + - [3, 5141.83] + - - [2368, 1024, 1, 3328, 2368, 2368, 2368, 1024] + - [0, 5811.99] + - - [448, 5056, 1, 3328, 448, 448, 448, 5056] + - [3, 5979.73] + - - [2944, 6784, 1, 256, 2944, 2944, 2944, 6784] + - [3, 6180.3] + - - [2368, 704, 1, 1792, 2368, 2368, 2368, 704] + - [3, 5564.88] + - - [5888, 3584, 1, 3328, 5888, 5888, 5888, 3584] + - [0, 6474.48] + - - [2368, 128, 1, 3328, 2368, 2368, 2368, 128] + - [3, 3259.33] + - - [704, 448, 1, 1792, 704, 704, 704, 448] + - [3, 3324.51] + - - [5888, 2944, 1, 1792, 5888, 5888, 5888, 2944] + - [4, 6482.19] + - - [4, 5888, 1, 256, 4, 4, 4, 5888] + - [6, 286.598] + - - [128, 2944, 1, 768, 128, 128, 128, 2944] + - [3, 3763.86] + - - [5888, 5888, 1, 1792, 5888, 5888, 5888, 5888] + - [4, 6529.83] + - - [2368, 4288, 1, 256, 2368, 2368, 2368, 4288] + - [3, 6130.54] + - - [128, 64, 1, 768, 128, 128, 128, 64] + - [2, 183.438] + - - [1024, 1024, 1, 3328, 1024, 1024, 1024, 1024] + - [0, 5832.51] + - - [4, 4, 1, 768, 4, 4, 4, 4] + - [6, 0.333103] + - - [1856, 3584, 1, 256, 1856, 1856, 1856, 3584] + - [3, 5952.46] + - - [1408, 128, 1, 256, 1408, 1408, 1408, 128] + - [5, 2248.52] + - - [3584, 5888, 1, 256, 3584, 3584, 3584, 5888] + - [3, 6204.05] + - - [2368, 256, 1, 256, 2368, 2368, 2368, 256] + - [3, 3795.29] + - - [2944, 3584, 1, 1792, 2944, 2944, 2944, 3584] + - [3, 6263.5] + - - [3584, 2368, 1, 1792, 3584, 3584, 3584, 2368] + - [3, 6246.68] + - - [4, 1408, 1, 3328, 4, 4, 4, 1408] + - [6, 119.296] + - - [1024, 64, 1, 1792, 1024, 1024, 1024, 64] + - [2, 1370.27] + - - [4288, 1408, 1, 1792, 4288, 4288, 4288, 1408] + - [4, 6226.78] + - - [704, 1408, 1, 256, 704, 704, 704, 1408] + - [3, 4865.92] + - - [1024, 5888, 1, 768, 1024, 1024, 1024, 5888] + - [3, 6195.87] + - - [256, 1408, 1, 256, 256, 256, 256, 1408] + - [3, 3016.15] + - - [2944, 3584, 1, 256, 2944, 2944, 2944, 3584] + - [3, 6064.24] + - - [3584, 1408, 1, 768, 3584, 3584, 3584, 1408] + - [4, 6078.58] + - - [1408, 5056, 1, 768, 1408, 1408, 1408, 5056] + - [4, 6180.91] + - - [64, 4, 1, 256, 64, 64, 64, 4] + - [6, 4.09587] + - - [5888, 1024, 1, 1792, 5888, 5888, 5888, 1024] + - [4, 6241.75] + - - [1408, 5888, 1, 1792, 1408, 1408, 1408, 5888] + - [4, 6489.99] + - - [1024, 5056, 1, 3328, 1024, 1024, 1024, 5056] + - [4, 6418.18] + - - [256, 448, 1, 768, 256, 256, 256, 448] + - [1, 2299.93] + - - [1856, 3584, 1, 1792, 1856, 1856, 1856, 3584] + - [0, 6298.67] + - - [1856, 256, 1, 1792, 1856, 1856, 1856, 256] + - [3, 4990.97] + - - [2944, 6784, 1, 3328, 2944, 2944, 2944, 6784] + - [0, 6451.09] + - - [5888, 4288, 1, 1792, 5888, 5888, 5888, 4288] + - [0, 6443.95] + - - [704, 1856, 1, 256, 704, 704, 704, 1856] + - [3, 5256.76] + - - [2368, 256, 1, 3328, 2368, 2368, 2368, 256] + - [3, 4514.33] + - - [2368, 6784, 1, 1792, 2368, 2368, 2368, 6784] + - [0, 6478.01] + - - [1024, 1024, 1, 1792, 1024, 1024, 1024, 1024] + - [0, 5728.65] + - - [2368, 5888, 1, 3328, 2368, 2368, 2368, 5888] + - [0, 6475.2] + - - [1408, 1408, 1, 1792, 1408, 1408, 1408, 1408] + - [3, 5823.74] + - - [448, 6784, 1, 3328, 448, 448, 448, 6784] + - [0, 6121.63] + - - [128, 4288, 1, 1792, 128, 128, 128, 4288] + - [3, 4032.12] + - - [1408, 4288, 1, 768, 1408, 1408, 1408, 4288] + - [4, 6071.3] + - - [1024, 128, 1, 768, 1024, 1024, 1024, 128] + - [2, 2409.42] + - - [5056, 4, 1, 256, 5056, 5056, 5056, 4] + - [6, 252.313] + - - [4288, 64, 1, 768, 4288, 4288, 4288, 64] + - [3, 2741.07] + - - [448, 64, 1, 3328, 448, 448, 448, 64] + - [2, 646.009] + - - [4, 704, 1, 3328, 4, 4, 4, 704] + - [6, 59.9876] + - - [4288, 1024, 1, 768, 4288, 4288, 4288, 1024] + - [3, 5933.75] + - - [6784, 3584, 1, 768, 6784, 6784, 6784, 3584] + - [0, 6409.17] + - - [704, 5056, 1, 256, 704, 704, 704, 5056] + - [3, 5758.91] + - - [2368, 6784, 1, 3328, 2368, 2368, 2368, 6784] + - [0, 6506.15] + - - [6784, 256, 1, 1792, 6784, 6784, 6784, 256] + - [3, 5796.52] + - - [128, 1408, 1, 256, 128, 128, 128, 1408] + - [5, 2264.85] + - - [6784, 1024, 1, 256, 6784, 6784, 6784, 1024] + - [3, 5878.44] + - - [256, 1856, 1, 3328, 256, 256, 256, 1856] + - [3, 5069.18] + - - [256, 6784, 1, 1792, 256, 256, 256, 6784] + - [3, 5788.54] + - - [704, 4, 1, 3328, 704, 704, 704, 4] + - [6, 59.7891] + - - [6784, 4, 1, 3328, 6784, 6784, 6784, 4] + - [6, 348.225] + - - [2944, 4, 1, 1792, 2944, 2944, 2944, 4] + - [6, 229.371] + - - [2368, 1408, 1, 3328, 2368, 2368, 2368, 1408] + - [3, 6157.71] + - - [6784, 128, 1, 3328, 6784, 6784, 6784, 128] + - [3, 4955.2] + - - [128, 64, 1, 3328, 128, 128, 128, 64] + - [2, 195.35] + - - [64, 5888, 1, 3328, 64, 64, 64, 5888] + - [3, 4038.61] + - - [2944, 4288, 1, 3328, 2944, 2944, 2944, 4288] + - [0, 6324.57] + - - [6784, 3584, 1, 256, 6784, 6784, 6784, 3584] + - [3, 6235.26] + - - [4288, 256, 1, 3328, 4288, 4288, 4288, 256] + - [3, 5073.28] + - - [1408, 2368, 1, 768, 1408, 1408, 1408, 2368] + - [3, 6043.27] + - - [1408, 2944, 1, 256, 1408, 1408, 1408, 2944] + - [3, 5897.59] + - - [1856, 64, 1, 1792, 1856, 1856, 1856, 64] + - [2, 2558.81] + - - [2368, 1024, 1, 1792, 2368, 2368, 2368, 1024] + - [0, 5748.16] + - - [6784, 1408, 1, 1792, 6784, 6784, 6784, 1408] + - [3, 6277.78] + - - [704, 1856, 1, 768, 704, 704, 704, 1856] + - [3, 5806.56] + - - [704, 128, 1, 768, 704, 704, 704, 128] + - [2, 1831.89] + - - [5056, 64, 1, 256, 5056, 5056, 5056, 64] + - [3, 2714.25] + - - [1408, 256, 1, 768, 1408, 1408, 1408, 256] + - [3, 3610.64] + - - [4, 448, 1, 3328, 4, 4, 4, 448] + - [6, 39.6791] + - - [5056, 2944, 1, 256, 5056, 5056, 5056, 2944] + - [3, 6193.94] + - - [1408, 1408, 1, 256, 1408, 1408, 1408, 1408] + - [3, 5331.7] + - - [5056, 5056, 1, 768, 5056, 5056, 5056, 5056] + - [4, 6376.77] + - - [64, 704, 1, 3328, 64, 64, 64, 704] + - [1, 1013.12] + - - [64, 704, 1, 768, 64, 64, 64, 704] + - [2, 945.605] + - - [256, 256, 1, 256, 256, 256, 256, 256] + - [1, 976.243] + - - [64, 64, 1, 1792, 64, 64, 64, 64] + - [2, 99.9875] + - - [3584, 5888, 1, 3328, 3584, 3584, 3584, 5888] + - [0, 6468.08] + - - [704, 2944, 1, 256, 704, 704, 704, 2944] + - [3, 5595.83] + - - [6784, 5056, 1, 768, 6784, 6784, 6784, 5056] + - [3, 6377.66] + - - [448, 64, 1, 768, 448, 448, 448, 64] + - [2, 629.793] + - - [5056, 128, 1, 3328, 5056, 5056, 5056, 128] + - [3, 4818.55] + - - [1408, 2944, 1, 768, 1408, 1408, 1408, 2944] + - [4, 6164.93] + - - [6784, 2944, 1, 1792, 6784, 6784, 6784, 2944] + - [0, 6423.69] + - - [448, 128, 1, 768, 448, 448, 448, 128] + - [2, 1208.38] + - - [5888, 704, 1, 256, 5888, 5888, 5888, 704] + - [3, 5890.32] + - - [5056, 1024, 1, 1792, 5056, 5056, 5056, 1024] + - [4, 6370.97] + - - [2944, 3584, 1, 768, 2944, 2944, 2944, 3584] + - [3, 6219.39] + - - [128, 448, 1, 3328, 128, 128, 128, 448] + - [2, 1267.23] + - - [2368, 1856, 1, 3328, 2368, 2368, 2368, 1856] + - [3, 6240.83] + - - [5888, 2944, 1, 768, 5888, 5888, 5888, 2944] + - [4, 6396.3] + - - [2368, 2944, 1, 1792, 2368, 2368, 2368, 2944] + - [3, 6257.78] + - - [1024, 448, 1, 256, 1024, 1024, 1024, 448] + - [3, 3530.93] + - - [448, 704, 1, 1792, 448, 448, 448, 704] + - [3, 3324.51] + - - [6784, 256, 1, 3328, 6784, 6784, 6784, 256] + - [3, 5834.92] + - - [1408, 448, 1, 256, 1408, 1408, 1408, 448] + - [3, 3913.73] + - - [256, 448, 1, 1792, 256, 256, 256, 448] + - [1, 2385.6] + - - [1024, 4, 1, 256, 1024, 1024, 1024, 4] + - [6, 59.4768] + - - [64, 2368, 1, 256, 64, 64, 64, 2368] + - [5, 1775.42] + - - [704, 2368, 1, 768, 704, 704, 704, 2368] + - [3, 5452.27] + - - [2944, 4, 1, 3328, 2944, 2944, 2944, 4] + - [6, 235.556] + - - [256, 1024, 1, 1792, 256, 256, 256, 1024] + - [3, 4161.16] + - - [4288, 4288, 1, 768, 4288, 4288, 4288, 4288] + - [0, 6299.55] + - - [1856, 1024, 1, 3328, 1856, 1856, 1856, 1024] + - [4, 5625.23] + - - [4, 704, 1, 768, 4, 4, 4, 704] + - [6, 58.9811] + - - [256, 6784, 1, 3328, 256, 256, 256, 6784] + - [3, 5837.54] + - - [1856, 5056, 1, 768, 1856, 1856, 1856, 5056] + - [3, 6289.35] + - - [4, 5888, 1, 1792, 4, 4, 4, 5888] + - [6, 367.347] + - - [4288, 5888, 1, 1792, 4288, 4288, 4288, 5888] + - [0, 6467.08] + - - [1408, 2368, 1, 1792, 1408, 1408, 1408, 2368] + - [3, 6127.59] + - - [4288, 4288, 1, 3328, 4288, 4288, 4288, 4288] + - [0, 6415.65] + - - [1856, 1856, 1, 256, 1856, 1856, 1856, 1856] + - [3, 5589.06] + - - [3584, 448, 1, 1792, 3584, 3584, 3584, 448] + - [3, 5352.4] + - - [3584, 5056, 1, 3328, 3584, 3584, 3584, 5056] + - [0, 6484.15] + - - [2944, 448, 1, 768, 2944, 2944, 2944, 448] + - [3, 5005.17] + - - [2368, 5056, 1, 3328, 2368, 2368, 2368, 5056] + - [3, 6357.52] + - - [256, 1408, 1, 3328, 256, 256, 256, 1408] + - [3, 3865.78] + - - [2944, 256, 1, 1792, 2944, 2944, 2944, 256] + - [3, 5501.56] + - - [704, 704, 1, 1792, 704, 704, 704, 704] + - [3, 5188.06] + - - [5888, 448, 1, 3328, 5888, 5888, 5888, 448] + - [3, 5735.04] + - - [1024, 128, 1, 256, 1024, 1024, 1024, 128] + - [5, 1735.56] + - - [6784, 448, 1, 3328, 6784, 6784, 6784, 448] + - [3, 6070.73] + - - [1856, 704, 1, 1792, 1856, 1856, 1856, 704] + - [3, 5970.72] + - - [5056, 1856, 1, 3328, 5056, 5056, 5056, 1856] + - [3, 6354.51] + - - [6784, 704, 1, 1792, 6784, 6784, 6784, 704] + - [3, 6055.04] + - - [128, 2368, 1, 3328, 128, 128, 128, 2368] + - [3, 3251.55] + - - [3584, 2368, 1, 768, 3584, 3584, 3584, 2368] + - [3, 6194.73] + - - [704, 4288, 1, 3328, 704, 704, 704, 4288] + - [0, 6069.76] + - - [1408, 1856, 1, 3328, 1408, 1408, 1408, 1856] + - [3, 6228.7] + - - [128, 704, 1, 1792, 128, 128, 128, 704] + - [2, 1948.1] + - - [4, 6784, 1, 1792, 4, 4, 4, 6784] + - [6, 341.905] + - - [448, 5888, 1, 768, 448, 448, 448, 5888] + - [3, 5619.12] + - - [5888, 2944, 1, 256, 5888, 5888, 5888, 2944] + - [4, 6159.33] + - - [1024, 1408, 1, 256, 1024, 1024, 1024, 1408] + - [3, 4870.66] + - - [64, 1408, 1, 256, 64, 64, 64, 1408] + - [1, 1319.57] + - - [1408, 1024, 1, 3328, 1408, 1408, 1408, 1024] + - [3, 5606.29] + - - [4288, 4288, 1, 1792, 4288, 4288, 4288, 4288] + - [0, 6386.65] + - - [5888, 1024, 1, 3328, 5888, 5888, 5888, 1024] + - [4, 6283.11] + - - [5888, 256, 1, 3328, 5888, 5888, 5888, 256] + - [3, 5862.13] + - - [256, 5056, 1, 3328, 256, 256, 256, 5056] + - [3, 5959.86] + - - [64, 448, 1, 768, 64, 64, 64, 448] + - [1, 637.903] + - - [6784, 1408, 1, 256, 6784, 6784, 6784, 1408] + - [3, 6077.02] + - - [256, 4288, 1, 1792, 256, 256, 256, 4288] + - [3, 5014.26] + - - [1408, 5888, 1, 3328, 1408, 1408, 1408, 5888] + - [4, 6531.23] + - - [448, 6784, 1, 256, 448, 448, 448, 6784] + - [3, 5659.11] + - - [448, 1408, 1, 3328, 448, 448, 448, 1408] + - [3, 4695.79] + - - [128, 128, 1, 768, 128, 128, 128, 128] + - [2, 357.611] + - - [2368, 2944, 1, 256, 2368, 2368, 2368, 2944] + - [3, 6020.11] + - - [1408, 4, 1, 256, 1408, 1408, 1408, 4] + - [6, 82.8235] + - - [4, 1856, 1, 3328, 4, 4, 4, 1856] + - [6, 155.64] + - - [2944, 64, 1, 1792, 2944, 2944, 2944, 64] + - [1, 3059.08] + - - [2368, 4, 1, 256, 2368, 2368, 2368, 4] + - [6, 139.893] + - - [1856, 2368, 1, 256, 1856, 1856, 1856, 2368] + - [3, 5910.01] + - - [6784, 704, 1, 256, 6784, 6784, 6784, 704] + - [3, 5799.43] + - - [448, 1856, 1, 256, 448, 448, 448, 1856] + - [3, 4087.54] + - - [256, 4288, 1, 768, 256, 256, 256, 4288] + - [3, 4863.65] + - - [448, 3584, 1, 3328, 448, 448, 448, 3584] + - [3, 5397.77] + - - [2944, 2368, 1, 1792, 2944, 2944, 2944, 2368] + - [3, 6256.86] + - - [448, 4288, 1, 1792, 448, 448, 448, 4288] + - [3, 5647.36] + - - [4288, 1856, 1, 256, 4288, 4288, 4288, 1856] + - [3, 6005.22] + - - [1856, 1408, 1, 768, 1856, 1856, 1856, 1408] + - [3, 6095.25] + - - [2368, 704, 1, 256, 2368, 2368, 2368, 704] + - [3, 5121.11] + - - [4, 2368, 1, 256, 4, 4, 4, 2368] + - [6, 131.463] + - - [128, 256, 1, 3328, 128, 128, 128, 256] + - [2, 742.016] + - - [5056, 2944, 1, 3328, 5056, 5056, 5056, 2944] + - [0, 6442.6] + - - [1024, 704, 1, 1792, 1024, 1024, 1024, 704] + - [3, 5221.38] + - - [704, 5056, 1, 768, 704, 704, 704, 5056] + - [3, 6007.34] + - - [448, 5888, 1, 3328, 448, 448, 448, 5888] + - [3, 5735.32] + - - [256, 4, 1, 3328, 256, 256, 256, 4] + - [6, 25.4312] + - - [1856, 4, 1, 1792, 1856, 1856, 1856, 4] + - [6, 149.664] + - - [1408, 704, 1, 1792, 1408, 1408, 1408, 704] + - [3, 5586.95] + - - [704, 128, 1, 256, 704, 704, 704, 128] + - [2, 1314.0] + - - [64, 2368, 1, 3328, 64, 64, 64, 2368] + - [2, 2540.57] + - - [2944, 1856, 1, 768, 2944, 2944, 2944, 1856] + - [3, 6218.42] + - - [4288, 4, 1, 256, 4288, 4288, 4288, 4] + - [6, 225.383] + - - [4, 128, 1, 256, 4, 4, 4, 128] + - [6, 10.2874] + - - [4, 128, 1, 768, 4, 4, 4, 128] + - [6, 13.7519] + - - [6784, 4288, 1, 768, 6784, 6784, 6784, 4288] + - [3, 6379.45] + - - [3584, 4, 1, 3328, 3584, 3584, 3584, 4] + - [6, 283.607] + - - [5056, 5056, 1, 1792, 5056, 5056, 5056, 5056] + - [4, 6500.37] + - - [3584, 5888, 1, 768, 3584, 3584, 3584, 5888] + - [0, 6339.41] + - - [1024, 6784, 1, 256, 1024, 1024, 1024, 6784] + - [3, 5907.37] + - - [5056, 3584, 1, 3328, 5056, 5056, 5056, 3584] + - [0, 6507.68] + - - [2368, 1408, 1, 768, 2368, 2368, 2368, 1408] + - [3, 6047.49] + - - [5888, 2368, 1, 3328, 5888, 5888, 5888, 2368] + - [4, 6333.97] + - - [2368, 5888, 1, 256, 2368, 2368, 2368, 5888] + - [3, 6127.29] + - - [1408, 1024, 1, 768, 1408, 1408, 1408, 1024] + - [3, 5377.01] + - - [6784, 128, 1, 1792, 6784, 6784, 6784, 128] + - [3, 4906.93] + - - [2944, 2368, 1, 256, 2944, 2944, 2944, 2368] + - [3, 6020.11] + - - [1856, 256, 1, 256, 1856, 1856, 1856, 256] + - [3, 4004.9] + - - [6784, 5888, 1, 768, 6784, 6784, 6784, 5888] + - [4, 6494.22] + - - [1856, 5888, 1, 768, 1856, 1856, 1856, 5888] + - [0, 6373.39] + - - [2944, 5888, 1, 1792, 2944, 2944, 2944, 5888] + - [4, 6483.99] + - - [1408, 4, 1, 768, 1408, 1408, 1408, 4] + - [6, 106.943] + - - [704, 5888, 1, 768, 704, 704, 704, 5888] + - [3, 6145.54] + - - [64, 1408, 1, 3328, 64, 64, 64, 1408] + - [1, 2020.18] + - - [6784, 64, 1, 256, 6784, 6784, 6784, 64] + - [3, 3606.92] + - - [64, 256, 1, 256, 64, 64, 64, 256] + - [1, 274.856] + - - [3584, 2944, 1, 768, 3584, 3584, 3584, 2944] + - [3, 6216.92] + - - [4288, 704, 1, 256, 4288, 4288, 4288, 704] + - [3, 5627.04] + - - [3584, 704, 1, 768, 3584, 3584, 3584, 704] + - [3, 5885.08] + - - [6784, 5056, 1, 1792, 6784, 6784, 6784, 5056] + - [4, 6446.5] + - - [64, 128, 1, 768, 64, 64, 64, 128] + - [1, 179.561] + - - [5056, 5888, 1, 768, 5056, 5056, 5056, 5888] + - [0, 6488.48] + - - [4, 4288, 1, 1792, 4, 4, 4, 4288] + - [6, 317.712] + - - [128, 5888, 1, 768, 128, 128, 128, 5888] + - [3, 5268.91] + - - [4, 4288, 1, 3328, 4, 4, 4, 4288] + - [6, 326.382] + - - [2944, 1408, 1, 1792, 2944, 2944, 2944, 1408] + - [4, 6327.01] + - - [5056, 4, 1, 3328, 5056, 5056, 5056, 4] + - [6, 359.984] + - - [64, 5056, 1, 256, 64, 64, 64, 5056] + - [3, 2707.68] + - - [448, 4, 1, 256, 448, 448, 448, 4] + - [6, 27.6473] + - - [128, 4, 1, 768, 128, 128, 128, 4] + - [6, 13.507] + - - [4, 5056, 1, 256, 4, 4, 4, 5056] + - [6, 252.32] + - - [4, 1856, 1, 1792, 4, 4, 4, 1856] + - [6, 150.039] + - - [5888, 448, 1, 256, 5888, 5888, 5888, 448] + - [3, 5365.54] + - - [256, 5056, 1, 1792, 256, 256, 256, 5056] + - [3, 5897.86] + - - [4288, 5056, 1, 256, 4288, 4288, 4288, 5056] + - [3, 6219.86] + - - [1856, 6784, 1, 768, 1856, 1856, 1856, 6784] + - [3, 6226.02] + - - [6784, 2944, 1, 256, 6784, 6784, 6784, 2944] + - [3, 6180.31] + - - [4288, 128, 1, 3328, 4288, 4288, 4288, 128] + - [3, 4085.95] + - - [704, 1024, 1, 256, 704, 704, 704, 1024] + - [3, 4448.8] + - - [4, 2368, 1, 768, 4, 4, 4, 2368] + - [6, 186.698] + - - [128, 1024, 1, 3328, 128, 128, 128, 1024] + - [1, 2587.27] + - - [5056, 4, 1, 768, 5056, 5056, 5056, 4] + - [6, 325.585] + - - [1024, 5056, 1, 768, 1024, 1024, 1024, 5056] + - [4, 6156.35] + - - [5888, 5056, 1, 3328, 5888, 5888, 5888, 5056] + - [4, 6574.83] + - - [5888, 2368, 1, 768, 5888, 5888, 5888, 2368] + - [3, 6255.89] + - - [5056, 5888, 1, 3328, 5056, 5056, 5056, 5888] + - [0, 6585.02] + - - [3584, 2368, 1, 3328, 3584, 3584, 3584, 2368] + - [3, 6262.87] + - - [2944, 64, 1, 3328, 2944, 2944, 2944, 64] + - [5, 3194.31] + - - [4288, 5056, 1, 3328, 4288, 4288, 4288, 5056] + - [0, 6489.99] + - - [64, 6784, 1, 3328, 64, 64, 64, 6784] + - [3, 4645.42] + - - [3584, 448, 1, 3328, 3584, 3584, 3584, 448] + - [3, 5397.77] + - - [6784, 4, 1, 1792, 6784, 6784, 6784, 4] + - [6, 354.267] + - - [1856, 6784, 1, 256, 1856, 1856, 1856, 6784] + - [3, 6096.06] + - - [448, 1024, 1, 3328, 448, 448, 448, 1024] + - [3, 4910.71] + - - [5056, 1856, 1, 256, 5056, 5056, 5056, 1856] + - [3, 6123.49] + - - [256, 64, 1, 256, 256, 256, 256, 64] + - [5, 255.05] + - - [1024, 4, 1, 3328, 1024, 1024, 1024, 4] + - [6, 81.8959] + - - [448, 5888, 1, 1792, 448, 448, 448, 5888] + - [3, 5705.19] + - - [2368, 5056, 1, 256, 2368, 2368, 2368, 5056] + - [3, 6152.59] + - - [704, 6784, 1, 1792, 704, 704, 704, 6784] + - [3, 6062.67] + - - [4288, 5888, 1, 768, 4288, 4288, 4288, 5888] + - [0, 6398.63] + - - [4, 64, 1, 768, 4, 4, 4, 64] + - [6, 5.52945] + - - [128, 4288, 1, 256, 128, 128, 128, 4288] + - [3, 3430.07] + - - [2944, 128, 1, 768, 2944, 2944, 2944, 128] + - [3, 3738.65] + - - [5888, 6784, 1, 1792, 5888, 5888, 5888, 6784] + - [4, 6561.44] + - - [704, 5888, 1, 1792, 704, 704, 704, 5888] + - [0, 6275.53] + - - [448, 6784, 1, 768, 448, 448, 448, 6784] + - [3, 5946.39] + - - [5056, 64, 1, 3328, 5056, 5056, 5056, 64] + - [3, 3467.11] + - - [1024, 5888, 1, 256, 1024, 1024, 1024, 5888] + - [3, 5869.51] + - - [2944, 1024, 1, 1792, 2944, 2944, 2944, 1024] + - [4, 6037.63] + - - [256, 1024, 1, 768, 256, 256, 256, 1024] + - [3, 3801.16] + - - [64, 1856, 1, 768, 64, 64, 64, 1856] + - [2, 2268.01] + - - [5056, 4288, 1, 256, 5056, 5056, 5056, 4288] + - [3, 6226.06] + - - [5056, 6784, 1, 3328, 5056, 5056, 5056, 6784] + - [0, 6560.53] + - - [5888, 1408, 1, 768, 5888, 5888, 5888, 1408] + - [4, 6367.25] + - - [1024, 4288, 1, 1792, 1024, 1024, 1024, 4288] + - [3, 6189.37] + - - [704, 448, 1, 3328, 704, 704, 704, 448] + - [3, 3393.9] + - - [5056, 1024, 1, 768, 5056, 5056, 5056, 1024] + - [0, 6211.21] + - - [2368, 128, 1, 1792, 2368, 2368, 2368, 128] + - [3, 3189.43] + - - [2944, 5056, 1, 1792, 2944, 2944, 2944, 5056] + - [0, 6387.31] + - - [5888, 5888, 1, 256, 5888, 5888, 5888, 5888] + - [4, 6291.28] + - - [2944, 1408, 1, 3328, 2944, 2944, 2944, 1408] + - [4, 6378.99] + - - [1408, 4288, 1, 256, 1408, 1408, 1408, 4288] + - [3, 5851.11] + - - [64, 3584, 1, 3328, 64, 64, 64, 3584] + - [1, 3785.86] + - - [2944, 2368, 1, 3328, 2944, 2944, 2944, 2368] + - [3, 6275.71] + - - [1024, 448, 1, 768, 1024, 1024, 1024, 448] + - [3, 4355.51] + - - [5056, 1408, 1, 1792, 5056, 5056, 5056, 1408] + - [4, 6327.92] + - - [6784, 5888, 1, 256, 6784, 6784, 6784, 5888] + - [4, 6327.33] + - - [256, 5888, 1, 1792, 256, 256, 256, 5888] + - [3, 5819.39] + - - [128, 1856, 1, 768, 128, 128, 128, 1856] + - [3, 3483.77] + - - [2368, 448, 1, 3328, 2368, 2368, 2368, 448] + - [3, 4903.92] + - - [1408, 256, 1, 3328, 1408, 1408, 1408, 256] + - [2, 3875.96] + - - [128, 2944, 1, 256, 128, 128, 128, 2944] + - [3, 3199.74] + - - [128, 5056, 1, 3328, 128, 128, 128, 5056] + - [3, 4810.58] + - - [3584, 4, 1, 768, 3584, 3584, 3584, 4] + - [6, 251.494] + - - [704, 256, 1, 256, 704, 704, 704, 256] + - [5, 2332.7] + - - [4, 2944, 1, 3328, 4, 4, 4, 2944] + - [6, 238.101] + - - [2944, 256, 1, 3328, 2944, 2944, 2944, 256] + - [3, 5599.42] + - - [4288, 704, 1, 3328, 4288, 4288, 4288, 704] + - [3, 6022.59] + - - [448, 4288, 1, 768, 448, 448, 448, 4288] + - [3, 5535.47] + - - [5888, 5056, 1, 256, 5888, 5888, 5888, 5056] + - [3, 6240.15] + - - [5888, 448, 1, 1792, 5888, 5888, 5888, 448] + - [3, 5711.31] + - - [1856, 2944, 1, 3328, 1856, 1856, 1856, 2944] + - [3, 6303.32] + - - [5888, 4, 1, 1792, 5888, 5888, 5888, 4] + - [6, 373.121] + - - [64, 4, 1, 768, 64, 64, 64, 4] + - [6, 5.32957] + - - [5888, 6784, 1, 3328, 5888, 5888, 5888, 6784] + - [4, 6583.91] + - - [256, 128, 1, 1792, 256, 256, 256, 128] + - [2, 747.833] + - - [6784, 1856, 1, 768, 6784, 6784, 6784, 1856] + - [3, 6226.91] + - - [1408, 128, 1, 3328, 1408, 1408, 1408, 128] + - [1, 3030.27] + - - [2944, 2944, 1, 3328, 2944, 2944, 2944, 2944] + - [0, 6352.97] + - - [2368, 4288, 1, 1792, 2368, 2368, 2368, 4288] + - [3, 6334.15] + - - [256, 704, 1, 256, 256, 256, 256, 704] + - [5, 2240.38] + - - [3584, 1408, 1, 3328, 3584, 3584, 3584, 1408] + - [4, 6265.21] + - - [4288, 5888, 1, 3328, 4288, 4288, 4288, 5888] + - [0, 6494.14] + - - [256, 5888, 1, 3328, 256, 256, 256, 5888] + - [3, 5871.79] + - - [5056, 1856, 1, 1792, 5056, 5056, 5056, 1856] + - [3, 6336.28] + - - [2368, 1856, 1, 1792, 2368, 2368, 2368, 1856] + - [3, 6213.97] + - - [2368, 1408, 1, 1792, 2368, 2368, 2368, 1408] + - [3, 6125.26] + - - [1408, 448, 1, 768, 1408, 1408, 1408, 448] + - [3, 4445.83] + - - [1024, 2368, 1, 3328, 1024, 1024, 1024, 2368] + - [0, 5786.67] + - - [256, 4288, 1, 256, 256, 256, 256, 4288] + - [3, 4370.56] + - - [448, 4, 1, 3328, 448, 448, 448, 4] + - [6, 39.5231] + - - [448, 1856, 1, 1792, 448, 448, 448, 1856] + - [3, 4700.82] + - - [448, 5056, 1, 768, 448, 448, 448, 5056] + - [3, 5827.24] + - - [2944, 64, 1, 768, 2944, 2944, 2944, 64] + - [1, 2831.08] + - - [4, 1408, 1, 1792, 4, 4, 4, 1408] + - [6, 113.823] + - - [4288, 6784, 1, 1792, 4288, 4288, 4288, 6784] + - [0, 6525.85] + - - [2944, 704, 1, 256, 2944, 2944, 2944, 704] + - [3, 5578.38] + - - [64, 5056, 1, 768, 64, 64, 64, 5056] + - [3, 3219.62] + - - [1856, 5056, 1, 1792, 1856, 1856, 1856, 5056] + - [3, 6337.87] + - - [1856, 704, 1, 768, 1856, 1856, 1856, 704] + - [3, 5801.59] + - - [2944, 4, 1, 768, 2944, 2944, 2944, 4] + - [6, 209.776] + - - [4, 6784, 1, 256, 4, 4, 4, 6784] + - [6, 296.771] + - - [5056, 5888, 1, 1792, 5056, 5056, 5056, 5888] + - [4, 6560.0] + - - [256, 5888, 1, 256, 256, 256, 256, 5888] + - [3, 5018.48] + - - [1856, 448, 1, 3328, 1856, 1856, 1856, 448] + - [3, 4755.71] + - - [1408, 6784, 1, 1792, 1408, 1408, 1408, 6784] + - [3, 6278.63] + - - [1024, 2944, 1, 3328, 1024, 1024, 1024, 2944] + - [4, 6100.29] + - - [6784, 1408, 1, 768, 6784, 6784, 6784, 1408] + - [3, 6229.15] + - - [1856, 1408, 1, 256, 1856, 1856, 1856, 1408] + - [3, 5763.34] + - - [4288, 1024, 1, 3328, 4288, 4288, 4288, 1024] + - [0, 5987.66] + - - [1024, 1024, 1, 256, 1024, 1024, 1024, 1024] + - [0, 4622.18] + - - [64, 5888, 1, 1792, 64, 64, 64, 5888] + - [3, 3958.34] + - - [2368, 3584, 1, 3328, 2368, 2368, 2368, 3584] + - [3, 6264.92] + - - [5888, 2368, 1, 1792, 5888, 5888, 5888, 2368] + - [3, 6297.51] + - - [1024, 2944, 1, 1792, 1024, 1024, 1024, 2944] + - [4, 6036.13] + - - [5888, 4, 1, 3328, 5888, 5888, 5888, 4] + - [6, 368.682] + - - [4, 4288, 1, 768, 4, 4, 4, 4288] + - [6, 292.48] + - - [1024, 3584, 1, 768, 1024, 1024, 1024, 3584] + - [3, 6080.17] + - - [704, 2944, 1, 3328, 704, 704, 704, 2944] + - [0, 6131.18] + - - [3584, 6784, 1, 768, 3584, 3584, 3584, 6784] + - [4, 6379.01] + - - [4, 4, 1, 3328, 4, 4, 4, 4] + - [6, 0.393663] + - - [128, 1408, 1, 1792, 128, 128, 128, 1408] + - [3, 2898.84] + - - [4288, 1408, 1, 768, 4288, 4288, 4288, 1408] + - [4, 6089.61] + - - [4288, 3584, 1, 768, 4288, 4288, 4288, 3584] + - [4, 6393.53] + - - [704, 2944, 1, 768, 704, 704, 704, 2944] + - [3, 5958.92] + - - [128, 3584, 1, 1792, 128, 128, 128, 3584] + - [3, 4795.95] + - - [4288, 704, 1, 1792, 4288, 4288, 4288, 704] + - [3, 5998.17] + - - [5888, 448, 1, 768, 5888, 5888, 5888, 448] + - [3, 5627.21] + - - [1024, 128, 1, 3328, 1024, 1024, 1024, 128] + - [2, 2644.2] + - - [704, 3584, 1, 3328, 704, 704, 704, 3584] + - [0, 6042.77] + - - [2944, 256, 1, 768, 2944, 2944, 2944, 256] + - [3, 5268.89] + - - [4, 1024, 1, 3328, 4, 4, 4, 1024] + - [6, 82.8174] + - - [5888, 2944, 1, 3328, 5888, 5888, 5888, 2944] + - [4, 6508.11] + - - [2944, 1856, 1, 1792, 2944, 2944, 2944, 1856] + - [3, 6280.97] + - - [4, 3584, 1, 768, 4, 4, 4, 3584] + - [6, 261.677] + - - [448, 2944, 1, 1792, 448, 448, 448, 2944] + - [3, 5099.29] + - - [1024, 4288, 1, 256, 1024, 1024, 1024, 4288] + - [3, 5739.26] + - - [4, 5056, 1, 3328, 4, 4, 4, 5056] + - [6, 367.706] + - - [2944, 128, 1, 256, 2944, 2944, 2944, 128] + - [3, 3145.64] + - - [64, 1408, 1, 1792, 64, 64, 64, 1408] + - [1, 1932.57] + - - [1856, 5056, 1, 3328, 1856, 1856, 1856, 5056] + - [3, 6354.32] + - - [448, 704, 1, 3328, 448, 448, 448, 704] + - [3, 3382.56] + - - [4288, 1856, 1, 1792, 4288, 4288, 4288, 1856] + - [3, 6231.27] + - - [1408, 5056, 1, 3328, 1408, 1408, 1408, 5056] + - [4, 6373.08] + - - [256, 256, 1, 768, 256, 256, 256, 256] + - [1, 1395.19] + - - [2368, 1856, 1, 768, 2368, 2368, 2368, 1856] + - [3, 6145.14] + - - [1856, 128, 1, 1792, 1856, 1856, 1856, 128] + - [1, 3833.95] + - - [3584, 4, 1, 1792, 3584, 3584, 3584, 4] + - [6, 273.724] + - - [1408, 5888, 1, 256, 1408, 1408, 1408, 5888] + - [3, 6066.17] + - - [3584, 128, 1, 1792, 3584, 3584, 3584, 128] + - [3, 4708.45] + - - [4288, 64, 1, 3328, 4288, 4288, 4288, 64] + - [3, 2941.86] + - - [4288, 2944, 1, 3328, 4288, 4288, 4288, 2944] + - [0, 6332.2] + - - [1856, 2368, 1, 3328, 1856, 1856, 1856, 2368] + - [3, 6241.62] + - - [1024, 1408, 1, 3328, 1024, 1024, 1024, 1408] + - [3, 5605.32] + - - [5056, 256, 1, 256, 5056, 5056, 5056, 256] + - [3, 5256.31] + - - [64, 448, 1, 3328, 64, 64, 64, 448] + - [1, 645.684] + - - [1024, 3584, 1, 256, 1024, 1024, 1024, 3584] + - [3, 5741.62] + - - [5056, 1408, 1, 3328, 5056, 5056, 5056, 1408] + - [4, 6380.46] + - - [1856, 64, 1, 768, 1856, 1856, 1856, 64] + - [2, 2289.07] + - - [5056, 1024, 1, 3328, 5056, 5056, 5056, 1024] + - [4, 6423.15] + - - [2368, 3584, 1, 768, 2368, 2368, 2368, 3584] + - [3, 6202.15] + - - [256, 1408, 1, 1792, 256, 256, 256, 1408] + - [3, 3784.6] + - - [5888, 1408, 1, 3328, 5888, 5888, 5888, 1408] + - [4, 6527.23] + - - [704, 2368, 1, 3328, 704, 704, 704, 2368] + - [3, 5600.97] + - - [5888, 4288, 1, 3328, 5888, 5888, 5888, 4288] + - [0, 6483.94] + - - [1408, 2368, 1, 3328, 1408, 1408, 1408, 2368] + - [3, 6157.45] + - - [704, 4288, 1, 768, 704, 704, 704, 4288] + - [3, 5914.13] + - - [5056, 5056, 1, 256, 5056, 5056, 5056, 5056] + - [3, 6227.97] + - - [448, 128, 1, 3328, 448, 448, 448, 128] + - [1, 1267.23] + - - [128, 448, 1, 1792, 128, 128, 128, 448] + - [1, 1221.15] + - - [1408, 4, 1, 3328, 1408, 1408, 1408, 4] + - [6, 117.251] + - - [1024, 1856, 1, 1792, 1024, 1024, 1024, 1856] + - [3, 5563.49] + - - [4, 448, 1, 256, 4, 4, 4, 448] + - [6, 27.5246] + - - [1024, 3584, 1, 3328, 1024, 1024, 1024, 3584] + - [0, 6330.9] + - - [1856, 4288, 1, 768, 1856, 1856, 1856, 4288] + - [3, 6183.44] + - - [1024, 4, 1, 768, 1024, 1024, 1024, 4] + - [6, 77.0709] + - - [5056, 704, 1, 256, 5056, 5056, 5056, 704] + - [3, 5756.22] + - - [4, 448, 1, 768, 4, 4, 4, 448] + - [6, 38.6258] + - - [256, 128, 1, 256, 256, 256, 256, 128] + - [5, 514.733] + - - [64, 5056, 1, 1792, 64, 64, 64, 5056] + - [3, 3410.86] + - - [4, 704, 1, 1792, 4, 4, 4, 704] + - [6, 66.0745] + - - [4288, 6784, 1, 768, 4288, 4288, 4288, 6784] + - [0, 6455.23] + - - [5888, 256, 1, 256, 5888, 5888, 5888, 256] + - [3, 5062.36] + - - [1856, 4, 1, 768, 1856, 1856, 1856, 4] + - [6, 140.971] + - - [3584, 64, 1, 256, 3584, 3584, 3584, 64] + - [2, 2607.59] + - - [2368, 2368, 1, 3328, 2368, 2368, 2368, 2368] + - [0, 6276.9] + - - [256, 4, 1, 1792, 256, 256, 256, 4] + - [6, 28.7045] + - - [128, 704, 1, 256, 128, 128, 128, 704] + - [5, 1390.27] + - - [6784, 704, 1, 768, 6784, 6784, 6784, 704] + - [3, 6001.16] + - - [1408, 64, 1, 1792, 1408, 1408, 1408, 64] + - [2, 1920.64] + - - [2368, 4, 1, 3328, 2368, 2368, 2368, 4] + - [6, 192.817] + - - [5056, 1024, 1, 256, 5056, 5056, 5056, 1024] + - [0, 5860.61] + - - [2368, 3584, 1, 256, 2368, 2368, 2368, 3584] + - [3, 6036.22] + - - [1024, 704, 1, 256, 1024, 1024, 1024, 704] + - [3, 4159.19] + - - [128, 128, 1, 3328, 128, 128, 128, 128] + - [2, 358.016] + - - [4288, 1024, 1, 256, 4288, 4288, 4288, 1024] + - [3, 5765.41] + - - [1024, 256, 1, 3328, 1024, 1024, 1024, 256] + - [3, 4326.71] + - - [448, 1024, 1, 768, 448, 448, 448, 1024] + - [3, 4547.05] + - - [1408, 1408, 1, 768, 1408, 1408, 1408, 1408] + - [3, 5710.93] + - - [3584, 1024, 1, 3328, 3584, 3584, 3584, 1024] + - [0, 6356.53] + - - [704, 256, 1, 1792, 704, 704, 704, 256] + - [2, 2953.83] + - - [448, 4, 1, 1792, 448, 448, 448, 4] + - [6, 42.2524] + - - [128, 2368, 1, 768, 128, 128, 128, 2368] + - [3, 3030.37] + - - [448, 448, 1, 768, 448, 448, 448, 448] + - [5, 3055.56] + - - [1024, 704, 1, 768, 1024, 1024, 1024, 704] + - [3, 4959.56] + - - [128, 1408, 1, 768, 128, 128, 128, 1408] + - [3, 2680.79] + - - [5056, 3584, 1, 256, 5056, 5056, 5056, 3584] + - [0, 6164.33] + - - [1408, 4, 1, 1792, 1408, 1408, 1408, 4] + - [6, 116.45] + - - [4, 256, 1, 1792, 4, 4, 4, 256] + - [6, 29.3855] + - - [5056, 5888, 1, 256, 5056, 5056, 5056, 5888] + - [4, 6283.6] + - - [1408, 1856, 1, 1792, 1408, 1408, 1408, 1856] + - [3, 6193.59] + - - [2944, 448, 1, 1792, 2944, 2944, 2944, 448] + - [3, 5101.73] + - - [1024, 2944, 1, 256, 1024, 1024, 1024, 2944] + - [3, 5506.54] + - - [704, 64, 1, 1792, 704, 704, 704, 64] + - [1, 981.067] + - - [3584, 256, 1, 768, 3584, 3584, 3584, 256] + - [3, 4977.72] + - - [6784, 1024, 1, 3328, 6784, 6784, 6784, 1024] + - [4, 6232.61] + - - [256, 3584, 1, 1792, 256, 256, 256, 3584] + - [3, 5165.38] + - - [5056, 704, 1, 768, 5056, 5056, 5056, 704] + - [3, 6010.28] + - - [1024, 4288, 1, 768, 1024, 1024, 1024, 4288] + - [3, 6076.31] + - - [2368, 5056, 1, 1792, 2368, 2368, 2368, 5056] + - [3, 6341.87] + - - [5888, 4, 1, 256, 5888, 5888, 5888, 4] + - [6, 288.636] + - - [64, 2944, 1, 768, 64, 64, 64, 2944] + - [1, 2778.72] + - - [5888, 3584, 1, 256, 5888, 5888, 5888, 3584] + - [3, 6198.78] + - - [1408, 1024, 1, 256, 1408, 1408, 1408, 1024] + - [3, 4992.68] + - - [5888, 5888, 1, 3328, 5888, 5888, 5888, 5888] + - [4, 6549.7] + - - [2368, 4, 1, 1792, 2368, 2368, 2368, 4] + - [6, 187.975] + - - [5056, 128, 1, 256, 5056, 5056, 5056, 128] + - [3, 4000.99] + - - [5056, 5056, 1, 3328, 5056, 5056, 5056, 5056] + - [4, 6533.0] + - - [2944, 4288, 1, 256, 2944, 2944, 2944, 4288] + - [3, 6099.96] + - - [3584, 6784, 1, 1792, 3584, 3584, 3584, 6784] + - [0, 6468.9] + - - [1856, 1408, 1, 3328, 1856, 1856, 1856, 1408] + - [0, 6265.94] + - - [1024, 1856, 1, 3328, 1024, 1024, 1024, 1856] + - [3, 5613.76] + - - [448, 4288, 1, 3328, 448, 448, 448, 4288] + - [3, 5685.09] + - - [2368, 6784, 1, 256, 2368, 2368, 2368, 6784] + - [3, 6160.39] + - - [1856, 5888, 1, 3328, 1856, 1856, 1856, 5888] + - [0, 6503.71] + - - [5888, 128, 1, 256, 5888, 5888, 5888, 128] + - [3, 4593.65] + - - [448, 5888, 1, 256, 448, 448, 448, 5888] + - [3, 5356.07] + - - [4, 5888, 1, 768, 4, 4, 4, 5888] + - [6, 362.825] + - - [128, 5888, 1, 1792, 128, 128, 128, 5888] + - [3, 5516.53] + - - [1024, 1408, 1, 1792, 1024, 1024, 1024, 1408] + - [3, 5538.96] + - - [64, 256, 1, 768, 64, 64, 64, 256] + - [2, 363.736] + - - [3584, 2944, 1, 3328, 3584, 3584, 3584, 2944] + - [3, 6279.28] + - - [2368, 64, 1, 3328, 2368, 2368, 2368, 64] + - [2, 2546.28] + - - [3584, 6784, 1, 3328, 3584, 3584, 3584, 6784] + - [0, 6506.07] + - - [64, 128, 1, 256, 64, 64, 64, 128] + - [1, 136.769] + - - [5056, 2368, 1, 3328, 5056, 5056, 5056, 2368] + - [3, 6356.32] + - - [448, 1024, 1, 256, 448, 448, 448, 1024] + - [3, 3820.26] + - - [5888, 64, 1, 1792, 5888, 5888, 5888, 64] + - [3, 3949.76] + - - [1856, 4, 1, 3328, 1856, 1856, 1856, 4] + - [6, 152.3] + - - [2944, 5056, 1, 768, 2944, 2944, 2944, 5056] + - [3, 6329.09] + - - [1408, 5056, 1, 256, 1408, 1408, 1408, 5056] + - [3, 5939.65] + - - [704, 2944, 1, 1792, 704, 704, 704, 2944] + - [3, 6084.02] + - - [128, 1856, 1, 256, 128, 128, 128, 1856] + - [3, 2727.6] + - - [1856, 6784, 1, 1792, 1856, 1856, 1856, 6784] + - [0, 6294.34] + - - [1856, 1024, 1, 256, 1856, 1856, 1856, 1024] + - [0, 5002.36] + - - [64, 4288, 1, 256, 64, 64, 64, 4288] + - [3, 2285.3] + - - [1408, 3584, 1, 768, 1408, 1408, 1408, 3584] + - [4, 6072.23] + - - [3584, 1408, 1, 256, 3584, 3584, 3584, 1408] + - [3, 5822.85] + - - [6784, 448, 1, 256, 6784, 6784, 6784, 448] + - [3, 5665.21] + - - [448, 256, 1, 768, 448, 448, 448, 256] + - [2, 2286.64] + - - [256, 4, 1, 256, 256, 256, 256, 4] + - [6, 19.9934] + - - [1024, 704, 1, 3328, 1024, 1024, 1024, 704] + - [3, 5304.21] + - - [5888, 4288, 1, 768, 5888, 5888, 5888, 4288] + - [4, 6351.58] + - - [4288, 64, 1, 256, 4288, 4288, 4288, 64] + - [3, 2318.86] + - - [704, 1856, 1, 1792, 704, 704, 704, 1856] + - [3, 5969.59] + - - [64, 4, 1, 3328, 64, 64, 64, 4] + - [6, 6.48688] + - - [5056, 2944, 1, 768, 5056, 5056, 5056, 2944] + - [0, 6334.54] + - - [1856, 256, 1, 3328, 1856, 1856, 1856, 256] + - [3, 5095.8] + - - [3584, 704, 1, 1792, 3584, 3584, 3584, 704] + - [3, 5975.92] + - - [704, 4, 1, 1792, 704, 704, 704, 4] + - [6, 63.3703] + - - [1856, 1408, 1, 1792, 1856, 1856, 1856, 1408] + - [3, 6196.62] + - - [256, 1024, 1, 3328, 256, 256, 256, 1024] + - [3, 4326.71] + - - [704, 128, 1, 1792, 704, 704, 704, 128] + - [1, 1963.91] + - - [1856, 2944, 1, 768, 1856, 1856, 1856, 2944] + - [3, 6219.11] + - - [3584, 128, 1, 768, 3584, 3584, 3584, 128] + - [3, 4367.52] + - - [2368, 6784, 1, 768, 2368, 2368, 2368, 6784] + - [0, 6400.44] + - - [5056, 128, 1, 768, 5056, 5056, 5056, 128] + - [3, 4518.29] + - - [1856, 4288, 1, 3328, 1856, 1856, 1856, 4288] + - [4, 6259.76] + - - [128, 256, 1, 768, 128, 128, 128, 256] + - [2, 712.217] + - - [1024, 256, 1, 256, 1024, 1024, 1024, 256] + - [3, 2885.16] + - - [5056, 2368, 1, 1792, 5056, 5056, 5056, 2368] + - [3, 6342.29] + - - [64, 2368, 1, 768, 64, 64, 64, 2368] + - [2, 2270.57] + - - [256, 128, 1, 3328, 256, 256, 256, 128] + - [1, 730.238] + - - [1408, 6784, 1, 3328, 1408, 1408, 1408, 6784] + - [3, 6294.98] + - - [6784, 1856, 1, 3328, 6784, 6784, 6784, 1856] + - [0, 6305.56] + - - [2368, 256, 1, 1792, 2368, 2368, 2368, 256] + - [3, 4438.55] + - - [1856, 2368, 1, 1792, 1856, 1856, 1856, 2368] + - [3, 6217.25] + - - [704, 448, 1, 768, 704, 704, 704, 448] + - [3, 3159.31] + - - [4, 64, 1, 3328, 4, 4, 4, 64] + - [6, 6.58345] + - - [1408, 4288, 1, 1792, 1408, 1408, 1408, 4288] + - [4, 6222.27] + - - [64, 4288, 1, 3328, 64, 64, 64, 4288] + - [3, 2946.08] + - - [704, 1024, 1, 1792, 704, 704, 704, 1024] + - [3, 5260.77] + - - [1024, 2368, 1, 256, 1024, 1024, 1024, 2368] + - [3, 5273.75] + - - [128, 5888, 1, 256, 128, 128, 128, 5888] + - [3, 4482.92] + - - [3584, 64, 1, 1792, 3584, 3584, 3584, 64] + - [2, 3674.77] + - - [64, 1024, 1, 768, 64, 64, 64, 1024] + - [1, 1255.92] + - - [1024, 4288, 1, 3328, 1024, 1024, 1024, 4288] + - [3, 6223.21] + - - [448, 3584, 1, 768, 448, 448, 448, 3584] + - [3, 5231.54] + - - [3584, 5056, 1, 256, 3584, 3584, 3584, 5056] + - [3, 6151.01] + - - [1024, 448, 1, 3328, 1024, 1024, 1024, 448] + - [3, 4864.35] + - - [256, 1856, 1, 768, 256, 256, 256, 1856] + - [3, 4599.51] + - - [6784, 704, 1, 3328, 6784, 6784, 6784, 704] + - [3, 6079.43] + - - [5056, 1408, 1, 768, 5056, 5056, 5056, 1408] + - [4, 6204.81] + - - [256, 64, 1, 1792, 256, 256, 256, 64] + - [2, 393.987] + - - [448, 5056, 1, 256, 448, 448, 448, 5056] + - [3, 5481.73] + - - [2368, 448, 1, 1792, 2368, 2368, 2368, 448] + - [3, 4856.87] + - - [128, 4288, 1, 768, 128, 128, 128, 4288] + - [3, 3878.99] + - - [5888, 128, 1, 768, 5888, 5888, 5888, 128] + - [3, 5268.89] + - - [704, 3584, 1, 1792, 704, 704, 704, 3584] + - [0, 5983.54] + - - [1024, 2368, 1, 768, 1024, 1024, 1024, 2368] + - [3, 5629.19] + - - [5056, 448, 1, 3328, 5056, 5056, 5056, 448] + - [3, 5979.03] + - - [3584, 256, 1, 256, 3584, 3584, 3584, 256] + - [3, 4523.25] + - - [448, 2944, 1, 256, 448, 448, 448, 2944] + - [3, 4641.6] + - - [6784, 64, 1, 768, 6784, 6784, 6784, 64] + - [3, 4303.46] + - - [6784, 6784, 1, 3328, 6784, 6784, 6784, 6784] + - [4, 6723.28] + - - [1408, 2944, 1, 1792, 1408, 1408, 1408, 2944] + - [4, 6319.43] + - - [128, 1408, 1, 3328, 128, 128, 128, 1408] + - [3, 2996.63] + - - [4288, 448, 1, 768, 4288, 4288, 4288, 448] + - [3, 5550.89] + - - [4, 3584, 1, 1792, 4, 4, 4, 3584] + - [6, 279.233] + - - [448, 2368, 1, 768, 448, 448, 448, 2368] + - [3, 4702.32] + - - [5888, 64, 1, 256, 5888, 5888, 5888, 64] + - [3, 3007.61] + - - [64, 5888, 1, 256, 64, 64, 64, 5888] + - [3, 3191.91] + - - [4, 256, 1, 3328, 4, 4, 4, 256] + - [6, 26.0357] + - - [448, 5056, 1, 1792, 448, 448, 448, 5056] + - [3, 5940.54] + - - [1024, 5056, 1, 1792, 1024, 1024, 1024, 5056] + - [4, 6347.72] + - - [2368, 4288, 1, 768, 2368, 2368, 2368, 4288] + - [3, 6288.0] + - - [4, 1408, 1, 768, 4, 4, 4, 1408] + - [6, 115.625] + - - [3584, 6784, 1, 256, 3584, 3584, 3584, 6784] + - [3, 6224.64] + - - [6784, 5888, 1, 1792, 6784, 6784, 6784, 5888] + - [4, 6562.02] + - - [3584, 2368, 1, 256, 3584, 3584, 3584, 2368] + - [3, 6018.87] + - - [5056, 4288, 1, 3328, 5056, 5056, 5056, 4288] + - [0, 6490.9] + - - [2944, 5056, 1, 3328, 2944, 2944, 2944, 5056] + - [0, 6434.54] + - - [448, 448, 1, 1792, 448, 448, 448, 448] + - [2, 3245.53] + - - [3584, 3584, 1, 256, 3584, 3584, 3584, 3584] + - [3, 6168.02] + - - [704, 5888, 1, 3328, 704, 704, 704, 5888] + - [0, 6330.85] + - - [128, 4, 1, 3328, 128, 128, 128, 4] + - [6, 12.8938] + - - [5888, 128, 1, 3328, 5888, 5888, 5888, 128] + - [3, 5598.49] + - - [2944, 5888, 1, 256, 2944, 2944, 2944, 5888] + - [3, 6158.06] + - - [2368, 4, 1, 768, 2368, 2368, 2368, 4] + - [6, 173.504] + - - [5888, 1856, 1, 1792, 5888, 5888, 5888, 1856] + - [3, 6334.4] + - - [2368, 2368, 1, 768, 2368, 2368, 2368, 2368] + - [3, 6103.54] + - - [1856, 2368, 1, 768, 1856, 1856, 1856, 2368] + - [3, 6146.8] +- null diff --git a/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Ailk_Bljk_SB.yaml b/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Ailk_Bljk_SB.yaml new file mode 100644 index 0000000000..224b150bb8 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Ailk_Bljk_SB.yaml @@ -0,0 +1,3246 @@ +- {MinimumRequiredVersion: 4.9.4} +- navi10 +- gfx1010 +- [NAVI] +- AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseInitialStrides: false +- - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 2 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: false + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 128 + LSCB: 8 + LSPA: 8 + LSPB: 64 + LVCA: 32 + LVCB: 4 + LVPA: 2 + LVPB: 32 + LdcEqualsLdd: true + LdsNumElements: 3584 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 512 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 3072 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 8 + MacroTile0: 128 + MacroTile1: 64 + MacroTileA: 128 + MacroTileB: 64 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 32 + NumGlobalWriteVectorsPerThread: 8 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 0 + SolutionNameMin: Cijk_Ailk_Bljk_SB_MT128x64x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: true + ThreadTile: [8, 4] + ThreadTile0: 8 + ThreadTile1: 4 + ThreadTileA: 8 + ThreadTileB: 4 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 3 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 16 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: false + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 64 + LSCB: 16 + LSPA: 16 + LSPB: 64 + LVCA: 16 + LVCB: 4 + LVPA: 4 + LVPB: 16 + LdcEqualsLdd: true + LdsNumElements: 16384 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 1024 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 3072 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 4 + MacroTile0: 64 + MacroTile1: 64 + MacroTileA: 64 + MacroTileB: 64 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 16 + NumGlobalWriteVectorsPerThread: 4 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 1 + SolutionNameMin: Cijk_Ailk_Bljk_SB_MT64x64x16_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: true + ThreadTile: [8, 8] + ThreadTile0: 8 + ThreadTile1: 8 + ThreadTileA: 8 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 2 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: false + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 128 + LSCB: 8 + LSPA: 8 + LSPB: 128 + LVCA: 32 + LVCB: 2 + LVPA: 2 + LVPB: 32 + LdcEqualsLdd: true + LdsNumElements: 4096 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 1024 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 3072 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 8 + MacroTile0: 128 + MacroTile1: 128 + MacroTileA: 128 + MacroTileB: 128 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 64 + NumGlobalWriteVectorsPerThread: 16 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 2 + SolutionNameMin: Cijk_Ailk_Bljk_SB_MT128x128x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: true + ThreadTile: [8, 8] + ThreadTile0: 8 + ThreadTile1: 8 + ThreadTileA: 8 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 3 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 16 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 2 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: false + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 64 + LSCB: 16 + LSPA: 16 + LSPB: 32 + LVCA: 16 + LVCB: 8 + LVPA: 4 + LVPB: 16 + LdcEqualsLdd: true + LdsNumElements: 8192 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 512 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 3072 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 4 + MacroTile0: 64 + MacroTile1: 32 + MacroTileA: 64 + MacroTileB: 32 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 8 + NumGlobalWriteVectorsPerThread: 2 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 3 + SolutionNameMin: Cijk_Ailk_Bljk_SB_MT64x32x16_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: true + ThreadTile: [8, 4] + ThreadTile0: 8 + ThreadTile1: 4 + ThreadTileA: 8 + ThreadTileB: 4 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 2 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 1 + GlobalLoadVectorWidthB: 1 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 32 + LSCB: 8 + LSPA: 8 + LSPB: 32 + LVCA: 32 + LVCB: 8 + LVPA: 8 + LVPB: 32 + LdcEqualsLdd: true + LdsNumElements: 4096 + LdsNumElementsAlignedA: 256 + LdsNumElementsAlignedB: 256 + LdsOffsetA: 0 + LdsOffsetA_Blk: 512 + LdsOffsetB: 256 + LdsOffsetB_Blk: 768 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 2 + MacroTile0: 32 + MacroTile1: 32 + MacroTileA: 32 + MacroTileB: 32 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 4 + NumGlobalWriteVectorsPerThread: 1 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 4 + SolutionNameMin: Cijk_Ailk_Bljk_SB_MT32x32x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: true + ThreadTile: [4, 4] + ThreadTile0: 4 + ThreadTile1: 4 + ThreadTileA: 4 + ThreadTileB: 4 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 3 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 16 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: false + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 64 + LSCB: 16 + LSPA: 16 + LSPB: 64 + LVCA: 16 + LVCB: 4 + LVPA: 4 + LVPB: 16 + LdcEqualsLdd: true + LdsNumElements: 7168 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 2048 + LdsOffsetA: 0 + LdsOffsetA_Blk: 4096 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 5120 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 16 + MacroTile0: 64 + MacroTile1: 128 + MacroTileA: 64 + MacroTileB: 128 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 32 + NumGlobalWriteVectorsPerThread: 8 + NumLoadsA: 1 + NumLoadsB: 2 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 2 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 5 + SolutionNameMin: Cijk_Ailk_Bljk_SB_MT64x128x16_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: true + ThreadTile: [4, 8] + ThreadTile0: 4 + ThreadTile1: 8 + ThreadTileA: 4 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 2 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 1 + GlobalLoadVectorWidthB: 1 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 2 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 1 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 8 + LSCB: 8 + LSPA: 8 + LSPB: 8 + LVCA: 8 + LVCB: 8 + LVPA: 8 + LVPB: 8 + LdcEqualsLdd: true + LdsNumElements: 409 + LdsNumElementsAlignedA: 64 + LdsNumElementsAlignedB: 64 + LdsOffsetA: 0 + LdsOffsetA_Blk: 128 + LdsOffsetB: 64 + LdsOffsetB_Blk: 192 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 2 + MacroTile0: 8 + MacroTile1: 8 + MacroTileA: 8 + MacroTileB: 8 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 1 + NumGlobalWriteVectorsPerThread: 1 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 64 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 6 + SolutionNameMin: Cijk_Ailk_Bljk_SB_MT8x8x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 4 + SubGroup1: 4 + SubGroupA: 4 + SubGroupB: 4 + SuppressNoLoadLoop: true + ThreadTile: [2, 2] + ThreadTile0: 2 + ThreadTile1: 2 + ThreadTileA: 2 + ThreadTileB: 2 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 2 + WorkGroup: [4, 4, 4] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 3 +- [2, 3, 0, 1] +- - - [4, 2368, 1, 1792, 4, 4, 4, 1792] + - [6, 176.125] + - - [2368, 4288, 1, 768, 2368, 2368, 2368, 768] + - [0, 6394.94] + - - [5888, 1856, 1, 768, 5888, 5888, 5888, 768] + - [0, 6534.38] + - - [3584, 2944, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 6390.83] + - - [5056, 64, 1, 1792, 5056, 5056, 5056, 1792] + - [3, 3437.82] + - - [1024, 64, 1, 3328, 1024, 1024, 1024, 3328] + - [3, 1282.37] + - - [1408, 704, 1, 1792, 1408, 1408, 1408, 1792] + - [0, 5417.8] + - - [704, 1408, 1, 256, 704, 704, 704, 256] + - [5, 4705.49] + - - [4288, 704, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 6126.98] + - - [2368, 5888, 1, 256, 2368, 2368, 2368, 256] + - [0, 6121.05] + - - [128, 128, 1, 3328, 128, 128, 128, 3328] + - [3, 404.884] + - - [6784, 64, 1, 256, 6784, 6784, 6784, 256] + - [0, 3062.17] + - - [256, 1024, 1, 256, 256, 256, 256, 256] + - [3, 3123.96] + - - [1408, 2944, 1, 768, 1408, 1408, 1408, 768] + - [0, 6311.6] + - - [1856, 5056, 1, 256, 1856, 1856, 1856, 256] + - [0, 5998.88] + - - [4, 128, 1, 256, 4, 4, 4, 256] + - [6, 10.2874] + - - [6784, 4, 1, 256, 6784, 6784, 6784, 256] + - [6, 301.544] + - - [1024, 2368, 1, 3328, 1024, 1024, 1024, 3328] + - [0, 5867.38] + - - [448, 704, 1, 1792, 448, 448, 448, 1792] + - [3, 3514.4] + - - [1024, 4, 1, 1792, 1024, 1024, 1024, 1792] + - [6, 82.7801] + - - [1024, 128, 1, 768, 1024, 1024, 1024, 768] + - [1, 2034.32] + - - [5056, 2368, 1, 768, 5056, 5056, 5056, 768] + - [0, 6306.7] + - - [5888, 1024, 1, 256, 5888, 5888, 5888, 256] + - [0, 5887.74] + - - [128, 2368, 1, 1792, 128, 128, 128, 1792] + - [3, 3326.9] + - - [256, 1408, 1, 1792, 256, 256, 256, 1792] + - [3, 3992.54] + - - [4, 4288, 1, 256, 4, 4, 4, 256] + - [6, 169.36] + - - [2944, 704, 1, 768, 2944, 2944, 2944, 768] + - [0, 5998.84] + - - [5056, 704, 1, 768, 5056, 5056, 5056, 768] + - [0, 6129.05] + - - [6784, 1856, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6458.97] + - - [256, 6784, 1, 3328, 256, 256, 256, 3328] + - [0, 5190.9] + - - [4288, 5056, 1, 256, 4288, 4288, 4288, 256] + - [0, 6330.21] + - - [2944, 128, 1, 256, 2944, 2944, 2944, 256] + - [3, 3071.43] + - - [1856, 64, 1, 768, 1856, 1856, 1856, 768] + - [3, 2463.04] + - - [3584, 4288, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 6580.47] + - - [6784, 1408, 1, 256, 6784, 6784, 6784, 256] + - [0, 6032.59] + - - [1024, 6784, 1, 1792, 1024, 1024, 1024, 1792] + - [0, 6236.18] + - - [4, 448, 1, 768, 4, 4, 4, 768] + - [6, 41.8442] + - - [128, 3584, 1, 256, 128, 128, 128, 256] + - [5, 3151.92] + - - [3584, 2368, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 6357.15] + - - [3584, 704, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 6134.99] + - - [4, 256, 1, 1792, 4, 4, 4, 1792] + - [6, 24.3339] + - - [448, 3584, 1, 1792, 448, 448, 448, 1792] + - [3, 4720.75] + - - [4, 6784, 1, 1792, 4, 4, 4, 1792] + - [6, 165.772] + - - [2368, 64, 1, 768, 2368, 2368, 2368, 768] + - [3, 2439.83] + - - [448, 6784, 1, 1792, 448, 448, 448, 1792] + - [5, 5791.04] + - - [1024, 1856, 1, 768, 1024, 1024, 1024, 768] + - [0, 5391.05] + - - [5888, 1408, 1, 256, 5888, 5888, 5888, 256] + - [0, 6178.67] + - - [1408, 6784, 1, 3328, 1408, 1408, 1408, 3328] + - [0, 6433.31] + - - [2368, 2368, 1, 768, 2368, 2368, 2368, 768] + - [0, 6259.27] + - - [2944, 256, 1, 256, 2944, 2944, 2944, 256] + - [3, 3668.44] + - - [1408, 448, 1, 3328, 1408, 1408, 1408, 3328] + - [3, 4614.73] + - - [448, 128, 1, 1792, 448, 448, 448, 1792] + - [3, 1388.62] + - - [3584, 4, 1, 256, 3584, 3584, 3584, 256] + - [6, 192.778] + - - [6784, 1856, 1, 256, 6784, 6784, 6784, 256] + - [0, 6177.4] + - - [1856, 3584, 1, 3328, 1856, 1856, 1856, 3328] + - [0, 6039.17] + - - [1408, 1408, 1, 768, 1408, 1408, 1408, 768] + - [0, 5730.02] + - - [1024, 448, 1, 3328, 1024, 1024, 1024, 3328] + - [5, 4189.07] + - - [6784, 704, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6002.21] + - - [5056, 3584, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 6666.37] + - - [2368, 256, 1, 3328, 2368, 2368, 2368, 3328] + - [3, 4372.28] + - - [4, 2944, 1, 3328, 4, 4, 4, 3328] + - [6, 200.022] + - - [4288, 2944, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 6477.85] + - - [4288, 256, 1, 3328, 4288, 4288, 4288, 3328] + - [3, 4355.11] + - - [1856, 1856, 1, 1792, 1856, 1856, 1856, 1792] + - [0, 6051.75] + - - [1024, 5888, 1, 256, 1024, 1024, 1024, 256] + - [0, 5767.15] + - - [4288, 3584, 1, 768, 4288, 4288, 4288, 768] + - [0, 6495.96] + - - [128, 3584, 1, 768, 128, 128, 128, 768] + - [5, 3924.28] + - - [2368, 704, 1, 3328, 2368, 2368, 2368, 3328] + - [0, 4980.78] + - - [448, 5888, 1, 3328, 448, 448, 448, 3328] + - [0, 5369.92] + - - [128, 1024, 1, 256, 128, 128, 128, 256] + - [3, 2167.32] + - - [256, 704, 1, 768, 256, 256, 256, 768] + - [3, 2965.89] + - - [128, 256, 1, 3328, 128, 128, 128, 3328] + - [3, 816.504] + - - [2944, 64, 1, 768, 2944, 2944, 2944, 768] + - [3, 3052.27] + - - [1408, 128, 1, 1792, 1408, 1408, 1408, 1792] + - [3, 3103.09] + - - [448, 4, 1, 768, 448, 448, 448, 768] + - [6, 37.916] + - - [4288, 2368, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 6489.58] + - - [64, 1408, 1, 256, 64, 64, 64, 256] + - [3, 1359.9] + - - [4, 1024, 1, 3328, 4, 4, 4, 3328] + - [6, 83.7598] + - - [4288, 6784, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 6585.62] + - - [4, 4288, 1, 768, 4, 4, 4, 768] + - [6, 241.613] + - - [4, 128, 1, 768, 4, 4, 4, 768] + - [6, 13.172] + - - [128, 5888, 1, 1792, 128, 128, 128, 1792] + - [3, 4538.74] + - - [704, 4288, 1, 3328, 704, 704, 704, 3328] + - [5, 5789.41] + - - [3584, 1024, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 6489.87] + - - [3584, 4, 1, 768, 3584, 3584, 3584, 768] + - [6, 250.641] + - - [6784, 2944, 1, 256, 6784, 6784, 6784, 256] + - [0, 6255.92] + - - [448, 128, 1, 768, 448, 448, 448, 768] + - [3, 1385.83] + - - [4, 4, 1, 256, 4, 4, 4, 256] + - [6, 0.239371] + - - [128, 128, 1, 1792, 128, 128, 128, 1792] + - [3, 450.91] + - - [5888, 3584, 1, 768, 5888, 5888, 5888, 768] + - [0, 6548.7] + - - [4, 2368, 1, 768, 4, 4, 4, 768] + - [6, 166.728] + - - [3584, 5888, 1, 768, 3584, 3584, 3584, 768] + - [0, 6548.89] + - - [1856, 1408, 1, 768, 1856, 1856, 1856, 768] + - [5, 5804.07] + - - [1408, 1856, 1, 3328, 1408, 1408, 1408, 3328] + - [0, 6364.08] + - - [4288, 64, 1, 768, 4288, 4288, 4288, 768] + - [3, 2853.79] + - - [3584, 5888, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 6606.61] + - - [1408, 6784, 1, 1792, 1408, 1408, 1408, 1792] + - [0, 6407.81] + - - [448, 4, 1, 3328, 448, 448, 448, 3328] + - [6, 39.3685] + - - [256, 448, 1, 3328, 256, 256, 256, 3328] + - [3, 2812.53] + - - [6784, 256, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 5200.59] + - - [1024, 704, 1, 256, 1024, 1024, 1024, 256] + - [0, 3426.88] + - - [64, 6784, 1, 3328, 64, 64, 64, 3328] + - [5, 3843.57] + - - [2944, 3584, 1, 768, 2944, 2944, 2944, 768] + - [0, 6338.67] + - - [64, 4, 1, 1792, 64, 64, 64, 1792] + - [6, 5.85348] + - - [256, 3584, 1, 3328, 256, 256, 256, 3328] + - [0, 5075.78] + - - [448, 6784, 1, 768, 448, 448, 448, 768] + - [5, 5664.18] + - - [6784, 704, 1, 768, 6784, 6784, 6784, 768] + - [0, 5875.12] + - - [256, 1408, 1, 3328, 256, 256, 256, 3328] + - [3, 4108.0] + - - [448, 64, 1, 1792, 448, 448, 448, 1792] + - [3, 707.049] + - - [704, 2944, 1, 3328, 704, 704, 704, 3328] + - [5, 5865.45] + - - [1024, 64, 1, 768, 1024, 1024, 1024, 768] + - [3, 1202.58] + - - [6784, 6784, 1, 768, 6784, 6784, 6784, 768] + - [0, 6672.01] + - - [6784, 3584, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 6642.69] + - - [3584, 1856, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 6436.47] + - - [2368, 4, 1, 3328, 2368, 2368, 2368, 3328] + - [6, 188.964] + - - [704, 5056, 1, 256, 704, 704, 704, 256] + - [5, 5247.86] + - - [2368, 4, 1, 1792, 2368, 2368, 2368, 1792] + - [6, 183.607] + - - [1856, 5056, 1, 768, 1856, 1856, 1856, 768] + - [0, 6224.56] + - - [1856, 448, 1, 1792, 1856, 1856, 1856, 1792] + - [0, 4538.53] + - - [128, 128, 1, 768, 128, 128, 128, 768] + - [3, 411.294] + - - [2944, 5056, 1, 768, 2944, 2944, 2944, 768] + - [0, 6518.62] + - - [1024, 3584, 1, 256, 1024, 1024, 1024, 256] + - [0, 5705.44] + - - [5056, 5056, 1, 768, 5056, 5056, 5056, 768] + - [0, 6524.0] + - - [1408, 256, 1, 256, 1408, 1408, 1408, 256] + - [3, 3121.98] + - - [2368, 128, 1, 1792, 2368, 2368, 2368, 1792] + - [3, 3329.92] + - - [1856, 4288, 1, 256, 1856, 1856, 1856, 256] + - [0, 5984.31] + - - [5056, 448, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 5502.19] + - - [128, 3584, 1, 3328, 128, 128, 128, 3328] + - [3, 4242.53] + - - [128, 4288, 1, 3328, 128, 128, 128, 3328] + - [3, 3995.89] + - - [2368, 448, 1, 1792, 2368, 2368, 2368, 1792] + - [3, 4401.25] + - - [128, 256, 1, 256, 128, 128, 128, 256] + - [3, 605.588] + - - [5888, 64, 1, 768, 5888, 5888, 5888, 768] + - [3, 3756.63] + - - [5056, 4288, 1, 256, 5056, 5056, 5056, 256] + - [0, 6329.68] + - - [256, 128, 1, 1792, 256, 256, 256, 1792] + - [3, 837.951] + - - [1856, 4288, 1, 768, 1856, 1856, 1856, 768] + - [0, 6219.31] + - - [1408, 2944, 1, 256, 1408, 1408, 1408, 256] + - [0, 5956.43] + - - [4, 2368, 1, 256, 4, 4, 4, 256] + - [6, 124.94] + - - [448, 64, 1, 768, 448, 448, 448, 768] + - [3, 716.299] + - - [4288, 2368, 1, 256, 4288, 4288, 4288, 256] + - [0, 6156.34] + - - [3584, 1024, 1, 768, 3584, 3584, 3584, 768] + - [0, 6292.32] + - - [4288, 1024, 1, 768, 4288, 4288, 4288, 768] + - [0, 5986.8] + - - [448, 1856, 1, 768, 448, 448, 448, 768] + - [0, 4278.22] + - - [1856, 2944, 1, 768, 1856, 1856, 1856, 768] + - [0, 6101.22] + - - [1856, 704, 1, 768, 1856, 1856, 1856, 768] + - [0, 4886.97] + - - [3584, 128, 1, 1792, 3584, 3584, 3584, 1792] + - [5, 4159.59] + - - [5056, 5888, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 6617.55] + - - [256, 2944, 1, 768, 256, 256, 256, 768] + - [3, 4333.77] + - - [1024, 6784, 1, 3328, 1024, 1024, 1024, 3328] + - [0, 6282.16] + - - [128, 128, 1, 256, 128, 128, 128, 256] + - [3, 307.726] + - - [4, 5888, 1, 3328, 4, 4, 4, 3328] + - [6, 170.142] + - - [1408, 5056, 1, 768, 1408, 1408, 1408, 768] + - [0, 6346.18] + - - [256, 1024, 1, 3328, 256, 256, 256, 3328] + - [3, 4737.44] + - - [2368, 3584, 1, 1792, 2368, 2368, 2368, 1792] + - [0, 6341.41] + - - [704, 64, 1, 3328, 704, 704, 704, 3328] + - [3, 1123.94] + - - [3584, 5888, 1, 256, 3584, 3584, 3584, 256] + - [0, 6343.81] + - - [3584, 64, 1, 3328, 3584, 3584, 3584, 3328] + - [3, 4044.39] + - - [5056, 5056, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 6582.77] + - - [1408, 2368, 1, 768, 1408, 1408, 1408, 768] + - [0, 5757.4] + - - [448, 5888, 1, 1792, 448, 448, 448, 1792] + - [0, 5308.85] + - - [2368, 256, 1, 1792, 2368, 2368, 2368, 1792] + - [3, 4289.26] + - - [1856, 2368, 1, 256, 1856, 1856, 1856, 256] + - [0, 5682.28] + - - [3584, 2368, 1, 768, 3584, 3584, 3584, 768] + - [0, 6232.45] + - - [4288, 448, 1, 256, 4288, 4288, 4288, 256] + - [0, 5075.6] + - - [256, 64, 1, 3328, 256, 256, 256, 3328] + - [3, 407.575] + - - [5056, 4, 1, 768, 5056, 5056, 5056, 768] + - [6, 323.574] + - - [1408, 5056, 1, 256, 1408, 1408, 1408, 256] + - [0, 6101.7] + - - [64, 1408, 1, 3328, 64, 64, 64, 3328] + - [3, 2142.03] + - - [5888, 128, 1, 3328, 5888, 5888, 5888, 3328] + - [3, 4530.32] + - - [1024, 1856, 1, 3328, 1024, 1024, 1024, 3328] + - [0, 5656.89] + - - [704, 1856, 1, 256, 704, 704, 704, 256] + - [0, 4341.87] + - - [2368, 3584, 1, 3328, 2368, 2368, 2368, 3328] + - [0, 6373.3] + - - [1408, 704, 1, 256, 1408, 1408, 1408, 256] + - [0, 4711.96] + - - [2944, 256, 1, 3328, 2944, 2944, 2944, 3328] + - [3, 4690.35] + - - [1856, 448, 1, 768, 1856, 1856, 1856, 768] + - [0, 4358.26] + - - [5056, 2368, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 6397.05] + - - [4, 64, 1, 3328, 4, 4, 4, 3328] + - [6, 6.20683] + - - [448, 1856, 1, 256, 448, 448, 448, 256] + - [5, 3771.05] + - - [1408, 2368, 1, 256, 1408, 1408, 1408, 256] + - [0, 5466.07] + - - [2944, 128, 1, 3328, 2944, 2944, 2944, 3328] + - [3, 4192.62] + - - [3584, 2944, 1, 768, 3584, 3584, 3584, 768] + - [0, 6316.71] + - - [4288, 1024, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 6077.81] + - - [1024, 704, 1, 1792, 1024, 1024, 1024, 1792] + - [3, 4025.72] + - - [4, 5888, 1, 1792, 4, 4, 4, 1792] + - [6, 168.966] + - - [3584, 128, 1, 3328, 3584, 3584, 3584, 3328] + - [5, 4232.95] + - - [448, 2944, 1, 1792, 448, 448, 448, 1792] + - [0, 5016.7] + - - [1408, 3584, 1, 1792, 1408, 1408, 1408, 1792] + - [0, 6314.38] + - - [5056, 5056, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 6604.03] + - - [2944, 2944, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 6470.48] + - - [128, 704, 1, 3328, 128, 128, 128, 3328] + - [3, 2169.58] + - - [1856, 128, 1, 3328, 1856, 1856, 1856, 3328] + - [3, 4277.82] + - - [5056, 704, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 6295.55] + - - [3584, 3584, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 6547.52] + - - [5056, 1024, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 6461.01] + - - [256, 448, 1, 768, 256, 256, 256, 768] + - [3, 2551.61] + - - [2944, 4288, 1, 768, 2944, 2944, 2944, 768] + - [0, 6404.1] + - - [704, 1024, 1, 3328, 704, 704, 704, 3328] + - [3, 4550.08] + - - [4, 256, 1, 3328, 4, 4, 4, 3328] + - [6, 24.9621] + - - [2368, 704, 1, 1792, 2368, 2368, 2368, 1792] + - [0, 4931.36] + - - [4288, 704, 1, 256, 4288, 4288, 4288, 256] + - [0, 5561.04] + - - [4288, 64, 1, 1792, 4288, 4288, 4288, 1792] + - [3, 2963.78] + - - [448, 3584, 1, 3328, 448, 448, 448, 3328] + - [3, 4797.85] + - - [4, 2944, 1, 1792, 4, 4, 4, 1792] + - [6, 198.382] + - - [4, 1024, 1, 256, 4, 4, 4, 256] + - [6, 62.3596] + - - [704, 128, 1, 256, 704, 704, 704, 256] + - [3, 1384.09] + - - [704, 448, 1, 768, 704, 704, 704, 768] + - [3, 3326.44] + - - [4, 5888, 1, 256, 4, 4, 4, 256] + - [6, 189.729] + - - [2368, 1856, 1, 1792, 2368, 2368, 2368, 1792] + - [0, 6071.67] + - - [704, 2944, 1, 256, 704, 704, 704, 256] + - [5, 5251.18] + - - [2368, 5056, 1, 3328, 2368, 2368, 2368, 3328] + - [0, 6406.12] + - - [2944, 6784, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 6607.85] + - - [448, 1856, 1, 1792, 448, 448, 448, 1792] + - [0, 4488.91] + - - [1024, 1024, 1, 768, 1024, 1024, 1024, 768] + - [5, 5344.8] + - - [128, 1856, 1, 256, 128, 128, 128, 256] + - [3, 2831.09] + - - [1024, 1856, 1, 1792, 1024, 1024, 1024, 1792] + - [0, 5578.34] + - - [448, 2944, 1, 768, 448, 448, 448, 768] + - [0, 4808.04] + - - [448, 2944, 1, 256, 448, 448, 448, 256] + - [5, 4340.98] + - - [2944, 128, 1, 768, 2944, 2944, 2944, 768] + - [3, 3830.26] + - - [4288, 5888, 1, 768, 4288, 4288, 4288, 768] + - [0, 6573.94] + - - [5888, 6784, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 6688.28] + - - [5888, 1024, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 6339.27] + - - [256, 2368, 1, 768, 256, 256, 256, 768] + - [3, 4094.45] + - - [128, 704, 1, 1792, 128, 128, 128, 1792] + - [3, 2076.12] + - - [704, 64, 1, 256, 704, 704, 704, 256] + - [3, 819.52] + - - [64, 5888, 1, 768, 64, 64, 64, 768] + - [3, 3148.17] + - - [1408, 6784, 1, 256, 1408, 1408, 1408, 256] + - [0, 6134.61] + - - [256, 448, 1, 256, 256, 256, 256, 256] + - [3, 1826.45] + - - [4, 256, 1, 768, 4, 4, 4, 768] + - [6, 27.0145] + - - [448, 5888, 1, 768, 448, 448, 448, 768] + - [0, 5110.86] + - - [1856, 6784, 1, 256, 1856, 1856, 1856, 256] + - [0, 6127.82] + - - [64, 5056, 1, 3328, 64, 64, 64, 3328] + - [3, 3453.93] + - - [1024, 5056, 1, 3328, 1024, 1024, 1024, 3328] + - [0, 6479.32] + - - [2368, 1408, 1, 768, 2368, 2368, 2368, 768] + - [0, 5759.31] + - - [2368, 6784, 1, 768, 2368, 2368, 2368, 768] + - [0, 6388.16] + - - [2944, 256, 1, 768, 2944, 2944, 2944, 768] + - [3, 4362.79] + - - [1024, 4288, 1, 256, 1024, 1024, 1024, 256] + - [0, 5548.28] + - - [4288, 2944, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 6457.5] + - - [2368, 5056, 1, 1792, 2368, 2368, 2368, 1792] + - [0, 6379.16] + - - [1856, 256, 1, 3328, 1856, 1856, 1856, 3328] + - [3, 4408.57] + - - [5888, 448, 1, 768, 5888, 5888, 5888, 768] + - [0, 5303.12] + - - [4288, 448, 1, 768, 4288, 4288, 4288, 768] + - [0, 5560.19] + - - [704, 5056, 1, 1792, 704, 704, 704, 1792] + - [5, 5824.35] + - - [1856, 1024, 1, 1792, 1856, 1856, 1856, 1792] + - [0, 5630.25] + - - [448, 4288, 1, 768, 448, 448, 448, 768] + - [5, 5198.8] + - - [5888, 3584, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 6630.14] + - - [448, 2944, 1, 3328, 448, 448, 448, 3328] + - [0, 5091.9] + - - [6784, 128, 1, 256, 6784, 6784, 6784, 256] + - [0, 4110.88] + - - [2944, 128, 1, 1792, 2944, 2944, 2944, 1792] + - [3, 4097.11] + - - [2368, 1024, 1, 1792, 2368, 2368, 2368, 1792] + - [0, 5851.36] + - - [3584, 1408, 1, 256, 3584, 3584, 3584, 256] + - [0, 5795.76] + - - [2368, 4288, 1, 3328, 2368, 2368, 2368, 3328] + - [0, 6499.2] + - - [1408, 704, 1, 768, 1408, 1408, 1408, 768] + - [0, 5208.73] + - - [1408, 3584, 1, 768, 1408, 1408, 1408, 768] + - [0, 6210.62] + - - [1024, 5888, 1, 3328, 1024, 1024, 1024, 3328] + - [0, 6329.2] + - - [3584, 64, 1, 768, 3584, 3584, 3584, 768] + - [3, 3636.23] + - - [4288, 1024, 1, 256, 4288, 4288, 4288, 256] + - [0, 5668.49] + - - [5056, 2944, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 6592.48] + - - [704, 6784, 1, 1792, 704, 704, 704, 1792] + - [0, 5932.84] + - - [5888, 2944, 1, 256, 5888, 5888, 5888, 256] + - [0, 6280.06] + - - [4288, 4288, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 6584.66] + - - [64, 4288, 1, 768, 64, 64, 64, 768] + - [3, 2489.27] + - - [5888, 704, 1, 256, 5888, 5888, 5888, 256] + - [0, 5919.52] + - - [4288, 448, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 5695.11] + - - [1408, 128, 1, 256, 1408, 1408, 1408, 256] + - [4, 2155.15] + - - [1024, 5056, 1, 768, 1024, 1024, 1024, 768] + - [0, 6237.9] + - - [2368, 5888, 1, 3328, 2368, 2368, 2368, 3328] + - [2, 6251.78] + - - [2944, 1024, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 6180.46] + - - [704, 448, 1, 1792, 704, 704, 704, 1792] + - [3, 3496.67] + - - [256, 4, 1, 1792, 256, 256, 256, 1792] + - [6, 28.7378] + - - [1024, 1408, 1, 1792, 1024, 1024, 1024, 1792] + - [0, 5542.48] + - - [6784, 2368, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 6624.87] + - - [3584, 4288, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 6547.21] + - - [6784, 448, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 6176.33] + - - [2368, 256, 1, 768, 2368, 2368, 2368, 768] + - [3, 4031.43] + - - [6784, 448, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6224.03] + - - [1024, 4, 1, 768, 1024, 1024, 1024, 768] + - [6, 77.3514] + - - [64, 3584, 1, 1792, 64, 64, 64, 1792] + - [3, 3679.64] + - - [64, 256, 1, 3328, 64, 64, 64, 3328] + - [3, 407.124] + - - [6784, 4288, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 6678.37] + - - [6784, 256, 1, 256, 6784, 6784, 6784, 256] + - [0, 4688.97] + - - [1408, 5056, 1, 3328, 1408, 1408, 1408, 3328] + - [0, 6463.1] + - - [5888, 2368, 1, 256, 5888, 5888, 5888, 256] + - [0, 6250.47] + - - [6784, 4, 1, 768, 6784, 6784, 6784, 768] + - [6, 396.81] + - - [64, 128, 1, 1792, 64, 64, 64, 1792] + - [3, 223.674] + - - [448, 704, 1, 3328, 448, 448, 448, 3328] + - [3, 3601.81] + - - [5056, 1024, 1, 256, 5056, 5056, 5056, 256] + - [0, 5938.4] + - - [4, 5056, 1, 768, 4, 4, 4, 768] + - [6, 262.754] + - - [704, 6784, 1, 768, 704, 704, 704, 768] + - [0, 5798.75] + - - [2368, 704, 1, 256, 2368, 2368, 2368, 256] + - [0, 4407.99] + - - [3584, 64, 1, 256, 3584, 3584, 3584, 256] + - [3, 2724.02] + - - [1856, 4, 1, 768, 1856, 1856, 1856, 768] + - [6, 136.23] + - - [1408, 4288, 1, 1792, 1408, 1408, 1408, 1792] + - [0, 6339.2] + - - [1408, 5888, 1, 1792, 1408, 1408, 1408, 1792] + - [0, 6567.55] + - - [256, 64, 1, 256, 256, 256, 256, 256] + - [3, 298.007] + - - [5056, 1856, 1, 768, 5056, 5056, 5056, 768] + - [0, 6214.22] + - - [1856, 5888, 1, 768, 1856, 1856, 1856, 768] + - [0, 6281.51] + - - [2944, 5056, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 6602.31] + - - [704, 128, 1, 3328, 704, 704, 704, 3328] + - [3, 2205.03] + - - [1856, 704, 1, 256, 1856, 1856, 1856, 256] + - [0, 4414.04] + - - [5888, 704, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 6407.04] + - - [64, 1024, 1, 256, 64, 64, 64, 256] + - [3, 989.019] + - - [256, 4288, 1, 1792, 256, 256, 256, 1792] + - [3, 4608.73] + - - [4, 3584, 1, 3328, 4, 4, 4, 3328] + - [6, 178.265] + - - [1024, 704, 1, 768, 1024, 1024, 1024, 768] + - [0, 3821.08] + - - [6784, 6784, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6759.07] + - - [3584, 256, 1, 256, 3584, 3584, 3584, 256] + - [0, 4367.48] + - - [448, 1856, 1, 3328, 448, 448, 448, 3328] + - [0, 4566.2] + - - [256, 1856, 1, 256, 256, 256, 256, 256] + - [3, 3297.23] + - - [128, 5056, 1, 3328, 128, 128, 128, 3328] + - [3, 4659.48] + - - [5056, 6784, 1, 768, 5056, 5056, 5056, 768] + - [0, 6552.39] + - - [1856, 1856, 1, 256, 1856, 1856, 1856, 256] + - [0, 5486.03] + - - [4, 5056, 1, 1792, 4, 4, 4, 1792] + - [6, 166.807] + - - [64, 128, 1, 768, 64, 64, 64, 768] + - [3, 203.673] + - - [3584, 6784, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 6636.02] + - - [4, 448, 1, 1792, 4, 4, 4, 1792] + - [6, 44.5082] + - - [64, 64, 1, 1792, 64, 64, 64, 1792] + - [3, 114.025] + - - [2944, 1024, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 6139.28] + - - [2944, 2944, 1, 768, 2944, 2944, 2944, 768] + - [0, 6390.6] + - - [4288, 5056, 1, 768, 4288, 4288, 4288, 768] + - [0, 6570.01] + - - [4, 1024, 1, 1792, 4, 4, 4, 1792] + - [6, 83.6886] + - - [2368, 1024, 1, 3328, 2368, 2368, 2368, 3328] + - [0, 5886.41] + - - [256, 5888, 1, 256, 256, 256, 256, 256] + - [0, 4841.24] + - - [6784, 256, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 5161.32] + - - [1408, 128, 1, 3328, 1408, 1408, 1408, 3328] + - [3, 3233.58] + - - [704, 4, 1, 3328, 704, 704, 704, 3328] + - [6, 58.8437] + - - [3584, 448, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 4817.08] + - - [4, 1856, 1, 3328, 4, 4, 4, 3328] + - [6, 145.776] + - - [2368, 2944, 1, 768, 2368, 2368, 2368, 768] + - [0, 6219.01] + - - [704, 448, 1, 3328, 704, 704, 704, 3328] + - [3, 3584.5] + - - [1856, 3584, 1, 256, 1856, 1856, 1856, 256] + - [0, 5728.46] + - - [2368, 2368, 1, 256, 2368, 2368, 2368, 256] + - [0, 5957.18] + - - [128, 704, 1, 768, 128, 128, 128, 768] + - [3, 2009.15] + - - [1856, 1408, 1, 3328, 1856, 1856, 1856, 3328] + - [5, 5976.0] + - - [1856, 6784, 1, 768, 1856, 1856, 1856, 768] + - [0, 6353.28] + - - [2368, 128, 1, 256, 2368, 2368, 2368, 256] + - [3, 2580.04] + - - [5056, 6784, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 6607.1] + - - [6784, 1024, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6287.73] + - - [128, 1408, 1, 1792, 128, 128, 128, 1792] + - [3, 3127.58] + - - [64, 2944, 1, 3328, 64, 64, 64, 3328] + - [3, 3315.67] + - - [1408, 4, 1, 256, 1408, 1408, 1408, 256] + - [6, 86.8916] + - - [256, 1856, 1, 3328, 256, 256, 256, 3328] + - [3, 4437.9] + - - [1024, 2944, 1, 1792, 1024, 1024, 1024, 1792] + - [0, 6060.2] + - - [2368, 4, 1, 256, 2368, 2368, 2368, 256] + - [6, 133.611] + - - [128, 1024, 1, 768, 128, 128, 128, 768] + - [3, 2675.05] + - - [4288, 5888, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 6656.8] + - - [4288, 1408, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 6327.11] + - - [1856, 448, 1, 3328, 1856, 1856, 1856, 3328] + - [0, 4597.1] + - - [64, 448, 1, 3328, 64, 64, 64, 3328] + - [3, 714.441] + - - [5056, 3584, 1, 768, 5056, 5056, 5056, 768] + - [0, 6549.87] + - - [1856, 64, 1, 1792, 1856, 1856, 1856, 1792] + - [3, 2752.43] + - - [704, 128, 1, 768, 704, 704, 704, 768] + - [3, 2152.66] + - - [448, 5056, 1, 1792, 448, 448, 448, 1792] + - [0, 5399.1] + - - [2944, 1856, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 6232.39] + - - [5888, 1856, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 6609.47] + - - [3584, 64, 1, 1792, 3584, 3584, 3584, 1792] + - [3, 3943.77] + - - [6784, 1856, 1, 768, 6784, 6784, 6784, 768] + - [0, 6363.19] + - - [64, 64, 1, 768, 64, 64, 64, 768] + - [3, 93.7442] + - - [1024, 1024, 1, 256, 1024, 1024, 1024, 256] + - [0, 4724.6] + - - [64, 704, 1, 3328, 64, 64, 64, 3328] + - [3, 1111.6] + - - [5056, 5888, 1, 256, 5056, 5056, 5056, 256] + - [0, 6359.33] + - - [5888, 5056, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 6708.42] + - - [2368, 1856, 1, 256, 2368, 2368, 2368, 256] + - [0, 5673.78] + - - [6784, 128, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 4698.4] + - - [1024, 4288, 1, 768, 1024, 1024, 1024, 768] + - [0, 5867.96] + - - [128, 4, 1, 3328, 128, 128, 128, 3328] + - [6, 13.1218] + - - [6784, 5888, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 6680.42] + - - [2944, 2944, 1, 256, 2944, 2944, 2944, 256] + - [0, 6166.92] + - - [128, 1408, 1, 256, 128, 128, 128, 256] + - [4, 2193.05] + - - [5888, 256, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 5814.74] + - - [4288, 4288, 1, 768, 4288, 4288, 4288, 768] + - [0, 6501.72] + - - [256, 6784, 1, 256, 256, 256, 256, 256] + - [0, 4489.06] + - - [5888, 704, 1, 768, 5888, 5888, 5888, 768] + - [0, 6278.41] + - - [5888, 1408, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 6588.64] + - - [4288, 6784, 1, 256, 4288, 4288, 4288, 256] + - [0, 6338.68] + - - [1408, 3584, 1, 3328, 1408, 1408, 1408, 3328] + - [0, 6351.55] + - - [128, 1408, 1, 768, 128, 128, 128, 768] + - [3, 2874.63] + - - [256, 2368, 1, 1792, 256, 256, 256, 1792] + - [3, 4346.46] + - - [256, 2944, 1, 1792, 256, 256, 256, 1792] + - [3, 4664.14] + - - [3584, 256, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 5093.97] + - - [448, 4288, 1, 1792, 448, 448, 448, 1792] + - [5, 5357.63] + - - [704, 3584, 1, 3328, 704, 704, 704, 3328] + - [5, 5765.83] + - - [1856, 1408, 1, 256, 1856, 1856, 1856, 256] + - [5, 5470.11] + - - [448, 5056, 1, 768, 448, 448, 448, 768] + - [0, 5197.82] + - - [3584, 5056, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 6660.86] + - - [5888, 4288, 1, 768, 5888, 5888, 5888, 768] + - [0, 6549.92] + - - [256, 256, 1, 3328, 256, 256, 256, 3328] + - [3, 1624.0] + - - [6784, 64, 1, 768, 6784, 6784, 6784, 768] + - [0, 3683.62] + - - [1408, 1856, 1, 768, 1408, 1408, 1408, 768] + - [0, 6170.2] + - - [704, 1856, 1, 1792, 704, 704, 704, 1792] + - [0, 5026.07] + - - [1856, 704, 1, 3328, 1856, 1856, 1856, 3328] + - [0, 5085.53] + - - [2368, 2944, 1, 3328, 2368, 2368, 2368, 3328] + - [0, 6325.77] + - - [704, 1856, 1, 768, 704, 704, 704, 768] + - [0, 4883.46] + - - [64, 5056, 1, 1792, 64, 64, 64, 1792] + - [3, 3182.08] + - - [5056, 2944, 1, 768, 5056, 5056, 5056, 768] + - [0, 6501.32] + - - [2944, 64, 1, 1792, 2944, 2944, 2944, 1792] + - [3, 3314.92] + - - [5056, 128, 1, 768, 5056, 5056, 5056, 768] + - [3, 4098.75] + - - [704, 2368, 1, 768, 704, 704, 704, 768] + - [0, 4753.5] + - - [2368, 2368, 1, 1792, 2368, 2368, 2368, 1792] + - [0, 6384.77] + - - [4288, 4, 1, 3328, 4288, 4288, 4288, 3328] + - [6, 319.744] + - - [704, 448, 1, 256, 704, 704, 704, 256] + - [3, 2868.32] + - - [3584, 448, 1, 768, 3584, 3584, 3584, 768] + - [0, 4673.42] + - - [5056, 4, 1, 3328, 5056, 5056, 5056, 3328] + - [6, 362.714] + - - [704, 2944, 1, 768, 704, 704, 704, 768] + - [5, 5656.19] + - - [3584, 448, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 4779.42] + - - [3584, 6784, 1, 256, 3584, 3584, 3584, 256] + - [0, 6372.38] + - - [128, 448, 1, 3328, 128, 128, 128, 3328] + - [3, 1403.2] + - - [3584, 4, 1, 1792, 3584, 3584, 3584, 1792] + - [6, 272.219] + - - [5056, 2944, 1, 256, 5056, 5056, 5056, 256] + - [0, 6257.22] + - - [704, 256, 1, 3328, 704, 704, 704, 3328] + - [3, 3255.68] + - - [64, 6784, 1, 1792, 64, 64, 64, 1792] + - [5, 3609.38] + - - [1408, 64, 1, 3328, 1408, 1408, 1408, 3328] + - [3, 2221.98] + - - [704, 128, 1, 1792, 704, 704, 704, 1792] + - [3, 2141.38] + - - [5888, 256, 1, 256, 5888, 5888, 5888, 256] + - [0, 5137.27] + - - [448, 5056, 1, 256, 448, 448, 448, 256] + - [5, 4648.4] + - - [2368, 6784, 1, 3328, 2368, 2368, 2368, 3328] + - [0, 6471.48] + - - [4288, 256, 1, 256, 4288, 4288, 4288, 256] + - [0, 3744.94] + - - [6784, 5888, 1, 256, 6784, 6784, 6784, 256] + - [0, 6406.61] + - - [64, 1856, 1, 768, 64, 64, 64, 768] + - [3, 2267.98] + - - [6784, 64, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 3953.4] + - - [2944, 448, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 5101.73] + - - [1408, 4, 1, 1792, 1408, 1408, 1408, 1792] + - [6, 114.396] + - - [64, 704, 1, 256, 64, 64, 64, 256] + - [3, 745.016] + - - [256, 3584, 1, 256, 256, 256, 256, 256] + - [0, 4279.08] + - - [3584, 256, 1, 768, 3584, 3584, 3584, 768] + - [0, 4843.39] + - - [1024, 128, 1, 1792, 1024, 1024, 1024, 1792] + - [1, 2140.98] + - - [256, 1024, 1, 1792, 256, 256, 256, 1792] + - [3, 4523.25] + - - [64, 1024, 1, 3328, 64, 64, 64, 3328] + - [3, 1593.24] + - - [704, 2944, 1, 1792, 704, 704, 704, 1792] + - [5, 5807.08] + - - [6784, 704, 1, 256, 6784, 6784, 6784, 256] + - [0, 5585.49] + - - [6784, 2368, 1, 768, 6784, 6784, 6784, 768] + - [0, 6544.8] + - - [1408, 2944, 1, 1792, 1408, 1408, 1408, 1792] + - [0, 6420.58] + - - [5888, 6784, 1, 256, 5888, 5888, 5888, 256] + - [0, 6408.39] + - - [128, 448, 1, 768, 128, 128, 128, 768] + - [3, 1412.18] + - - [704, 6784, 1, 256, 704, 704, 704, 256] + - [0, 5338.02] + - - [1856, 704, 1, 1792, 1856, 1856, 1856, 1792] + - [0, 5042.1] + - - [1024, 2944, 1, 256, 1024, 1024, 1024, 256] + - [0, 5409.31] + - - [448, 64, 1, 3328, 448, 448, 448, 3328] + - [3, 715.234] + - - [3584, 5888, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 6631.11] + - - [1024, 256, 1, 3328, 1024, 1024, 1024, 3328] + - [1, 4025.04] + - - [5056, 64, 1, 256, 5056, 5056, 5056, 256] + - [3, 2656.24] + - - [1856, 2368, 1, 1792, 1856, 1856, 1856, 1792] + - [0, 6081.39] + - - [5056, 128, 1, 256, 5056, 5056, 5056, 256] + - [3, 3388.73] + - - [256, 5056, 1, 3328, 256, 256, 256, 3328] + - [0, 5036.84] + - - [3584, 5056, 1, 256, 3584, 3584, 3584, 256] + - [0, 6354.48] + - - [448, 5056, 1, 3328, 448, 448, 448, 3328] + - [0, 5470.24] + - - [2368, 5056, 1, 256, 2368, 2368, 2368, 256] + - [0, 6115.3] + - - [6784, 704, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 5966.85] + - - [256, 704, 1, 256, 256, 256, 256, 256] + - [4, 2208.64] + - - [128, 5888, 1, 3328, 128, 128, 128, 3328] + - [3, 4684.51] + - - [1856, 5056, 1, 3328, 1856, 1856, 1856, 3328] + - [0, 6319.16] + - - [5888, 5888, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 6690.28] + - - [4, 448, 1, 3328, 4, 4, 4, 3328] + - [6, 43.66] + - - [6784, 448, 1, 768, 6784, 6784, 6784, 768] + - [0, 6056.09] + - - [4288, 1856, 1, 768, 4288, 4288, 4288, 768] + - [0, 6229.18] + - - [1408, 1024, 1, 256, 1408, 1408, 1408, 256] + - [0, 4957.91] + - - [6784, 2944, 1, 768, 6784, 6784, 6784, 768] + - [0, 6501.66] + - - [2368, 1024, 1, 256, 2368, 2368, 2368, 256] + - [0, 5297.06] + - - [1024, 1408, 1, 256, 1024, 1024, 1024, 256] + - [0, 4777.27] + - - [6784, 1024, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 6258.24] + - - [2944, 5888, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 6546.03] + - - [2944, 4, 1, 3328, 2944, 2944, 2944, 3328] + - [6, 235.033] + - - [3584, 128, 1, 768, 3584, 3584, 3584, 768] + - [5, 3943.78] + - - [128, 4, 1, 256, 128, 128, 128, 256] + - [6, 10.4083] + - - [5056, 2368, 1, 256, 5056, 5056, 5056, 256] + - [0, 6090.1] + - - [6784, 1408, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 6381.27] + - - [2368, 1408, 1, 1792, 2368, 2368, 2368, 1792] + - [0, 5866.84] + - - [1856, 4288, 1, 1792, 1856, 1856, 1856, 1792] + - [0, 6312.37] + - - [5056, 4288, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 6655.94] + - - [4288, 1856, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 6326.65] + - - [5056, 704, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 6244.17] + - - [448, 1024, 1, 3328, 448, 448, 448, 3328] + - [3, 4291.11] + - - [1024, 448, 1, 256, 1024, 1024, 1024, 256] + - [0, 3209.33] + - - [5056, 448, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 5464.79] + - - [1408, 1408, 1, 1792, 1408, 1408, 1408, 1792] + - [0, 5888.8] + - - [256, 256, 1, 768, 256, 256, 256, 768] + - [3, 1551.29] + - - [64, 6784, 1, 768, 64, 64, 64, 768] + - [3, 3095.87] + - - [704, 704, 1, 3328, 704, 704, 704, 3328] + - [3, 4611.93] + - - [5056, 704, 1, 256, 5056, 5056, 5056, 256] + - [0, 5713.43] + - - [128, 2368, 1, 256, 128, 128, 128, 256] + - [3, 2573.7] + - - [2368, 256, 1, 256, 2368, 2368, 2368, 256] + - [3, 3384.49] + - - [1856, 5888, 1, 3328, 1856, 1856, 1856, 3328] + - [0, 6372.69] + - - [256, 256, 1, 1792, 256, 256, 256, 1792] + - [3, 1585.41] + - - [5056, 1024, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 6497.42] + - - [1408, 448, 1, 768, 1408, 1408, 1408, 768] + - [3, 4257.67] + - - [5888, 64, 1, 3328, 5888, 5888, 5888, 3328] + - [3, 4098.24] + - - [2944, 1408, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 6458.18] + - - [2368, 6784, 1, 256, 2368, 2368, 2368, 256] + - [0, 6207.22] + - - [2368, 448, 1, 3328, 2368, 2368, 2368, 3328] + - [3, 4477.75] + - - [704, 704, 1, 256, 704, 704, 704, 256] + - [3, 3425.61] + - - [5888, 128, 1, 768, 5888, 5888, 5888, 768] + - [3, 4223.66] + - - [2368, 64, 1, 1792, 2368, 2368, 2368, 1792] + - [3, 2631.89] + - - [256, 1024, 1, 768, 256, 256, 256, 768] + - [3, 4155.69] + - - [6784, 4, 1, 3328, 6784, 6784, 6784, 3328] + - [6, 440.438] + - - [1024, 1856, 1, 256, 1024, 1024, 1024, 256] + - [0, 4923.61] + - - [128, 2944, 1, 768, 128, 128, 128, 768] + - [3, 3641.09] + - - [1856, 3584, 1, 768, 1856, 1856, 1856, 768] + - [0, 5930.43] + - - [1408, 704, 1, 3328, 1408, 1408, 1408, 3328] + - [0, 5448.1] + - - [5888, 256, 1, 768, 5888, 5888, 5888, 768] + - [0, 5653.96] + - - [2944, 6784, 1, 768, 2944, 2944, 2944, 768] + - [0, 6533.66] + - - [1024, 448, 1, 1792, 1024, 1024, 1024, 1792] + - [5, 4077.09] + - - [1408, 4, 1, 3328, 1408, 1408, 1408, 3328] + - [6, 115.802] + - - [2368, 704, 1, 768, 2368, 2368, 2368, 768] + - [0, 4813.06] + - - [4, 4288, 1, 3328, 4, 4, 4, 3328] + - [6, 167.847] + - - [128, 64, 1, 1792, 128, 128, 128, 1792] + - [3, 227.526] + - - [1024, 256, 1, 768, 1024, 1024, 1024, 768] + - [1, 3623.78] + - - [6784, 4, 1, 1792, 6784, 6784, 6784, 1792] + - [6, 428.217] + - - [256, 128, 1, 3328, 256, 256, 256, 3328] + - [3, 830.789] + - - [5056, 4288, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 6637.19] + - - [6784, 5056, 1, 768, 6784, 6784, 6784, 768] + - [0, 6614.55] + - - [1408, 3584, 1, 256, 1408, 1408, 1408, 256] + - [0, 5947.96] + - - [128, 2368, 1, 3328, 128, 128, 128, 3328] + - [3, 3433.55] + - - [6784, 6784, 1, 256, 6784, 6784, 6784, 256] + - [0, 6452.06] + - - [6784, 2944, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6590.96] + - - [2944, 704, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 6200.99] + - - [1024, 5888, 1, 768, 1024, 1024, 1024, 768] + - [0, 6132.65] + - - [6784, 128, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 4813.35] + - - [448, 128, 1, 256, 448, 448, 448, 256] + - [3, 896.711] + - - [1024, 2368, 1, 256, 1024, 1024, 1024, 256] + - [0, 5147.41] + - - [4288, 5888, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 6638.65] + - - [256, 64, 1, 768, 256, 256, 256, 768] + - [3, 413.293] + - - [704, 4288, 1, 1792, 704, 704, 704, 1792] + - [5, 5733.95] + - - [4288, 3584, 1, 256, 4288, 4288, 4288, 256] + - [0, 6277.15] + - - [704, 4, 1, 256, 704, 704, 704, 256] + - [6, 41.4118] + - - [5056, 2368, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 6375.23] + - - [3584, 3584, 1, 256, 3584, 3584, 3584, 256] + - [0, 6228.6] + - - [5056, 4288, 1, 768, 5056, 5056, 5056, 768] + - [0, 6560.04] + - - [128, 2944, 1, 1792, 128, 128, 128, 1792] + - [3, 3984.29] + - - [4, 704, 1, 768, 4, 4, 4, 768] + - [6, 59.8271] + - - [2368, 1856, 1, 768, 2368, 2368, 2368, 768] + - [0, 5971.97] + - - [4, 4, 1, 3328, 4, 4, 4, 3328] + - [6, 0.36882] + - - [1856, 256, 1, 1792, 1856, 1856, 1856, 1792] + - [3, 4288.87] + - - [1408, 448, 1, 256, 1408, 1408, 1408, 256] + - [3, 3621.13] + - - [1856, 4, 1, 1792, 1856, 1856, 1856, 1792] + - [6, 149.789] + - - [4288, 3584, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 6588.32] + - - [4, 704, 1, 3328, 4, 4, 4, 3328] + - [6, 58.2745] + - - [64, 5888, 1, 3328, 64, 64, 64, 3328] + - [3, 3945.43] + - - [2944, 5888, 1, 256, 2944, 2944, 2944, 256] + - [0, 6314.48] + - - [2944, 2368, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 6325.9] + - - [6784, 64, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 3900.22] + - - [2944, 256, 1, 1792, 2944, 2944, 2944, 1792] + - [3, 4615.73] + - - [5888, 2368, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 6623.07] + - - [448, 256, 1, 3328, 448, 448, 448, 3328] + - [3, 2771.68] + - - [4, 5888, 1, 768, 4, 4, 4, 768] + - [6, 240.808] + - - [128, 5888, 1, 256, 128, 128, 128, 256] + - [5, 3427.08] + - - [2944, 5056, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 6578.32] + - - [4, 448, 1, 256, 4, 4, 4, 256] + - [6, 31.1208] + - - [6784, 1408, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6414.95] + - - [4, 1408, 1, 256, 4, 4, 4, 256] + - [6, 82.8235] + - - [5056, 448, 1, 768, 5056, 5056, 5056, 768] + - [0, 5348.15] + - - [1024, 704, 1, 3328, 1024, 1024, 1024, 3328] + - [1, 4072.87] + - - [1856, 256, 1, 768, 1856, 1856, 1856, 768] + - [5, 4051.03] + - - [1856, 64, 1, 256, 1856, 1856, 1856, 256] + - [3, 1732.1] + - - [64, 2368, 1, 256, 64, 64, 64, 256] + - [3, 1837.69] + - - [1856, 4, 1, 256, 1856, 1856, 1856, 256] + - [6, 110.116] + - - [1024, 256, 1, 1792, 1024, 1024, 1024, 1792] + - [1, 3907.34] + - - [64, 2368, 1, 3328, 64, 64, 64, 3328] + - [3, 2741.03] + - - [5056, 256, 1, 768, 5056, 5056, 5056, 768] + - [0, 4876.19] + - - [4288, 2944, 1, 768, 4288, 4288, 4288, 768] + - [0, 6389.72] + - - [4288, 256, 1, 1792, 4288, 4288, 4288, 1792] + - [3, 4300.48] + - - [448, 1024, 1, 1792, 448, 448, 448, 1792] + - [3, 4183.12] + - - [1408, 1408, 1, 256, 1408, 1408, 1408, 256] + - [0, 5327.53] + - - [2944, 3584, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 6407.67] + - - [4, 1856, 1, 1792, 4, 4, 4, 1792] + - [6, 140.419] + - - [256, 128, 1, 256, 256, 256, 256, 256] + - [3, 526.708] + - - [256, 2944, 1, 256, 256, 256, 256, 256] + - [3, 3683.99] + - - [704, 1856, 1, 3328, 704, 704, 704, 3328] + - [0, 5085.09] + - - [2944, 5888, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 6572.75] + - - [2944, 6784, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 6584.05] + - - [704, 3584, 1, 256, 704, 704, 704, 256] + - [5, 5256.0] + - - [5888, 4288, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 6647.28] + - - [64, 1408, 1, 768, 64, 64, 64, 768] + - [3, 1835.46] + - - [2368, 4288, 1, 256, 2368, 2368, 2368, 256] + - [0, 6147.71] + - - [1856, 64, 1, 3328, 1856, 1856, 1856, 3328] + - [3, 2867.58] + - - [4, 704, 1, 1792, 4, 4, 4, 1792] + - [6, 60.8784] + - - [256, 4288, 1, 768, 256, 256, 256, 768] + - [3, 4355.5] + - - [1408, 256, 1, 1792, 1408, 1408, 1408, 1792] + - [3, 4018.31] + - - [2944, 448, 1, 256, 2944, 2944, 2944, 256] + - [0, 4580.92] + - - [4, 1408, 1, 3328, 4, 4, 4, 3328] + - [6, 113.161] + - - [4288, 1024, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 6108.92] + - - [5056, 256, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 4987.52] + - - [3584, 448, 1, 256, 3584, 3584, 3584, 256] + - [0, 4352.09] + - - [448, 128, 1, 3328, 448, 448, 448, 3328] + - [3, 1420.22] + - - [1024, 2368, 1, 1792, 1024, 1024, 1024, 1792] + - [0, 5811.36] + - - [64, 3584, 1, 3328, 64, 64, 64, 3328] + - [3, 3909.37] + - - [1024, 1024, 1, 1792, 1024, 1024, 1024, 1792] + - [0, 5564.03] + - - [1856, 2368, 1, 3328, 1856, 1856, 1856, 3328] + - [0, 6120.87] + - - [1408, 5888, 1, 768, 1408, 1408, 1408, 768] + - [0, 6487.39] + - - [5888, 64, 1, 256, 5888, 5888, 5888, 256] + - [3, 2718.78] + - - [4288, 5056, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 6640.88] + - - [4288, 4, 1, 768, 4288, 4288, 4288, 768] + - [6, 288.68] + - - [2368, 1408, 1, 256, 2368, 2368, 2368, 256] + - [0, 5424.89] + - - [2944, 1856, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 6262.15] + - - [4, 6784, 1, 3328, 4, 4, 4, 3328] + - [6, 170.009] + - - [4, 6784, 1, 256, 4, 4, 4, 256] + - [6, 197.847] + - - [1024, 4288, 1, 3328, 1024, 1024, 1024, 3328] + - [0, 6083.49] + - - [448, 1408, 1, 768, 448, 448, 448, 768] + - [3, 4316.68] + - - [6784, 2944, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 6567.76] + - - [448, 1024, 1, 256, 448, 448, 448, 256] + - [3, 3248.79] + - - [4, 4, 1, 768, 4, 4, 4, 768] + - [6, 0.312398] + - - [128, 4288, 1, 256, 128, 128, 128, 256] + - [3, 2973.08] + - - [5888, 1856, 1, 256, 5888, 5888, 5888, 256] + - [0, 6245.49] + - - [128, 3584, 1, 1792, 128, 128, 128, 1792] + - [5, 4161.16] + - - [1856, 1856, 1, 768, 1856, 1856, 1856, 768] + - [0, 5905.09] + - - [4288, 5056, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 6662.95] + - - [64, 256, 1, 1792, 64, 64, 64, 1792] + - [3, 433.171] + - - [256, 5056, 1, 1792, 256, 256, 256, 1792] + - [0, 5001.07] + - - [3584, 2368, 1, 256, 3584, 3584, 3584, 256] + - [0, 5954.1] + - - [2944, 64, 1, 256, 2944, 2944, 2944, 256] + - [3, 2367.8] + - - [1024, 4288, 1, 1792, 1024, 1024, 1024, 1792] + - [0, 6038.08] + - - [1408, 4288, 1, 256, 1408, 1408, 1408, 256] + - [0, 5978.51] + - - [128, 64, 1, 256, 128, 128, 128, 256] + - [3, 142.984] + - - [6784, 6784, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 6739.88] + - - [64, 5888, 1, 256, 64, 64, 64, 256] + - [5, 2163.27] + - - [4, 1856, 1, 256, 4, 4, 4, 256] + - [6, 101.411] + - - [448, 448, 1, 3328, 448, 448, 448, 3328] + - [3, 3646.17] + - - [1408, 1408, 1, 3328, 1408, 1408, 1408, 3328] + - [0, 5935.73] + - - [2944, 4288, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 6487.49] + - - [2368, 5888, 1, 1792, 2368, 2368, 2368, 1792] + - [0, 6384.03] + - - [448, 2368, 1, 256, 448, 448, 448, 256] + - [3, 3684.68] + - - [4288, 128, 1, 3328, 4288, 4288, 4288, 3328] + - [3, 3806.52] + - - [4, 6784, 1, 768, 4, 4, 4, 768] + - [6, 239.437] + - - [1408, 2944, 1, 3328, 1408, 1408, 1408, 3328] + - [0, 6466.48] + - - [64, 4, 1, 3328, 64, 64, 64, 3328] + - [6, 6.07247] + - - [1024, 6784, 1, 768, 1024, 1024, 1024, 768] + - [0, 6090.7] + - - [448, 1408, 1, 256, 448, 448, 448, 256] + - [3, 3603.19] + - - [5888, 448, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 5418.78] + - - [4, 4288, 1, 1792, 4, 4, 4, 1792] + - [6, 158.31] + - - [448, 1024, 1, 768, 448, 448, 448, 768] + - [3, 3934.0] + - - [2944, 448, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 5140.49] + - - [5056, 2944, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 6568.12] + - - [704, 6784, 1, 3328, 704, 704, 704, 3328] + - [0, 5992.66] + - - [4288, 1856, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 6308.02] + - - [2368, 1024, 1, 768, 2368, 2368, 2368, 768] + - [0, 5680.05] + - - [128, 1856, 1, 1792, 128, 128, 128, 1792] + - [3, 4099.21] + - - [128, 256, 1, 1792, 128, 128, 128, 1792] + - [3, 813.864] + - - [4, 128, 1, 3328, 4, 4, 4, 3328] + - [6, 12.515] + - - [64, 256, 1, 768, 64, 64, 64, 768] + - [3, 408.324] + - - [704, 4, 1, 768, 704, 704, 704, 768] + - [6, 56.3618] + - - [256, 5056, 1, 256, 256, 256, 256, 256] + - [0, 4219.89] + - - [448, 6784, 1, 256, 448, 448, 448, 256] + - [5, 5272.73] + - - [6784, 3584, 1, 768, 6784, 6784, 6784, 768] + - [0, 6567.12] + - - [448, 256, 1, 1792, 448, 448, 448, 1792] + - [3, 2717.4] + - - [448, 64, 1, 256, 448, 448, 448, 256] + - [3, 516.086] + - - [128, 448, 1, 256, 128, 128, 128, 256] + - [3, 921.738] + - - [1856, 128, 1, 768, 1856, 1856, 1856, 768] + - [3, 3783.43] + - - [128, 64, 1, 768, 128, 128, 128, 768] + - [3, 206.149] + - - [128, 6784, 1, 3328, 128, 128, 128, 3328] + - [0, 4750.64] + - - [4, 2944, 1, 768, 4, 4, 4, 768] + - [6, 199.823] + - - [2944, 4288, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 6459.27] + - - [448, 448, 1, 256, 448, 448, 448, 256] + - [3, 2391.78] + - - [5056, 5888, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 6635.16] + - - [128, 704, 1, 256, 128, 128, 128, 256] + - [3, 1371.91] + - - [6784, 256, 1, 768, 6784, 6784, 6784, 768] + - [0, 5042.18] + - - [2368, 448, 1, 256, 2368, 2368, 2368, 256] + - [0, 3658.92] + - - [1856, 5888, 1, 256, 1856, 1856, 1856, 256] + - [0, 6063.02] + - - [448, 2368, 1, 768, 448, 448, 448, 768] + - [3, 4264.72] + - - [6784, 1024, 1, 768, 6784, 6784, 6784, 768] + - [0, 6174.77] + - - [5056, 1024, 1, 768, 5056, 5056, 5056, 768] + - [0, 6341.07] + - - [1024, 1408, 1, 3328, 1024, 1024, 1024, 3328] + - [0, 5597.56] + - - [4, 1856, 1, 768, 4, 4, 4, 768] + - [6, 134.563] + - - [5888, 256, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 5871.78] + - - [5888, 4288, 1, 256, 5888, 5888, 5888, 256] + - [0, 6354.04] + - - [1856, 256, 1, 256, 1856, 1856, 1856, 256] + - [5, 3378.65] + - - [5888, 448, 1, 256, 5888, 5888, 5888, 256] + - [0, 4976.0] + - - [2944, 5888, 1, 768, 2944, 2944, 2944, 768] + - [0, 6496.64] + - - [256, 2944, 1, 3328, 256, 256, 256, 3328] + - [3, 4773.67] + - - [448, 5888, 1, 256, 448, 448, 448, 256] + - [5, 4701.41] + - - [128, 256, 1, 768, 128, 128, 128, 768] + - [3, 797.485] + - - [64, 256, 1, 256, 64, 64, 64, 256] + - [3, 301.174] + - - [704, 4, 1, 1792, 704, 704, 704, 1792] + - [6, 61.0419] + - - [256, 5056, 1, 768, 256, 256, 256, 768] + - [0, 4834.04] + - - [4288, 128, 1, 768, 4288, 4288, 4288, 768] + - [3, 3615.28] + - - [128, 4, 1, 768, 128, 128, 128, 768] + - [6, 13.4728] + - - [128, 5056, 1, 1792, 128, 128, 128, 1792] + - [3, 4516.97] + - - [448, 256, 1, 256, 448, 448, 448, 256] + - [3, 1708.42] + - - [4288, 4288, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 6561.11] + - - [4288, 6784, 1, 768, 4288, 4288, 4288, 768] + - [0, 6533.53] + - - [128, 64, 1, 3328, 128, 128, 128, 3328] + - [3, 219.464] + - - [64, 5888, 1, 1792, 64, 64, 64, 1792] + - [3, 3662.54] + - - [2368, 2944, 1, 256, 2368, 2368, 2368, 256] + - [0, 5948.75] + - - [3584, 4, 1, 3328, 3584, 3584, 3584, 3328] + - [6, 283.982] + - - [4, 2368, 1, 3328, 4, 4, 4, 3328] + - [6, 172.636] + - - [448, 448, 1, 768, 448, 448, 448, 768] + - [3, 3201.26] + - - [2368, 448, 1, 768, 2368, 2368, 2368, 768] + - [3, 4202.8] + - - [128, 4288, 1, 768, 128, 128, 128, 768] + - [3, 3585.67] + - - [64, 5056, 1, 256, 64, 64, 64, 256] + - [5, 1948.22] + - - [5056, 6784, 1, 256, 5056, 5056, 5056, 256] + - [0, 6362.7] + - - [6784, 128, 1, 768, 6784, 6784, 6784, 768] + - [0, 4600.32] + - - [5056, 4, 1, 1792, 5056, 5056, 5056, 1792] + - [6, 350.213] + - - [2368, 1408, 1, 3328, 2368, 2368, 2368, 3328] + - [0, 5903.6] + - - [448, 4, 1, 256, 448, 448, 448, 256] + - [6, 28.4075] + - - [704, 256, 1, 1792, 704, 704, 704, 1792] + - [3, 3196.37] + - - [3584, 4288, 1, 768, 3584, 3584, 3584, 768] + - [0, 6461.71] + - - [6784, 2368, 1, 256, 6784, 6784, 6784, 256] + - [0, 6295.91] + - - [2944, 6784, 1, 256, 2944, 2944, 2944, 256] + - [0, 6352.07] + - - [5888, 5056, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 6742.43] + - - [5056, 4, 1, 256, 5056, 5056, 5056, 256] + - [6, 250.513] + - - [4288, 5888, 1, 256, 4288, 4288, 4288, 256] + - [0, 6378.19] + - - [5888, 1856, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 6647.42] + - - [256, 704, 1, 1792, 256, 256, 256, 1792] + - [3, 3189.36] + - - [1408, 64, 1, 256, 1408, 1408, 1408, 256] + - [3, 1564.93] + - - [5888, 128, 1, 256, 5888, 5888, 5888, 256] + - [0, 3617.51] + - - [448, 4, 1, 1792, 448, 448, 448, 1792] + - [6, 42.0067] + - - [128, 5056, 1, 256, 128, 128, 128, 256] + - [3, 3289.03] + - - [1024, 3584, 1, 3328, 1024, 1024, 1024, 3328] + - [0, 6449.51] + - - [1856, 4288, 1, 3328, 1856, 1856, 1856, 3328] + - [0, 6339.1] + - - [448, 256, 1, 768, 448, 448, 448, 768] + - [3, 2684.11] + - - [4288, 3584, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 6565.94] + - - [5056, 1408, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 6449.28] + - - [64, 5056, 1, 768, 64, 64, 64, 768] + - [3, 2819.17] + - - [64, 1856, 1, 256, 64, 64, 64, 256] + - [3, 1769.44] + - - [448, 4288, 1, 3328, 448, 448, 448, 3328] + - [5, 5424.26] + - - [64, 2944, 1, 256, 64, 64, 64, 256] + - [3, 2268.84] + - - [2944, 448, 1, 768, 2944, 2944, 2944, 768] + - [0, 4988.72] + - - [1024, 1024, 1, 3328, 1024, 1024, 1024, 3328] + - [0, 5664.87] + - - [64, 4288, 1, 1792, 64, 64, 64, 1792] + - [3, 2805.95] + - - [448, 448, 1, 1792, 448, 448, 448, 1792] + - [3, 3528.57] + - - [4288, 1856, 1, 256, 4288, 4288, 4288, 256] + - [0, 5958.38] + - - [448, 3584, 1, 256, 448, 448, 448, 256] + - [5, 4104.22] + - - [128, 4288, 1, 1792, 128, 128, 128, 1792] + - [3, 3885.79] + - - [704, 704, 1, 1792, 704, 704, 704, 1792] + - [3, 4522.67] + - - [704, 3584, 1, 768, 704, 704, 704, 768] + - [5, 5581.2] + - - [5056, 1856, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 6306.97] + - - [448, 2368, 1, 1792, 448, 448, 448, 1792] + - [3, 4420.2] + - - [5888, 2368, 1, 768, 5888, 5888, 5888, 768] + - [0, 6495.38] + - - [2368, 3584, 1, 256, 2368, 2368, 2368, 256] + - [0, 6037.45] + - - [4, 256, 1, 256, 4, 4, 4, 256] + - [6, 20.1069] + - - [64, 2368, 1, 1792, 64, 64, 64, 1792] + - [3, 2600.17] + - - [448, 1408, 1, 3328, 448, 448, 448, 3328] + - [3, 4610.22] + - - [6784, 4288, 1, 256, 6784, 6784, 6784, 256] + - [0, 6389.44] + - - [5888, 3584, 1, 256, 5888, 5888, 5888, 256] + - [0, 6291.82] + - - [6784, 1856, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 6430.13] + - - [4, 4, 1, 1792, 4, 4, 4, 1792] + - [6, 0.357397] + - - [256, 5888, 1, 1792, 256, 256, 256, 1792] + - [0, 5796.25] + - - [3584, 2368, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 6325.78] + - - [1408, 5888, 1, 256, 1408, 1408, 1408, 256] + - [0, 6255.56] + - - [4, 128, 1, 1792, 4, 4, 4, 1792] + - [6, 12.2755] + - - [1408, 64, 1, 1792, 1408, 1408, 1408, 1792] + - [3, 2114.38] + - - [4, 1408, 1, 1792, 4, 4, 4, 1792] + - [6, 110.41] + - - [128, 5056, 1, 768, 128, 128, 128, 768] + - [3, 4059.07] + - - [2944, 2368, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 6297.5] + - - [1856, 1408, 1, 1792, 1856, 1856, 1856, 1792] + - [5, 5939.3] + - - [256, 4, 1, 3328, 256, 256, 256, 3328] + - [6, 25.4031] + - - [704, 256, 1, 256, 704, 704, 704, 256] + - [4, 2216.49] + - - [448, 704, 1, 256, 448, 448, 448, 256] + - [3, 2831.06] + - - [704, 4288, 1, 768, 704, 704, 704, 768] + - [5, 5572.92] + - - [3584, 1856, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 6474.04] + - - [128, 6784, 1, 256, 128, 128, 128, 256] + - [5, 3808.27] + - - [6784, 5056, 1, 1792, 6784, 6784, 6784, 1792] + - [0, 6687.56] + - - [4, 1024, 1, 768, 4, 4, 4, 768] + - [6, 79.8236] + - - [2944, 1024, 1, 768, 2944, 2944, 2944, 768] + - [0, 6021.01] + - - [128, 1856, 1, 768, 128, 128, 128, 768] + - [3, 3698.27] + - - [5888, 2944, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 6532.36] + - - [64, 3584, 1, 256, 64, 64, 64, 256] + - [3, 2508.56] + - - [3584, 704, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 6074.03] + - - [5888, 2368, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 6594.34] + - - [704, 704, 1, 768, 704, 704, 704, 768] + - [3, 4292.74] + - - [704, 3584, 1, 1792, 704, 704, 704, 1792] + - [5, 5723.74] + - - [1024, 5056, 1, 1792, 1024, 1024, 1024, 1792] + - [0, 6405.17] + - - [5056, 6784, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 6625.97] + - - [5888, 4, 1, 1792, 5888, 5888, 5888, 1792] + - [6, 414.972] + - - [256, 2368, 1, 256, 256, 256, 256, 256] + - [3, 3434.42] + - - [64, 4, 1, 768, 64, 64, 64, 768] + - [6, 5.24533] + - - [704, 2368, 1, 256, 704, 704, 704, 256] + - [0, 4322.01] + - - [448, 704, 1, 768, 448, 448, 448, 768] + - [3, 3309.62] + - - [2944, 704, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 6141.39] + - - [4, 3584, 1, 1792, 4, 4, 4, 1792] + - [6, 184.766] + - - [6784, 4288, 1, 768, 6784, 6784, 6784, 768] + - [0, 6605.8] + - - [5056, 128, 1, 3328, 5056, 5056, 5056, 3328] + - [3, 4393.34] + - - [2944, 2368, 1, 768, 2944, 2944, 2944, 768] + - [0, 6220.08] + - - [64, 448, 1, 1792, 64, 64, 64, 1792] + - [3, 681.349] + - - [5888, 128, 1, 1792, 5888, 5888, 5888, 1792] + - [3, 4458.82] + - - [1856, 1024, 1, 256, 1856, 1856, 1856, 256] + - [0, 4919.93] + - - [2944, 1408, 1, 1792, 2944, 2944, 2944, 1792] + - [0, 6423.05] + - - [1408, 5056, 1, 1792, 1408, 1408, 1408, 1792] + - [0, 6432.38] + - - [2944, 1024, 1, 256, 2944, 2944, 2944, 256] + - [0, 5634.58] + - - [704, 5056, 1, 3328, 704, 704, 704, 3328] + - [5, 5881.97] + - - [6784, 2368, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6650.32] + - - [5888, 448, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 5388.2] + - - [5056, 256, 1, 256, 5056, 5056, 5056, 256] + - [0, 4455.28] + - - [5888, 704, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 6453.25] + - - [5888, 4288, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 6617.29] + - - [5056, 1408, 1, 256, 5056, 5056, 5056, 256] + - [0, 6025.47] + - - [5056, 1408, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 6413.94] + - - [128, 6784, 1, 768, 128, 128, 128, 768] + - [5, 4487.94] + - - [1408, 1024, 1, 1792, 1408, 1408, 1408, 1792] + - [0, 5574.37] + - - [1408, 1024, 1, 3328, 1408, 1408, 1408, 3328] + - [0, 5625.76] + - - [2944, 1408, 1, 256, 2944, 2944, 2944, 256] + - [0, 5981.31] + - - [128, 1856, 1, 3328, 128, 128, 128, 3328] + - [3, 4288.13] + - - [4288, 128, 1, 1792, 4288, 4288, 4288, 1792] + - [3, 3732.85] + - - [1408, 128, 1, 768, 1408, 1408, 1408, 768] + - [3, 2883.52] + - - [6784, 5888, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6705.63] + - - [2944, 64, 1, 3328, 2944, 2944, 2944, 3328] + - [3, 3417.4] + - - [5888, 4, 1, 256, 5888, 5888, 5888, 256] + - [6, 290.688] + - - [5888, 1408, 1, 768, 5888, 5888, 5888, 768] + - [0, 6453.78] + - - [5056, 256, 1, 3328, 5056, 5056, 5056, 3328] + - [0, 5048.64] + - - [1024, 64, 1, 256, 1024, 1024, 1024, 256] + - [3, 1029.5] + - - [2944, 4288, 1, 256, 2944, 2944, 2944, 256] + - [0, 6203.16] + - - [3584, 1024, 1, 256, 3584, 3584, 3584, 256] + - [0, 5855.59] + - - [1408, 2368, 1, 3328, 1408, 1408, 1408, 3328] + - [0, 5910.59] + - - [6784, 1408, 1, 768, 6784, 6784, 6784, 768] + - [0, 6299.68] + - - [256, 128, 1, 768, 256, 256, 256, 768] + - [3, 814.705] + - - [704, 1408, 1, 1792, 704, 704, 704, 1792] + - [5, 5333.45] + - - [1856, 448, 1, 256, 1856, 1856, 1856, 256] + - [0, 3857.08] + - - [6784, 1024, 1, 256, 6784, 6784, 6784, 256] + - [0, 5900.1] + - - [3584, 1856, 1, 768, 3584, 3584, 3584, 768] + - [0, 6333.45] + - - [5888, 5888, 1, 256, 5888, 5888, 5888, 256] + - [0, 6421.43] + - - [448, 6784, 1, 3328, 448, 448, 448, 3328] + - [5, 5832.92] + - - [256, 4, 1, 768, 256, 256, 256, 768] + - [6, 25.9574] + - - [64, 1024, 1, 1792, 64, 64, 64, 1792] + - [3, 1524.43] + - - [3584, 4288, 1, 256, 3584, 3584, 3584, 256] + - [0, 6216.16] + - - [6784, 5888, 1, 768, 6784, 6784, 6784, 768] + - [0, 6610.52] + - - [1856, 3584, 1, 1792, 1856, 1856, 1856, 1792] + - [0, 6008.45] + - - [128, 6784, 1, 1792, 128, 128, 128, 1792] + - [5, 4651.58] + - - [4, 5056, 1, 256, 4, 4, 4, 256] + - [6, 183.444] + - - [2944, 5056, 1, 256, 2944, 2944, 2944, 256] + - [0, 6304.77] + - - [3584, 1856, 1, 256, 3584, 3584, 3584, 256] + - [0, 5989.66] + - - [1856, 1024, 1, 3328, 1856, 1856, 1856, 3328] + - [0, 5686.28] + - - [2944, 704, 1, 256, 2944, 2944, 2944, 256] + - [0, 5552.4] + - - [6784, 4288, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6700.31] + - - [256, 5888, 1, 768, 256, 256, 256, 768] + - [0, 5567.35] + - - [128, 4, 1, 1792, 128, 128, 128, 1792] + - [6, 14.2696] + - - [4288, 4, 1, 1792, 4288, 4288, 4288, 1792] + - [6, 310.577] + - - [4288, 2368, 1, 1792, 4288, 4288, 4288, 1792] + - [0, 6460.93] + - - [1856, 2944, 1, 3328, 1856, 1856, 1856, 3328] + - [0, 6269.5] + - - [3584, 3584, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 6584.43] + - - [2368, 4, 1, 768, 2368, 2368, 2368, 768] + - [6, 172.894] + - - [2368, 3584, 1, 768, 2368, 2368, 2368, 768] + - [0, 6269.31] + - - [4288, 64, 1, 3328, 4288, 4288, 4288, 3328] + - [3, 3004.22] + - - [3584, 3584, 1, 768, 3584, 3584, 3584, 768] + - [0, 6468.51] + - - [4288, 704, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 6183.21] + - - [2944, 1856, 1, 256, 2944, 2944, 2944, 256] + - [0, 5880.8] + - - [1408, 256, 1, 768, 1408, 1408, 1408, 768] + - [3, 3707.37] + - - [2944, 4, 1, 256, 2944, 2944, 2944, 256] + - [6, 160.222] + - - [4288, 2368, 1, 768, 4288, 4288, 4288, 768] + - [0, 6398.44] + - - [704, 5888, 1, 256, 704, 704, 704, 256] + - [5, 5571.85] + - - [704, 2368, 1, 1792, 704, 704, 704, 1792] + - [0, 4924.73] + - - [704, 1408, 1, 768, 704, 704, 704, 768] + - [5, 5169.42] + - - [256, 64, 1, 1792, 256, 256, 256, 1792] + - [3, 442.848] + - - [5888, 4, 1, 768, 5888, 5888, 5888, 768] + - [6, 379.755] + - - [5056, 3584, 1, 256, 5056, 5056, 5056, 256] + - [0, 6275.51] + - - [2368, 64, 1, 3328, 2368, 2368, 2368, 3328] + - [3, 2731.14] + - - [128, 1024, 1, 1792, 128, 128, 128, 1792] + - [3, 2874.71] + - - [256, 4, 1, 256, 256, 256, 256, 256] + - [6, 19.7703] + - - [5056, 5056, 1, 256, 5056, 5056, 5056, 256] + - [0, 6324.72] + - - [64, 1024, 1, 768, 64, 64, 64, 768] + - [3, 1412.6] + - - [3584, 704, 1, 256, 3584, 3584, 3584, 256] + - [0, 5511.83] + - - [256, 4288, 1, 256, 256, 256, 256, 256] + - [3, 3708.34] + - - [5056, 1408, 1, 768, 5056, 5056, 5056, 768] + - [0, 6313.61] + - - [5888, 6784, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 6708.94] + - - [4, 3584, 1, 256, 4, 4, 4, 256] + - [6, 160.856] + - - [4288, 1408, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 6363.08] + - - [64, 2944, 1, 1792, 64, 64, 64, 1792] + - [3, 3115.54] + - - [5888, 2944, 1, 3328, 5888, 5888, 5888, 3328] + - [0, 6559.35] + - - [256, 1856, 1, 768, 256, 256, 256, 768] + - [3, 4047.7] + - - [4, 5056, 1, 3328, 4, 4, 4, 3328] + - [6, 171.272] + - - [256, 256, 1, 256, 256, 256, 256, 256] + - [3, 1048.54] + - - [4288, 64, 1, 256, 4288, 4288, 4288, 256] + - [3, 2359.2] + - - [704, 5056, 1, 768, 704, 704, 704, 768] + - [5, 5653.04] + - - [448, 4288, 1, 256, 448, 448, 448, 256] + - [5, 4705.08] + - - [1856, 5888, 1, 1792, 1856, 1856, 1856, 1792] + - [0, 6342.9] + - - [64, 4, 1, 256, 64, 64, 64, 256] + - [6, 4.02148] + - - [704, 2368, 1, 3328, 704, 704, 704, 3328] + - [0, 4973.84] + - - [64, 4288, 1, 3328, 64, 64, 64, 3328] + - [3, 2960.93] + - - [2944, 1856, 1, 768, 2944, 2944, 2944, 768] + - [0, 6151.57] + - - [2368, 4288, 1, 1792, 2368, 2368, 2368, 1792] + - [0, 6468.59] + - - [1024, 128, 1, 256, 1024, 1024, 1024, 256] + - [4, 1671.5] + - - [4, 2944, 1, 256, 4, 4, 4, 256] + - [6, 142.298] + - - [64, 128, 1, 256, 64, 64, 64, 256] + - [3, 148.224] + - - [2368, 5056, 1, 768, 2368, 2368, 2368, 768] + - [0, 6312.79] + - - [1408, 4288, 1, 768, 1408, 1408, 1408, 768] + - [0, 6244.52] + - - [1024, 448, 1, 768, 1024, 1024, 1024, 768] + - [0, 3885.8] + - - [1856, 2944, 1, 1792, 1856, 1856, 1856, 1792] + - [0, 6223.01] + - - [2368, 128, 1, 3328, 2368, 2368, 2368, 3328] + - [3, 3406.92] + - - [3584, 2944, 1, 256, 3584, 3584, 3584, 256] + - [0, 6083.45] + - - [1024, 3584, 1, 768, 1024, 1024, 1024, 768] + - [0, 6206.1] + - - [1408, 1024, 1, 768, 1408, 1408, 1408, 768] + - [0, 5427.76] + - - [5888, 5056, 1, 768, 5888, 5888, 5888, 768] + - [0, 6644.93] + - - [1408, 256, 1, 3328, 1408, 1408, 1408, 3328] + - [3, 4084.17] + - - [5888, 5888, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 6670.3] + - - [6784, 5056, 1, 256, 6784, 6784, 6784, 256] + - [0, 6400.49] + - - [128, 2944, 1, 3328, 128, 128, 128, 3328] + - [3, 4209.3] + - - [64, 1856, 1, 1792, 64, 64, 64, 1792] + - [3, 2522.87] + - - [1856, 1856, 1, 3328, 1856, 1856, 1856, 3328] + - [0, 6101.59] + - - [5888, 6784, 1, 768, 5888, 5888, 5888, 768] + - [0, 6616.54] + - - [4288, 4288, 1, 256, 4288, 4288, 4288, 256] + - [0, 6300.79] + - - [2944, 2944, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 6502.44] + - - [704, 1024, 1, 768, 704, 704, 704, 768] + - [3, 4263.57] + - - [1408, 448, 1, 1792, 1408, 1408, 1408, 1792] + - [3, 4518.65] + - - [64, 2944, 1, 768, 64, 64, 64, 768] + - [3, 2743.61] + - - [1408, 2368, 1, 1792, 1408, 1408, 1408, 1792] + - [0, 5880.96] + - - [1856, 5056, 1, 1792, 1856, 1856, 1856, 1792] + - [0, 6287.66] + - - [64, 448, 1, 768, 64, 64, 64, 768] + - [3, 704.418] + - - [5888, 5056, 1, 256, 5888, 5888, 5888, 256] + - [0, 6427.26] + - - [4288, 2944, 1, 256, 4288, 4288, 4288, 256] + - [0, 6155.9] + - - [256, 2368, 1, 3328, 256, 256, 256, 3328] + - [3, 4460.36] + - - [1024, 1408, 1, 768, 1024, 1024, 1024, 768] + - [0, 5336.69] + - - [3584, 2944, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 6425.59] + - - [4288, 1408, 1, 768, 4288, 4288, 4288, 768] + - [0, 6227.74] + - - [1024, 3584, 1, 1792, 1024, 1024, 1024, 1792] + - [0, 6388.15] + - - [1408, 4288, 1, 3328, 1408, 1408, 1408, 3328] + - [0, 6369.2] + - - [64, 64, 1, 256, 64, 64, 64, 256] + - [3, 55.9494] + - - [5888, 1024, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 6304.87] + - - [1856, 2944, 1, 256, 1856, 1856, 1856, 256] + - [0, 5831.77] + - - [1856, 1024, 1, 768, 1856, 1856, 1856, 768] + - [0, 5488.66] + - - [64, 4288, 1, 256, 64, 64, 64, 256] + - [3, 1996.66] + - - [5056, 1856, 1, 256, 5056, 5056, 5056, 256] + - [0, 5992.23] + - - [4288, 1408, 1, 256, 4288, 4288, 4288, 256] + - [0, 5937.69] + - - [3584, 1408, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 6289.33] + - - [2368, 128, 1, 768, 2368, 2368, 2368, 768] + - [3, 3074.86] + - - [4, 64, 1, 1792, 4, 4, 4, 1792] + - [6, 6.06558] + - - [64, 1856, 1, 3328, 64, 64, 64, 3328] + - [3, 2725.53] + - - [3584, 6784, 1, 768, 3584, 3584, 3584, 768] + - [0, 6597.89] + - - [5056, 64, 1, 768, 5056, 5056, 5056, 768] + - [3, 3266.63] + - - [5056, 64, 1, 3328, 5056, 5056, 5056, 3328] + - [3, 3500.5] + - - [1024, 5888, 1, 1792, 1024, 1024, 1024, 1792] + - [0, 6263.45] + - - [2944, 2368, 1, 256, 2944, 2944, 2944, 256] + - [0, 5978.27] + - - [256, 6784, 1, 768, 256, 256, 256, 768] + - [0, 5005.74] + - - [4, 1408, 1, 768, 4, 4, 4, 768] + - [6, 105.973] + - - [64, 1408, 1, 1792, 64, 64, 64, 1792] + - [3, 2033.52] + - - [2944, 4, 1, 1792, 2944, 2944, 2944, 1792] + - [6, 229.554] + - - [256, 1856, 1, 1792, 256, 256, 256, 1792] + - [3, 4332.52] + - - [2944, 4, 1, 768, 2944, 2944, 2944, 768] + - [6, 205.54] + - - [704, 5888, 1, 1792, 704, 704, 704, 1792] + - [5, 5973.11] + - - [448, 2368, 1, 3328, 448, 448, 448, 3328] + - [3, 4511.66] + - - [1856, 2368, 1, 768, 1856, 1856, 1856, 768] + - [0, 5978.24] + - - [1856, 6784, 1, 3328, 1856, 1856, 1856, 3328] + - [0, 6471.68] + - - [64, 448, 1, 256, 64, 64, 64, 256] + - [3, 483.358] + - - [1024, 6784, 1, 256, 1024, 1024, 1024, 256] + - [0, 5766.87] + - - [4288, 4, 1, 256, 4288, 4288, 4288, 256] + - [6, 222.844] + - - [5056, 448, 1, 256, 5056, 5056, 5056, 256] + - [0, 4982.77] + - - [64, 3584, 1, 768, 64, 64, 64, 768] + - [3, 3068.52] + - - [1856, 6784, 1, 1792, 1856, 1856, 1856, 1792] + - [0, 6444.0] + - - [5888, 1024, 1, 768, 5888, 5888, 5888, 768] + - [0, 6210.64] + - - [4, 64, 1, 768, 4, 4, 4, 768] + - [6, 5.76986] + - - [448, 1408, 1, 1792, 448, 448, 448, 1792] + - [3, 4500.0] + - - [3584, 1024, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 6443.32] + - - [1024, 64, 1, 1792, 1024, 1024, 1024, 1792] + - [3, 1241.51] + - - [3584, 6784, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 6672.95] + - - [2944, 3584, 1, 256, 2944, 2944, 2944, 256] + - [0, 6147.04] + - - [6784, 3584, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6664.89] + - - [6784, 3584, 1, 256, 6784, 6784, 6784, 256] + - [0, 6333.0] + - - [64, 704, 1, 1792, 64, 64, 64, 1792] + - [3, 1046.03] + - - [2944, 1408, 1, 768, 2944, 2944, 2944, 768] + - [0, 6311.6] + - - [704, 256, 1, 768, 704, 704, 704, 768] + - [3, 2951.85] + - - [704, 1408, 1, 3328, 704, 704, 704, 3328] + - [5, 5391.38] + - - [5056, 3584, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 6642.68] + - - [5888, 64, 1, 1792, 5888, 5888, 5888, 1792] + - [3, 3996.51] + - - [2368, 64, 1, 256, 2368, 2368, 2368, 256] + - [3, 1863.87] + - - [128, 1024, 1, 3328, 128, 128, 128, 3328] + - [3, 2993.71] + - - [3584, 5056, 1, 768, 3584, 3584, 3584, 768] + - [0, 6549.65] + - - [1408, 1856, 1, 256, 1408, 1408, 1408, 256] + - [0, 5726.78] + - - [3584, 1408, 1, 3328, 3584, 3584, 3584, 3328] + - [0, 6338.24] + - - [5056, 5888, 1, 768, 5056, 5056, 5056, 768] + - [0, 6555.17] + - - [1856, 128, 1, 1792, 1856, 1856, 1856, 1792] + - [3, 4128.65] + - - [128, 448, 1, 1792, 128, 128, 128, 1792] + - [3, 1369.43] + - - [704, 1024, 1, 1792, 704, 704, 704, 1792] + - [3, 4463.64] + - - [4, 704, 1, 256, 4, 4, 4, 256] + - [6, 43.8368] + - - [256, 3584, 1, 1792, 256, 256, 256, 1792] + - [0, 5012.56] + - - [256, 448, 1, 1792, 256, 256, 256, 1792] + - [3, 2683.22] + - - [704, 5888, 1, 768, 704, 704, 704, 768] + - [5, 5847.83] + - - [3584, 704, 1, 768, 3584, 3584, 3584, 768] + - [0, 5961.51] + - - [256, 5888, 1, 3328, 256, 256, 256, 3328] + - [0, 5873.82] + - - [5888, 3584, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 6600.88] + - - [2368, 5888, 1, 768, 2368, 2368, 2368, 768] + - [0, 6327.35] + - - [5056, 1856, 1, 1792, 5056, 5056, 5056, 1792] + - [0, 6281.05] + - - [2368, 2368, 1, 3328, 2368, 2368, 2368, 3328] + - [0, 6430.55] + - - [1024, 5056, 1, 256, 1024, 1024, 1024, 256] + - [0, 5852.92] + - - [3584, 1408, 1, 768, 3584, 3584, 3584, 768] + - [0, 6195.17] + - - [704, 64, 1, 1792, 704, 704, 704, 1792] + - [3, 1082.38] + - - [6784, 5056, 1, 3328, 6784, 6784, 6784, 3328] + - [0, 6709.14] + - - [1408, 6784, 1, 768, 1408, 1408, 1408, 768] + - [0, 6344.48] + - - [1408, 1856, 1, 1792, 1408, 1408, 1408, 1792] + - [0, 6310.43] + - - [1024, 4, 1, 3328, 1024, 1024, 1024, 3328] + - [6, 82.1518] + - - [64, 704, 1, 768, 64, 64, 64, 768] + - [3, 1035.77] + - - [64, 2368, 1, 768, 64, 64, 64, 768] + - [3, 2331.22] + - - [1024, 256, 1, 256, 1024, 1024, 1024, 256] + - [1, 2912.96] + - - [4288, 704, 1, 768, 4288, 4288, 4288, 768] + - [0, 5987.66] + - - [1024, 128, 1, 3328, 1024, 1024, 1024, 3328] + - [1, 2249.29] + - - [3584, 5056, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 6628.41] + - - [2944, 3584, 1, 3328, 2944, 2944, 2944, 3328] + - [0, 6432.91] + - - [256, 4288, 1, 3328, 256, 256, 256, 3328] + - [3, 4714.85] + - - [256, 6784, 1, 1792, 256, 256, 256, 1792] + - [0, 5137.97] + - - [256, 704, 1, 3328, 256, 256, 256, 3328] + - [3, 3298.12] + - - [256, 3584, 1, 768, 256, 256, 256, 768] + - [0, 4794.57] + - - [1024, 2368, 1, 768, 1024, 1024, 1024, 768] + - [0, 5629.18] + - - [1024, 2944, 1, 768, 1024, 1024, 1024, 768] + - [0, 5886.08] + - - [64, 6784, 1, 256, 64, 64, 64, 256] + - [5, 2174.6] + - - [4288, 128, 1, 256, 4288, 4288, 4288, 256] + - [3, 3020.4] + - - [1024, 4, 1, 256, 1024, 1024, 1024, 256] + - [6, 59.2265] + - - [5888, 4, 1, 3328, 5888, 5888, 5888, 3328] + - [6, 425.459] + - - [128, 2368, 1, 768, 128, 128, 128, 768] + - [3, 3045.06] + - - [256, 1408, 1, 256, 256, 256, 256, 256] + - [3, 3053.08] + - - [4, 3584, 1, 768, 4, 4, 4, 768] + - [6, 228.664] + - - [704, 64, 1, 768, 704, 704, 704, 768] + - [3, 1109.57] + - - [64, 128, 1, 3328, 64, 64, 64, 3328] + - [3, 219.724] + - - [1408, 64, 1, 768, 1408, 1408, 1408, 768] + - [3, 1979.35] + - - [6784, 448, 1, 256, 6784, 6784, 6784, 256] + - [0, 5653.01] + - - [4288, 448, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 5745.26] + - - [4288, 6784, 1, 3328, 4288, 4288, 4288, 3328] + - [0, 6603.52] + - - [1024, 2944, 1, 3328, 1024, 1024, 1024, 3328] + - [0, 6144.57] + - - [128, 5888, 1, 768, 128, 128, 128, 768] + - [3, 4212.27] + - - [1408, 5888, 1, 3328, 1408, 1408, 1408, 3328] + - [0, 6605.59] + - - [5888, 2944, 1, 768, 5888, 5888, 5888, 768] + - [0, 6472.31] + - - [3584, 128, 1, 256, 3584, 3584, 3584, 256] + - [0, 3282.43] + - - [128, 2944, 1, 256, 128, 128, 128, 256] + - [3, 2724.46] + - - [704, 5888, 1, 3328, 704, 704, 704, 3328] + - [5, 6027.77] + - - [1856, 128, 1, 256, 1856, 1856, 1856, 256] + - [3, 2870.64] + - - [5888, 5888, 1, 768, 5888, 5888, 5888, 768] + - [0, 6602.74] + - - [3584, 256, 1, 1792, 3584, 3584, 3584, 1792] + - [0, 5026.18] + - - [4, 64, 1, 256, 4, 4, 4, 256] + - [6, 4.7564] + - - [2368, 2944, 1, 1792, 2368, 2368, 2368, 1792] + - [0, 6294.68] + - - [2368, 6784, 1, 1792, 2368, 2368, 2368, 1792] + - [0, 6447.15] + - - [128, 1408, 1, 3328, 128, 128, 128, 3328] + - [3, 3270.14] + - - [448, 3584, 1, 768, 448, 448, 448, 768] + - [0, 4521.11] + - - [4288, 256, 1, 768, 4288, 4288, 4288, 768] + - [0, 4131.0] + - - [5888, 1408, 1, 1792, 5888, 5888, 5888, 1792] + - [0, 6545.26] + - - [1408, 4, 1, 768, 1408, 1408, 1408, 768] + - [6, 105.019] + - - [256, 1408, 1, 768, 256, 256, 256, 768] + - [3, 3718.4] + - - [2368, 1856, 1, 3328, 2368, 2368, 2368, 3328] + - [0, 6119.54] + - - [5056, 128, 1, 1792, 5056, 5056, 5056, 1792] + - [3, 4302.23] + - - [704, 4288, 1, 256, 704, 704, 704, 256] + - [5, 5213.65] + - - [1856, 4, 1, 3328, 1856, 1856, 1856, 3328] + - [6, 151.332] + - - [704, 1024, 1, 256, 704, 704, 704, 256] + - [3, 3595.0] + - - [64, 64, 1, 3328, 64, 64, 64, 3328] + - [3, 117.061] +- null diff --git a/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Alik_Bjlk_SB.yaml b/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Alik_Bjlk_SB.yaml new file mode 100644 index 0000000000..7cfdf4771e --- /dev/null +++ b/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Alik_Bjlk_SB.yaml @@ -0,0 +1,3081 @@ +- {MinimumRequiredVersion: 4.9.4} +- navi10 +- gfx1010 +- [NAVI] +- AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: true + UseBeta: true + UseInitialStrides: false +- - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 16 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 2 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: false + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 16 + LSCB: 64 + LSPA: 32 + LSPB: 16 + LVCA: 8 + LVCB: 16 + LVPA: 16 + LVPB: 4 + LdcEqualsLdd: true + LdsNumElements: 8192 + LdsNumElementsAlignedA: 512 + LdsNumElementsAlignedB: 1024 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 512 + LdsOffsetB_Blk: 2560 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 4 + MacroTile0: 32 + MacroTile1: 64 + MacroTileA: 32 + MacroTileB: 64 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 8 + NumGlobalWriteVectorsPerThread: 2 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 0 + SolutionNameMin: Cijk_Alik_Bjlk_SB_MT32x64x16_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: true + ThreadTile: [4, 8] + ThreadTile0: 4 + ThreadTile1: 8 + ThreadTileA: 4 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 2 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: false + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 8 + LSCB: 128 + LSPA: 128 + LSPB: 8 + LVCA: 2 + LVCB: 32 + LVPA: 32 + LVPB: 2 + LdcEqualsLdd: true + LdsNumElements: 4096 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 1024 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 3072 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 8 + MacroTile0: 128 + MacroTile1: 128 + MacroTileA: 128 + MacroTileB: 128 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 64 + NumGlobalWriteVectorsPerThread: 16 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 1 + SolutionNameMin: Cijk_Alik_Bjlk_SB_MT128x128x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: true + ThreadTile: [8, 8] + ThreadTile0: 8 + ThreadTile1: 8 + ThreadTileA: 8 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 3 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 16 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: false + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 16 + LSCB: 64 + LSPA: 64 + LSPB: 16 + LVCA: 4 + LVCB: 16 + LVPA: 16 + LVPB: 4 + LdcEqualsLdd: true + LdsNumElements: 16384 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 1024 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 3072 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 4 + MacroTile0: 64 + MacroTile1: 64 + MacroTileA: 64 + MacroTileB: 64 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 16 + NumGlobalWriteVectorsPerThread: 4 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 2 + SolutionNameMin: Cijk_Alik_Bjlk_SB_MT64x64x16_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: true + ThreadTile: [8, 8] + ThreadTile0: 8 + ThreadTile1: 8 + ThreadTileA: 8 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 4 + WorkGroupMappingType: B + _staggerStrideShift: 2 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 16 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: false + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 16 + LSCB: 128 + LSPA: 64 + LSPB: 8 + LVCA: 4 + LVCB: 32 + LVPA: 16 + LVPB: 2 + LdcEqualsLdd: true + LdsNumElements: 7168 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 2048 + LdsOffsetA: 0 + LdsOffsetA_Blk: 4096 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 5120 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 16 + MacroTile0: 64 + MacroTile1: 128 + MacroTileA: 64 + MacroTileB: 128 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 32 + NumGlobalWriteVectorsPerThread: 8 + NumLoadsA: 1 + NumLoadsB: 2 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 2 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 3 + SolutionNameMin: Cijk_Alik_Bjlk_SB_MT64x128x16_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: true + ThreadTile: [4, 8] + ThreadTile0: 4 + ThreadTile1: 8 + ThreadTileA: 4 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: false + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 4 + WorkGroupMappingType: B + _staggerStrideShift: 2 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 1 + GlobalLoadVectorWidthB: 1 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 8 + LSCB: 32 + LSPA: 32 + LSPB: 8 + LVCA: 8 + LVCB: 32 + LVPA: 32 + LVPB: 8 + LdcEqualsLdd: true + LdsNumElements: 4096 + LdsNumElementsAlignedA: 256 + LdsNumElementsAlignedB: 256 + LdsOffsetA: 0 + LdsOffsetA_Blk: 512 + LdsOffsetB: 256 + LdsOffsetB_Blk: 768 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 2 + MacroTile0: 32 + MacroTile1: 32 + MacroTileA: 32 + MacroTileB: 32 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 4 + NumGlobalWriteVectorsPerThread: 1 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 4 + SolutionNameMin: Cijk_Alik_Bjlk_SB_MT32x32x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: true + ThreadTile: [4, 4] + ThreadTile0: 4 + ThreadTile1: 4 + ThreadTileA: 4 + ThreadTileB: 4 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 4 + WorkGroupMappingType: B + _staggerStrideShift: 3 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 1 + GlobalLoadVectorWidthB: 1 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 2 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 1 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 8 + LSCB: 8 + LSPA: 8 + LSPB: 8 + LVCA: 8 + LVCB: 8 + LVPA: 8 + LVPB: 8 + LdcEqualsLdd: true + LdsNumElements: 409 + LdsNumElementsAlignedA: 64 + LdsNumElementsAlignedB: 64 + LdsOffsetA: 0 + LdsOffsetA_Blk: 128 + LdsOffsetB: 64 + LdsOffsetB_Blk: 192 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 2 + MacroTile0: 8 + MacroTile1: 8 + MacroTileA: 8 + MacroTileB: 8 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 1 + NumGlobalWriteVectorsPerThread: 1 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 64 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 1 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: true + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 5 + SolutionNameMin: Cijk_Alik_Bjlk_SB_MT8x8x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 4 + SubGroup1: 4 + SubGroupA: 4 + SubGroupB: 4 + SuppressNoLoadLoop: true + ThreadTile: [2, 2] + ThreadTile0: 2 + ThreadTile1: 2 + ThreadTileA: 2 + ThreadTileB: 2 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 2 + WorkGroup: [4, 4, 4] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 3 +- [2, 3, 0, 1] +- - - [256, 4288, 1, 3328, 256, 256, 3328, 4288] + - [0, 4653.47] + - - [1408, 2944, 1, 256, 1408, 1408, 256, 2944] + - [3, 5673.39] + - - [704, 4288, 1, 1792, 704, 704, 1792, 4288] + - [3, 5911.75] + - - [4288, 2944, 1, 1792, 4288, 4288, 1792, 2944] + - [3, 6191.49] + - - [128, 704, 1, 1792, 128, 128, 1792, 704] + - [0, 2137.19] + - - [4288, 1856, 1, 3328, 4288, 4288, 3328, 1856] + - [1, 6194.06] + - - [1856, 64, 1, 768, 1856, 1856, 768, 64] + - [0, 2487.93] + - - [3584, 5056, 1, 256, 3584, 3584, 256, 5056] + - [1, 5871.88] + - - [5056, 448, 1, 256, 5056, 5056, 256, 448] + - [3, 4430.07] + - - [2944, 64, 1, 1792, 2944, 2944, 1792, 64] + - [0, 3288.62] + - - [2368, 256, 1, 3328, 2368, 2368, 3328, 256] + - [2, 3581.67] + - - [704, 1856, 1, 768, 704, 704, 768, 1856] + - [0, 4746.58] + - - [1408, 704, 1, 1792, 1408, 1408, 1792, 704] + - [0, 4648.95] + - - [704, 704, 1, 768, 704, 704, 768, 704] + - [0, 4299.92] + - - [704, 4, 1, 256, 704, 704, 256, 4] + - [5, 46.565] + - - [3584, 6784, 1, 256, 3584, 3584, 256, 6784] + - [1, 6050.74] + - - [448, 2368, 1, 768, 448, 448, 768, 2368] + - [0, 4286.35] + - - [1408, 1856, 1, 1792, 1408, 1408, 1792, 1856] + - [1, 5169.53] + - - [1856, 704, 1, 3328, 1856, 1856, 3328, 704] + - [0, 5001.07] + - - [2368, 128, 1, 768, 2368, 2368, 768, 128] + - [0, 3148.79] + - - [256, 6784, 1, 256, 256, 256, 256, 6784] + - [1, 4432.71] + - - [5056, 2944, 1, 1792, 5056, 5056, 1792, 2944] + - [3, 6299.17] + - - [2944, 5056, 1, 1792, 2944, 2944, 1792, 5056] + - [3, 6278.3] + - - [2368, 1024, 1, 3328, 2368, 2368, 3328, 1024] + - [3, 5317.01] + - - [6784, 6784, 1, 256, 6784, 6784, 256, 6784] + - [1, 6312.02] + - - [256, 2368, 1, 1792, 256, 256, 1792, 2368] + - [0, 4288.02] + - - [1408, 3584, 1, 3328, 1408, 1408, 3328, 3584] + - [1, 6209.79] + - - [1856, 5056, 1, 1792, 1856, 1856, 1792, 5056] + - [3, 6028.54] + - - [1856, 128, 1, 3328, 1856, 1856, 3328, 128] + - [0, 4225.33] + - - [2944, 3584, 1, 768, 2944, 2944, 768, 3584] + - [3, 6067.29] + - - [1408, 2368, 1, 256, 1408, 1408, 256, 2368] + - [3, 5067.07] + - - [5888, 2368, 1, 768, 5888, 5888, 768, 2368] + - [1, 6105.04] + - - [1856, 2368, 1, 256, 1856, 1856, 256, 2368] + - [3, 5351.93] + - - [448, 64, 1, 3328, 448, 448, 3328, 64] + - [0, 710.896] + - - [2944, 1856, 1, 256, 2944, 2944, 256, 1856] + - [3, 5482.97] + - - [2368, 2944, 1, 768, 2368, 2368, 768, 2944] + - [1, 6045.3] + - - [6784, 128, 1, 3328, 6784, 6784, 3328, 128] + - [3, 4830.65] + - - [4288, 5888, 1, 3328, 4288, 4288, 3328, 5888] + - [1, 6405.38] + - - [64, 448, 1, 1792, 64, 64, 1792, 448] + - [0, 692.923] + - - [2944, 5056, 1, 256, 2944, 2944, 256, 5056] + - [3, 5922.95] + - - [2944, 448, 1, 768, 2944, 2944, 768, 448] + - [3, 4673.32] + - - [5056, 704, 1, 768, 5056, 5056, 768, 704] + - [1, 5229.31] + - - [1024, 1024, 1, 3328, 1024, 1024, 3328, 1024] + - [3, 5841.92] + - - [5888, 4, 1, 768, 5888, 5888, 768, 4] + - [5, 252.514] + - - [2368, 64, 1, 3328, 2368, 2368, 3328, 64] + - [0, 2759.94] + - - [2944, 4, 1, 1792, 2944, 2944, 1792, 4] + - [5, 224.668] + - - [4, 4, 1, 256, 4, 4, 256, 4] + - [5, 0.228489] + - - [2944, 6784, 1, 256, 2944, 2944, 256, 6784] + - [3, 6035.49] + - - [64, 704, 1, 1792, 64, 64, 1792, 704] + - [0, 1081.32] + - - [1024, 5056, 1, 768, 1024, 1024, 768, 5056] + - [1, 6094.85] + - - [1856, 2944, 1, 256, 1856, 1856, 256, 2944] + - [3, 5641.87] + - - [1408, 1408, 1, 3328, 1408, 1408, 3328, 1408] + - [1, 5820.92] + - - [64, 2368, 1, 3328, 64, 64, 3328, 2368] + - [0, 2763.3] + - - [1024, 2944, 1, 3328, 1024, 1024, 3328, 2944] + - [1, 6021.66] + - - [4288, 3584, 1, 256, 4288, 4288, 256, 3584] + - [3, 6002.62] + - - [448, 256, 1, 768, 448, 448, 768, 256] + - [0, 2817.67] + - - [1024, 448, 1, 256, 1024, 1024, 256, 448] + - [0, 3202.85] + - - [128, 2944, 1, 768, 128, 128, 768, 2944] + - [0, 3731.52] + - - [4288, 6784, 1, 768, 4288, 4288, 768, 6784] + - [3, 6336.84] + - - [256, 5056, 1, 768, 256, 256, 768, 5056] + - [0, 4628.96] + - - [4288, 64, 1, 1792, 4288, 4288, 1792, 64] + - [0, 3073.56] + - - [2944, 1408, 1, 3328, 2944, 2944, 3328, 1408] + - [1, 6317.12] + - - [1024, 6784, 1, 768, 1024, 1024, 768, 6784] + - [1, 5962.65] + - - [5056, 64, 1, 256, 5056, 5056, 256, 64] + - [0, 2631.27] + - - [4, 3584, 1, 256, 4, 4, 256, 3584] + - [5, 193.531] + - - [256, 448, 1, 768, 256, 256, 768, 448] + - [0, 2696.27] + - - [2368, 4, 1, 768, 2368, 2368, 768, 4] + - [5, 173.504] + - - [448, 1856, 1, 768, 448, 448, 768, 1856] + - [3, 4334.16] + - - [128, 128, 1, 1792, 128, 128, 1792, 128] + - [0, 460.342] + - - [3584, 4, 1, 3328, 3584, 3584, 3328, 4] + - [5, 179.408] + - - [6784, 1856, 1, 256, 6784, 6784, 256, 1856] + - [3, 5736.82] + - - [4288, 1024, 1, 256, 4288, 4288, 256, 1024] + - [3, 5425.31] + - - [2944, 128, 1, 3328, 2944, 2944, 3328, 128] + - [0, 4174.03] + - - [64, 704, 1, 3328, 64, 64, 3328, 704] + - [0, 1114.05] + - - [6784, 1856, 1, 3328, 6784, 6784, 3328, 1856] + - [3, 6188.71] + - - [5056, 2368, 1, 1792, 5056, 5056, 1792, 2368] + - [1, 6227.89] + - - [1024, 256, 1, 1792, 1024, 1024, 1792, 256] + - [0, 4484.9] + - - [128, 2368, 1, 1792, 128, 128, 1792, 2368] + - [0, 3343.6] + - - [256, 2944, 1, 256, 256, 256, 256, 2944] + - [0, 3642.81] + - - [3584, 5056, 1, 3328, 3584, 3584, 3328, 5056] + - [3, 6374.1] + - - [448, 5888, 1, 1792, 448, 448, 1792, 5888] + - [3, 5193.09] + - - [128, 1408, 1, 256, 128, 128, 256, 1408] + - [4, 2177.78] + - - [64, 5888, 1, 768, 64, 64, 768, 5888] + - [0, 3800.49] + - - [2368, 2368, 1, 1792, 2368, 2368, 1792, 2368] + - [3, 6129.75] + - - [256, 448, 1, 1792, 256, 256, 1792, 448] + - [0, 2610.04] + - - [1024, 6784, 1, 3328, 1024, 1024, 3328, 6784] + - [1, 6165.65] + - - [1024, 5056, 1, 1792, 1024, 1024, 1792, 5056] + - [1, 6278.69] + - - [5888, 64, 1, 768, 5888, 5888, 768, 64] + - [0, 3845.38] + - - [64, 6784, 1, 256, 64, 64, 256, 6784] + - [0, 2971.22] + - - [3584, 6784, 1, 768, 3584, 3584, 768, 6784] + - [1, 6304.36] + - - [704, 2944, 1, 256, 704, 704, 256, 2944] + - [3, 5365.27] + - - [2368, 5888, 1, 3328, 2368, 2368, 3328, 5888] + - [3, 6361.2] + - - [704, 64, 1, 3328, 704, 704, 3328, 64] + - [0, 1114.66] + - - [1408, 448, 1, 1792, 1408, 1408, 1792, 448] + - [0, 4521.35] + - - [256, 1408, 1, 3328, 256, 256, 3328, 1408] + - [0, 4093.48] + - - [128, 1856, 1, 256, 128, 128, 256, 1856] + - [0, 2783.09] + - - [704, 4, 1, 1792, 704, 704, 1792, 4] + - [5, 61.7617] + - - [4, 704, 1, 1792, 4, 4, 1792, 704] + - [5, 62.0996] + - - [1856, 128, 1, 768, 1856, 1856, 768, 128] + - [0, 3818.67] + - - [3584, 6784, 1, 3328, 3584, 3584, 3328, 6784] + - [1, 6421.6] + - - [256, 256, 1, 3328, 256, 256, 3328, 256] + - [0, 1665.34] + - - [1408, 256, 1, 3328, 1408, 1408, 3328, 256] + - [0, 3995.51] + - - [1024, 64, 1, 768, 1024, 1024, 768, 64] + - [0, 1415.54] + - - [64, 64, 1, 256, 64, 64, 256, 64] + - [0, 56.8488] + - - [256, 448, 1, 3328, 256, 256, 3328, 448] + - [0, 2734.91] + - - [2944, 6784, 1, 1792, 2944, 2944, 1792, 6784] + - [1, 6313.33] + - - [4, 1856, 1, 768, 4, 4, 768, 1856] + - [5, 148.879] + - - [256, 1856, 1, 256, 256, 256, 256, 1856] + - [0, 3344.23] + - - [2368, 5056, 1, 1792, 2368, 2368, 1792, 5056] + - [1, 6227.76] + - - [128, 4, 1, 1792, 128, 128, 1792, 4] + - [5, 11.9556] + - - [64, 1856, 1, 256, 64, 64, 256, 1856] + - [0, 1732.1] + - - [5888, 128, 1, 256, 5888, 5888, 256, 128] + - [3, 3622.53] + - - [128, 4, 1, 3328, 128, 128, 3328, 4] + - [5, 12.3803] + - - [448, 1408, 1, 3328, 448, 448, 3328, 1408] + - [0, 4643.48] + - - [448, 4, 1, 1792, 448, 448, 1792, 4] + - [5, 44.7838] + - - [4, 5888, 1, 256, 4, 4, 256, 5888] + - [5, 273.135] + - - [448, 704, 1, 768, 448, 448, 768, 704] + - [0, 3392.02] + - - [5888, 64, 1, 256, 5888, 5888, 256, 64] + - [0, 2980.11] + - - [1856, 4288, 1, 768, 1856, 1856, 768, 4288] + - [1, 6034.66] + - - [448, 256, 1, 3328, 448, 448, 3328, 256] + - [0, 2902.85] + - - [128, 4288, 1, 256, 128, 128, 256, 4288] + - [0, 3069.29] + - - [2368, 704, 1, 256, 2368, 2368, 256, 704] + - [1, 4118.11] + - - [256, 128, 1, 768, 256, 256, 768, 128] + - [0, 820.602] + - - [4288, 1024, 1, 1792, 4288, 4288, 1792, 1024] + - [3, 5788.05] + - - [5056, 4, 1, 3328, 5056, 5056, 3328, 4] + - [5, 167.639] + - - [2944, 2944, 1, 3328, 2944, 2944, 3328, 2944] + - [3, 6240.19] + - - [1856, 64, 1, 1792, 1856, 1856, 1792, 64] + - [0, 2771.03] + - - [704, 2944, 1, 3328, 704, 704, 3328, 2944] + - [3, 6031.51] + - - [4288, 5056, 1, 256, 4288, 4288, 256, 5056] + - [3, 5992.06] + - - [2944, 256, 1, 768, 2944, 2944, 768, 256] + - [2, 4038.13] + - - [4, 1024, 1, 768, 4, 4, 768, 1024] + - [5, 79.3764] + - - [1408, 704, 1, 768, 1408, 1408, 768, 704] + - [0, 4410.66] + - - [1856, 1024, 1, 1792, 1856, 1856, 1792, 1024] + - [1, 5487.8] + - - [5056, 4288, 1, 256, 5056, 5056, 256, 4288] + - [3, 6029.18] + - - [5056, 64, 1, 3328, 5056, 5056, 3328, 64] + - [0, 3674.83] + - - [704, 6784, 1, 768, 704, 704, 768, 6784] + - [1, 5666.06] + - - [2944, 1024, 1, 768, 2944, 2944, 768, 1024] + - [1, 5804.12] + - - [1408, 64, 1, 3328, 1408, 1408, 3328, 64] + - [0, 2206.25] + - - [5056, 4, 1, 1792, 5056, 5056, 1792, 4] + - [5, 164.012] + - - [256, 5056, 1, 3328, 256, 256, 3328, 5056] + - [0, 4967.16] + - - [4288, 1024, 1, 768, 4288, 4288, 768, 1024] + - [3, 5734.21] + - - [5056, 4288, 1, 1792, 5056, 5056, 1792, 4288] + - [3, 6342.26] + - - [4288, 2368, 1, 768, 4288, 4288, 768, 2368] + - [3, 6075.72] + - - [2944, 5056, 1, 3328, 2944, 2944, 3328, 5056] + - [3, 6316.95] + - - [4, 5888, 1, 1792, 4, 4, 1792, 5888] + - [5, 367.112] + - - [704, 1408, 1, 256, 704, 704, 256, 1408] + - [3, 4724.99] + - - [6784, 6784, 1, 1792, 6784, 6784, 1792, 6784] + - [1, 6628.56] + - - [1408, 2368, 1, 3328, 1408, 1408, 3328, 2368] + - [3, 5688.19] + - - [1856, 704, 1, 1792, 1856, 1856, 1792, 704] + - [0, 4915.11] + - - [1024, 256, 1, 768, 1024, 1024, 768, 256] + - [0, 4111.68] + - - [4, 1408, 1, 3328, 4, 4, 3328, 1408] + - [5, 118.515] + - - [5888, 1856, 1, 1792, 5888, 5888, 1792, 1856] + - [1, 6184.31] + - - [128, 1024, 1, 3328, 128, 128, 3328, 1024] + - [0, 2993.72] + - - [256, 5888, 1, 1792, 256, 256, 1792, 5888] + - [3, 5562.83] + - - [4, 6784, 1, 1792, 4, 4, 1792, 6784] + - [5, 369.42] + - - [2368, 3584, 1, 256, 2368, 2368, 256, 3584] + - [3, 5760.02] + - - [5888, 5056, 1, 768, 5888, 5888, 768, 5056] + - [1, 6396.79] + - - [2368, 4288, 1, 768, 2368, 2368, 768, 4288] + - [3, 6092.94] + - - [3584, 3584, 1, 768, 3584, 3584, 768, 3584] + - [3, 6178.35] + - - [4, 1408, 1, 768, 4, 4, 768, 1408] + - [5, 112.724] + - - [4288, 128, 1, 256, 4288, 4288, 256, 128] + - [0, 3084.28] + - - [64, 5056, 1, 256, 64, 64, 256, 5056] + - [0, 2570.76] + - - [1024, 128, 1, 1792, 1024, 1024, 1792, 128] + - [0, 3014.08] + - - [4, 5888, 1, 768, 4, 4, 768, 5888] + - [5, 345.374] + - - [1408, 1024, 1, 3328, 1408, 1408, 3328, 1024] + - [3, 5459.81] + - - [64, 1024, 1, 256, 64, 64, 256, 1024] + - [0, 980.464] + - - [3584, 2368, 1, 768, 3584, 3584, 768, 2368] + - [3, 5935.65] + - - [5056, 5056, 1, 3328, 5056, 5056, 3328, 5056] + - [1, 6474.64] + - - [6784, 4, 1, 768, 6784, 6784, 768, 4] + - [5, 313.296] + - - [1856, 1856, 1, 768, 1856, 1856, 768, 1856] + - [3, 5697.86] + - - [704, 128, 1, 1792, 704, 704, 1792, 128] + - [0, 2171.24] + - - [2944, 2368, 1, 1792, 2944, 2944, 1792, 2368] + - [1, 6127.97] + - - [2944, 1024, 1, 3328, 2944, 2944, 3328, 1024] + - [1, 6024.07] + - - [5888, 4288, 1, 1792, 5888, 5888, 1792, 4288] + - [1, 6372.56] + - - [3584, 64, 1, 1792, 3584, 3584, 1792, 64] + - [0, 4003.54] + - - [1408, 6784, 1, 768, 1408, 1408, 768, 6784] + - [3, 6057.33] + - - [2368, 448, 1, 768, 2368, 2368, 768, 448] + - [0, 4196.4] + - - [4288, 256, 1, 1792, 4288, 4288, 1792, 256] + - [3, 4191.22] + - - [256, 3584, 1, 768, 256, 256, 768, 3584] + - [3, 4794.58] + - - [2944, 64, 1, 3328, 2944, 2944, 3328, 64] + - [0, 3405.05] + - - [5056, 5888, 1, 1792, 5056, 5056, 1792, 5888] + - [1, 6482.78] + - - [704, 5888, 1, 1792, 704, 704, 1792, 5888] + - [3, 6175.08] + - - [1024, 704, 1, 768, 1024, 1024, 768, 704] + - [0, 4145.34] + - - [6784, 64, 1, 1792, 6784, 6784, 1792, 64] + - [0, 3947.13] + - - [1856, 1024, 1, 256, 1856, 1856, 256, 1024] + - [3, 4890.63] + - - [3584, 128, 1, 1792, 3584, 3584, 1792, 128] + - [3, 4161.17] + - - [5888, 2944, 1, 3328, 5888, 5888, 3328, 2944] + - [1, 6438.13] + - - [704, 5056, 1, 1792, 704, 704, 1792, 5056] + - [3, 6013.09] + - - [5888, 704, 1, 256, 5888, 5888, 256, 704] + - [3, 4884.18] + - - [4288, 448, 1, 1792, 4288, 4288, 1792, 448] + - [0, 4881.03] + - - [4, 1024, 1, 256, 4, 4, 256, 1024] + - [5, 55.7309] + - - [4, 64, 1, 1792, 4, 4, 1792, 64] + - [5, 6.16834] + - - [5888, 3584, 1, 768, 5888, 5888, 768, 3584] + - [1, 6250.86] + - - [1408, 4, 1, 3328, 1408, 1408, 3328, 4] + - [5, 117.034] + - - [6784, 704, 1, 256, 6784, 6784, 256, 704] + - [1, 5137.02] + - - [1024, 5888, 1, 3328, 1024, 1024, 3328, 5888] + - [1, 6214.94] + - - [1408, 2944, 1, 768, 1408, 1408, 768, 2944] + - [1, 6119.31] + - - [1408, 1024, 1, 768, 1408, 1408, 768, 1024] + - [3, 5215.68] + - - [1856, 3584, 1, 768, 1856, 1856, 768, 3584] + - [3, 6110.69] + - - [704, 1408, 1, 768, 704, 704, 768, 1408] + - [3, 5272.89] + - - [256, 256, 1, 1792, 256, 256, 1792, 256] + - [0, 1594.98] + - - [4288, 4288, 1, 3328, 4288, 4288, 3328, 4288] + - [3, 6303.51] + - - [4, 704, 1, 3328, 4, 4, 3328, 704] + - [5, 58.5443] + - - [2944, 2368, 1, 256, 2944, 2944, 256, 2368] + - [3, 5544.88] + - - [704, 4288, 1, 768, 704, 704, 768, 4288] + - [3, 5756.42] + - - [4288, 2368, 1, 3328, 4288, 4288, 3328, 2368] + - [3, 6222.03] + - - [2944, 704, 1, 1792, 2944, 2944, 1792, 704] + - [0, 4941.22] + - - [64, 1408, 1, 3328, 64, 64, 3328, 1408] + - [0, 2189.53] + - - [1024, 128, 1, 3328, 1024, 1024, 3328, 128] + - [0, 3159.15] + - - [4, 128, 1, 256, 4, 4, 256, 128] + - [5, 10.3475] + - - [5888, 448, 1, 1792, 5888, 5888, 1792, 448] + - [1, 5190.13] + - - [6784, 4288, 1, 256, 6784, 6784, 256, 4288] + - [1, 6025.28] + - - [3584, 3584, 1, 1792, 3584, 3584, 1792, 3584] + - [3, 6276.56] + - - [64, 3584, 1, 1792, 64, 64, 1792, 3584] + - [2, 3696.82] + - - [704, 1024, 1, 1792, 704, 704, 1792, 1024] + - [3, 3961.71] + - - [1408, 6784, 1, 256, 1408, 1408, 256, 6784] + - [3, 5817.84] + - - [64, 4, 1, 3328, 64, 64, 3328, 4] + - [5, 6.18018] + - - [1024, 2944, 1, 256, 1024, 1024, 256, 2944] + - [1, 5243.27] + - - [5888, 1856, 1, 768, 5888, 5888, 768, 1856] + - [1, 6063.69] + - - [704, 6784, 1, 3328, 704, 704, 3328, 6784] + - [1, 5861.39] + - - [1024, 6784, 1, 1792, 1024, 1024, 1792, 6784] + - [1, 6120.82] + - - [128, 6784, 1, 1792, 128, 128, 1792, 6784] + - [3, 4789.48] + - - [2368, 256, 1, 768, 2368, 2368, 768, 256] + - [0, 3357.38] + - - [2944, 448, 1, 3328, 2944, 2944, 3328, 448] + - [3, 4980.22] + - - [4, 256, 1, 1792, 4, 4, 1792, 256] + - [5, 29.2122] + - - [128, 64, 1, 1792, 128, 128, 1792, 64] + - [0, 231.244] + - - [2368, 2368, 1, 768, 2368, 2368, 768, 2368] + - [3, 6025.11] + - - [1024, 128, 1, 768, 1024, 1024, 768, 128] + - [0, 2819.31] + - - [704, 704, 1, 256, 704, 704, 256, 704] + - [0, 3502.68] + - - [4, 1856, 1, 3328, 4, 4, 3328, 1856] + - [5, 154.702] + - - [6784, 5056, 1, 1792, 6784, 6784, 1792, 5056] + - [1, 6392.33] + - - [64, 3584, 1, 256, 64, 64, 256, 3584] + - [2, 2469.47] + - - [1024, 3584, 1, 768, 1024, 1024, 768, 3584] + - [3, 5924.91] + - - [3584, 4288, 1, 256, 3584, 3584, 256, 4288] + - [1, 5898.62] + - - [128, 3584, 1, 3328, 128, 128, 3328, 3584] + - [3, 4272.45] + - - [64, 4, 1, 256, 64, 64, 256, 4] + - [5, 4.05833] + - - [5888, 6784, 1, 1792, 5888, 5888, 1792, 6784] + - [1, 6502.91] + - - [2368, 1856, 1, 768, 2368, 2368, 768, 1856] + - [3, 5739.55] + - - [4, 704, 1, 768, 4, 4, 768, 704] + - [5, 58.1587] + - - [1024, 1856, 1, 3328, 1024, 1024, 3328, 1856] + - [1, 5548.48] + - - [64, 256, 1, 1792, 64, 64, 1792, 256] + - [0, 452.457] + - - [4288, 1856, 1, 768, 4288, 4288, 768, 1856] + - [1, 6004.37] + - - [5056, 1856, 1, 1792, 5056, 5056, 1792, 1856] + - [3, 6009.55] + - - [3584, 4288, 1, 3328, 3584, 3584, 3328, 4288] + - [1, 6450.23] + - - [6784, 2944, 1, 1792, 6784, 6784, 1792, 2944] + - [1, 6309.21] + - - [6784, 64, 1, 768, 6784, 6784, 768, 64] + - [0, 3720.18] + - - [1856, 128, 1, 256, 1856, 1856, 256, 128] + - [0, 2901.11] + - - [704, 3584, 1, 256, 704, 704, 256, 3584] + - [3, 5362.69] + - - [4, 4288, 1, 768, 4, 4, 768, 4288] + - [5, 292.477] + - - [448, 64, 1, 256, 448, 448, 256, 64] + - [4, 458.738] + - - [704, 128, 1, 3328, 704, 704, 3328, 128] + - [0, 2252.89] + - - [5888, 5888, 1, 3328, 5888, 5888, 3328, 5888] + - [1, 6487.04] + - - [128, 6784, 1, 256, 128, 128, 256, 6784] + - [3, 3907.51] + - - [1408, 3584, 1, 1792, 1408, 1408, 1792, 3584] + - [1, 6157.7] + - - [5888, 1408, 1, 1792, 5888, 5888, 1792, 1408] + - [1, 6405.23] + - - [64, 6784, 1, 3328, 64, 64, 3328, 6784] + - [3, 4036.04] + - - [1024, 448, 1, 3328, 1024, 1024, 3328, 448] + - [3, 4231.23] + - - [5056, 3584, 1, 1792, 5056, 5056, 1792, 3584] + - [3, 6365.39] + - - [256, 1408, 1, 1792, 256, 256, 1792, 1408] + - [0, 3977.99] + - - [256, 704, 1, 3328, 256, 256, 3328, 704] + - [0, 3227.14] + - - [448, 6784, 1, 3328, 448, 448, 3328, 6784] + - [3, 6021.49] + - - [1856, 3584, 1, 1792, 1856, 1856, 1792, 3584] + - [3, 6189.43] + - - [1856, 1856, 1, 3328, 1856, 1856, 3328, 1856] + - [3, 5880.88] + - - [4288, 3584, 1, 768, 4288, 4288, 768, 3584] + - [1, 6325.02] + - - [2944, 128, 1, 1792, 2944, 2944, 1792, 128] + - [0, 4075.14] + - - [5056, 1408, 1, 768, 5056, 5056, 768, 1408] + - [1, 6135.2] + - - [256, 1024, 1, 1792, 256, 256, 1792, 1024] + - [0, 4370.53] + - - [4288, 128, 1, 768, 4288, 4288, 768, 128] + - [0, 3590.22] + - - [5056, 5888, 1, 256, 5056, 5056, 256, 5888] + - [3, 6173.89] + - - [1024, 5056, 1, 3328, 1024, 1024, 3328, 5056] + - [1, 6349.17] + - - [1408, 5888, 1, 1792, 1408, 1408, 1792, 5888] + - [1, 6422.46] + - - [1024, 2368, 1, 768, 1024, 1024, 768, 2368] + - [3, 5469.99] + - - [704, 2368, 1, 1792, 704, 704, 1792, 2368] + - [0, 4904.39] + - - [4288, 5056, 1, 3328, 4288, 4288, 3328, 5056] + - [3, 6374.94] + - - [128, 2368, 1, 256, 128, 128, 256, 2368] + - [0, 2735.0] + - - [128, 448, 1, 256, 128, 128, 256, 448] + - [0, 850.525] + - - [5888, 128, 1, 3328, 5888, 5888, 3328, 128] + - [0, 4643.41] + - - [3584, 128, 1, 768, 3584, 3584, 768, 128] + - [3, 3914.6] + - - [128, 6784, 1, 768, 128, 128, 768, 6784] + - [3, 4597.99] + - - [64, 5888, 1, 1792, 64, 64, 1792, 5888] + - [0, 4102.66] + - - [4, 6784, 1, 256, 4, 4, 256, 6784] + - [5, 277.456] + - - [448, 448, 1, 768, 448, 448, 768, 448] + - [0, 3266.66] + - - [64, 128, 1, 768, 64, 64, 768, 128] + - [0, 207.659] + - - [2368, 5888, 1, 1792, 2368, 2368, 1792, 5888] + - [3, 6329.2] + - - [4, 4288, 1, 3328, 4, 4, 3328, 4288] + - [5, 322.691] + - - [64, 1856, 1, 1792, 64, 64, 1792, 1856] + - [0, 2680.55] + - - [6784, 5056, 1, 256, 6784, 6784, 256, 5056] + - [1, 6109.55] + - - [704, 3584, 1, 768, 704, 704, 768, 3584] + - [3, 5750.55] + - - [128, 1856, 1, 1792, 128, 128, 1792, 1856] + - [0, 4108.0] + - - [1024, 4, 1, 256, 1024, 1024, 256, 4] + - [5, 66.1479] + - - [6784, 2368, 1, 3328, 6784, 6784, 3328, 2368] + - [1, 6328.51] + - - [1024, 1408, 1, 256, 1024, 1024, 256, 1408] + - [3, 4554.58] + - - [4288, 1408, 1, 768, 4288, 4288, 768, 1408] + - [1, 6037.94] + - - [2944, 3584, 1, 1792, 2944, 2944, 1792, 3584] + - [3, 6143.07] + - - [1024, 256, 1, 3328, 1024, 1024, 3328, 256] + - [0, 4647.71] + - - [2368, 5056, 1, 768, 2368, 2368, 768, 5056] + - [1, 6118.34] + - - [5888, 4, 1, 3328, 5888, 5888, 3328, 4] + - [5, 168.196] + - - [3584, 2368, 1, 1792, 3584, 3584, 1792, 2368] + - [3, 6051.53] + - - [2944, 448, 1, 1792, 2944, 2944, 1792, 448] + - [3, 4903.38] + - - [5056, 3584, 1, 768, 5056, 5056, 768, 3584] + - [3, 6290.25] + - - [6784, 4288, 1, 1792, 6784, 6784, 1792, 4288] + - [1, 6348.48] + - - [2368, 64, 1, 768, 2368, 2368, 768, 64] + - [0, 2478.32] + - - [2944, 256, 1, 256, 2944, 2944, 256, 256] + - [3, 3668.48] + - - [4, 4, 1, 768, 4, 4, 768, 4] + - [5, 0.320861] + - - [5056, 704, 1, 256, 5056, 5056, 256, 704] + - [1, 4766.01] + - - [448, 704, 1, 3328, 448, 448, 3328, 704] + - [0, 3635.09] + - - [128, 2944, 1, 1792, 128, 128, 1792, 2944] + - [0, 4071.5] + - - [1408, 5056, 1, 1792, 1408, 1408, 1792, 5056] + - [1, 6260.77] + - - [64, 5888, 1, 3328, 64, 64, 3328, 5888] + - [0, 4045.37] + - - [1024, 704, 1, 256, 1024, 1024, 256, 704] + - [0, 3455.4] + - - [6784, 2944, 1, 768, 6784, 6784, 768, 2944] + - [3, 6235.98] + - - [4288, 704, 1, 256, 4288, 4288, 256, 704] + - [3, 4528.01] + - - [2368, 4288, 1, 3328, 2368, 2368, 3328, 4288] + - [3, 6222.63] + - - [3584, 1024, 1, 3328, 3584, 3584, 3328, 1024] + - [3, 6068.77] + - - [5056, 704, 1, 3328, 5056, 5056, 3328, 704] + - [1, 5414.02] + - - [1856, 4, 1, 1792, 1856, 1856, 1792, 4] + - [5, 141.193] + - - [128, 256, 1, 256, 128, 128, 256, 256] + - [0, 592.897] + - - [4, 128, 1, 1792, 4, 4, 1792, 128] + - [5, 14.6407] + - - [256, 1024, 1, 256, 256, 256, 256, 1024] + - [0, 2980.1] + - - [5888, 704, 1, 768, 5888, 5888, 768, 704] + - [3, 5338.62] + - - [256, 2368, 1, 256, 256, 256, 256, 2368] + - [0, 3503.34] + - - [4, 5888, 1, 3328, 4, 4, 3328, 5888] + - [5, 375.087] + - - [5056, 1408, 1, 3328, 5056, 5056, 3328, 1408] + - [1, 6315.46] + - - [64, 2368, 1, 768, 64, 64, 768, 2368] + - [0, 2474.4] + - - [1024, 1024, 1, 256, 1024, 1024, 256, 1024] + - [3, 4718.5] + - - [448, 2944, 1, 768, 448, 448, 768, 2944] + - [3, 4756.23] + - - [1024, 704, 1, 1792, 1024, 1024, 1792, 704] + - [0, 4387.3] + - - [4, 5056, 1, 1792, 4, 4, 1792, 5056] + - [5, 351.471] + - - [4, 2368, 1, 1792, 4, 4, 1792, 2368] + - [5, 187.514] + - - [448, 1408, 1, 768, 448, 448, 768, 1408] + - [0, 4342.49] + - - [64, 64, 1, 768, 64, 64, 768, 64] + - [0, 95.8596] + - - [3584, 256, 1, 3328, 3584, 3584, 3328, 256] + - [3, 5094.0] + - - [448, 128, 1, 3328, 448, 448, 3328, 128] + - [0, 1432.07] + - - [2944, 4288, 1, 768, 2944, 2944, 768, 4288] + - [3, 6087.21] + - - [1408, 448, 1, 256, 1408, 1408, 256, 448] + - [0, 3657.63] + - - [4, 3584, 1, 768, 4, 4, 768, 3584] + - [5, 254.506] + - - [6784, 128, 1, 1792, 6784, 6784, 1792, 128] + - [3, 4768.82] + - - [1856, 2944, 1, 768, 1856, 1856, 768, 2944] + - [3, 5913.98] + - - [6784, 3584, 1, 1792, 6784, 6784, 1792, 3584] + - [1, 6395.35] + - - [5888, 3584, 1, 3328, 5888, 5888, 3328, 3584] + - [3, 6361.89] + - - [128, 1024, 1, 768, 128, 128, 768, 1024] + - [0, 2493.42] + - - [256, 6784, 1, 1792, 256, 256, 1792, 6784] + - [1, 5051.5] + - - [1024, 704, 1, 3328, 1024, 1024, 3328, 704] + - [0, 4503.91] + - - [5056, 256, 1, 1792, 5056, 5056, 1792, 256] + - [3, 4880.27] + - - [2368, 704, 1, 1792, 2368, 2368, 1792, 704] + - [0, 4860.65] + - - [5888, 1408, 1, 768, 5888, 5888, 768, 1408] + - [1, 6296.83] + - - [256, 1408, 1, 768, 256, 256, 768, 1408] + - [0, 3759.59] + - - [64, 2944, 1, 3328, 64, 64, 3328, 2944] + - [0, 3416.04] + - - [3584, 704, 1, 1792, 3584, 3584, 1792, 704] + - [1, 4983.93] + - - [5056, 256, 1, 3328, 5056, 5056, 3328, 256] + - [3, 4920.92] + - - [6784, 1024, 1, 768, 6784, 6784, 768, 1024] + - [1, 5942.97] + - - [1024, 4, 1, 1792, 1024, 1024, 1792, 4] + - [5, 83.2675] + - - [448, 448, 1, 256, 448, 448, 256, 448] + - [0, 2416.81] + - - [4288, 1408, 1, 1792, 4288, 4288, 1792, 1408] + - [1, 6167.11] + - - [6784, 4, 1, 256, 6784, 6784, 256, 4] + - [5, 313.647] + - - [1024, 4288, 1, 256, 1024, 1024, 256, 4288] + - [3, 5171.98] + - - [448, 1024, 1, 256, 448, 448, 256, 1024] + - [3, 3183.58] + - - [128, 2944, 1, 256, 128, 128, 256, 2944] + - [0, 3207.61] + - - [5888, 4288, 1, 256, 5888, 5888, 256, 4288] + - [1, 6000.97] + - - [4288, 256, 1, 256, 4288, 4288, 256, 256] + - [3, 3676.01] + - - [128, 64, 1, 256, 128, 128, 256, 64] + - [0, 130.465] + - - [256, 1856, 1, 3328, 256, 256, 3328, 1856] + - [3, 4532.14] + - - [2368, 2368, 1, 3328, 2368, 2368, 3328, 2368] + - [3, 6172.04] + - - [6784, 1408, 1, 3328, 6784, 6784, 3328, 1408] + - [3, 6166.99] + - - [1856, 6784, 1, 1792, 1856, 1856, 1792, 6784] + - [3, 6177.55] + - - [1408, 256, 1, 1792, 1408, 1408, 1792, 256] + - [0, 3920.76] + - - [256, 2944, 1, 3328, 256, 256, 3328, 2944] + - [0, 4729.02] + - - [704, 5888, 1, 768, 704, 704, 768, 5888] + - [3, 6037.64] + - - [2368, 1408, 1, 3328, 2368, 2368, 3328, 1408] + - [3, 5704.43] + - - [5888, 4, 1, 1792, 5888, 5888, 1792, 4] + - [5, 168.817] + - - [448, 128, 1, 1792, 448, 448, 1792, 128] + - [0, 1394.2] + - - [3584, 448, 1, 3328, 3584, 3584, 3328, 448] + - [0, 4777.86] + - - [5888, 448, 1, 256, 5888, 5888, 256, 448] + - [1, 4492.9] + - - [704, 448, 1, 768, 704, 704, 768, 448] + - [0, 3378.0] + - - [3584, 1024, 1, 768, 3584, 3584, 768, 1024] + - [3, 5999.65] + - - [5056, 128, 1, 768, 5056, 5056, 768, 128] + - [0, 4206.69] + - - [2368, 448, 1, 256, 2368, 2368, 256, 448] + - [0, 3597.91] + - - [3584, 1024, 1, 256, 3584, 3584, 256, 1024] + - [3, 5367.45] + - - [448, 5056, 1, 768, 448, 448, 768, 5056] + - [3, 5096.33] + - - [2368, 1024, 1, 1792, 2368, 2368, 1792, 1024] + - [3, 5645.28] + - - [256, 3584, 1, 256, 256, 256, 256, 3584] + - [3, 4059.96] + - - [4288, 6784, 1, 3328, 4288, 4288, 3328, 6784] + - [3, 6433.22] + - - [1856, 1856, 1, 1792, 1856, 1856, 1792, 1856] + - [3, 5826.26] + - - [1024, 4, 1, 768, 1024, 1024, 768, 4] + - [5, 86.8434] + - - [4, 64, 1, 3328, 4, 4, 3328, 64] + - [5, 6.41098] + - - [3584, 1856, 1, 1792, 3584, 3584, 1792, 1856] + - [1, 5841.4] + - - [128, 1856, 1, 3328, 128, 128, 3328, 1856] + - [0, 4279.54] + - - [64, 2368, 1, 1792, 64, 64, 1792, 2368] + - [0, 2703.73] + - - [1024, 6784, 1, 256, 1024, 1024, 256, 6784] + - [1, 5491.22] + - - [1856, 6784, 1, 768, 1856, 1856, 768, 6784] + - [3, 6103.77] + - - [6784, 4288, 1, 768, 6784, 6784, 768, 4288] + - [1, 6270.05] + - - [1024, 3584, 1, 256, 1024, 1024, 256, 3584] + - [3, 5349.33] + - - [448, 3584, 1, 768, 448, 448, 768, 3584] + - [3, 4501.57] + - - [4288, 4288, 1, 1792, 4288, 4288, 1792, 4288] + - [3, 6268.14] + - - [2944, 64, 1, 768, 2944, 2944, 768, 64] + - [0, 3038.01] + - - [3584, 3584, 1, 3328, 3584, 3584, 3328, 3584] + - [3, 6312.22] + - - [4288, 5888, 1, 768, 4288, 4288, 768, 5888] + - [1, 6306.76] + - - [2944, 6784, 1, 768, 2944, 2944, 768, 6784] + - [1, 6245.19] + - - [3584, 1408, 1, 1792, 3584, 3584, 1792, 1408] + - [1, 6155.84] + - - [704, 64, 1, 768, 704, 704, 768, 64] + - [0, 1076.35] + - - [4, 64, 1, 256, 4, 4, 256, 64] + - [5, 5.11401] + - - [64, 1024, 1, 1792, 64, 64, 1792, 1024] + - [0, 1246.39] + - - [1024, 256, 1, 256, 1024, 1024, 256, 256] + - [0, 3113.23] + - - [5056, 3584, 1, 256, 5056, 5056, 256, 3584] + - [3, 6081.14] + - - [128, 5888, 1, 3328, 128, 128, 3328, 5888] + - [0, 4649.15] + - - [3584, 448, 1, 256, 3584, 3584, 256, 448] + - [1, 3997.78] + - - [4, 3584, 1, 1792, 4, 4, 1792, 3584] + - [5, 274.158] + - - [4288, 2944, 1, 3328, 4288, 4288, 3328, 2944] + - [3, 6221.49] + - - [2368, 1408, 1, 768, 2368, 2368, 768, 1408] + - [3, 5573.61] + - - [4288, 64, 1, 3328, 4288, 4288, 3328, 64] + - [0, 3172.77] + - - [4288, 6784, 1, 256, 4288, 4288, 256, 6784] + - [3, 6147.23] + - - [64, 4288, 1, 768, 64, 64, 768, 4288] + - [0, 2797.69] + - - [704, 5888, 1, 256, 704, 704, 256, 5888] + - [3, 5646.55] + - - [1024, 2368, 1, 1792, 1024, 1024, 1792, 2368] + - [3, 5614.48] + - - [4288, 128, 1, 3328, 4288, 4288, 3328, 128] + - [0, 3930.94] + - - [4288, 5888, 1, 256, 4288, 4288, 256, 5888] + - [3, 6095.29] + - - [1024, 2368, 1, 256, 1024, 1024, 256, 2368] + - [3, 4955.67] + - - [4288, 1856, 1, 256, 4288, 4288, 256, 1856] + - [3, 5618.79] + - - [1856, 448, 1, 3328, 1856, 1856, 3328, 448] + - [3, 4598.82] + - - [2944, 4, 1, 256, 2944, 2944, 256, 4] + - [5, 164.767] + - - [1024, 1408, 1, 3328, 1024, 1024, 3328, 1408] + - [3, 5452.0] + - - [5056, 5056, 1, 1792, 5056, 5056, 1792, 5056] + - [1, 6446.18] + - - [448, 2368, 1, 1792, 448, 448, 1792, 2368] + - [0, 4453.2] + - - [128, 128, 1, 3328, 128, 128, 3328, 128] + - [0, 408.028] + - - [256, 128, 1, 3328, 256, 256, 3328, 128] + - [0, 813.799] + - - [448, 448, 1, 3328, 448, 448, 3328, 448] + - [0, 3588.14] + - - [128, 5056, 1, 256, 128, 128, 256, 5056] + - [0, 3550.08] + - - [64, 2944, 1, 768, 64, 64, 768, 2944] + - [0, 3019.25] + - - [5056, 5888, 1, 3328, 5056, 5056, 3328, 5888] + - [1, 6512.1] + - - [5056, 128, 1, 3328, 5056, 5056, 3328, 128] + - [0, 4565.12] + - - [1856, 448, 1, 1792, 1856, 1856, 1792, 448] + - [3, 4570.52] + - - [64, 4, 1, 768, 64, 64, 768, 4] + - [5, 5.41657] + - - [5056, 6784, 1, 1792, 5056, 5056, 1792, 6784] + - [3, 6417.15] + - - [5888, 256, 1, 256, 5888, 5888, 256, 256] + - [3, 4787.87] + - - [1408, 6784, 1, 3328, 1408, 1408, 3328, 6784] + - [3, 6166.1] + - - [64, 5056, 1, 3328, 64, 64, 3328, 5056] + - [0, 3593.09] + - - [1856, 1408, 1, 256, 1856, 1856, 256, 1408] + - [3, 5554.22] + - - [5888, 64, 1, 1792, 5888, 5888, 1792, 64] + - [0, 4139.92] + - - [2944, 4288, 1, 3328, 2944, 2944, 3328, 4288] + - [3, 6212.23] + - - [4288, 4, 1, 768, 4288, 4288, 768, 4] + - [5, 287.746] + - - [128, 4288, 1, 3328, 128, 128, 3328, 4288] + - [0, 3995.26] + - - [3584, 256, 1, 1792, 3584, 3584, 1792, 256] + - [3, 5031.9] + - - [448, 5056, 1, 1792, 448, 448, 1792, 5056] + - [3, 5268.31] + - - [1408, 128, 1, 1792, 1408, 1408, 1792, 128] + - [0, 3127.59] + - - [64, 4288, 1, 1792, 64, 64, 1792, 4288] + - [0, 2974.42] + - - [256, 64, 1, 3328, 256, 256, 3328, 64] + - [0, 404.886] + - - [6784, 1408, 1, 1792, 6784, 6784, 1792, 1408] + - [3, 6137.73] + - - [1024, 5888, 1, 256, 1024, 1024, 256, 5888] + - [1, 5543.18] + - - [2944, 2944, 1, 256, 2944, 2944, 256, 2944] + - [3, 5846.75] + - - [448, 4288, 1, 768, 448, 448, 768, 4288] + - [3, 5311.12] + - - [5056, 4, 1, 256, 5056, 5056, 256, 4] + - [5, 260.797] + - - [6784, 256, 1, 1792, 6784, 6784, 1792, 256] + - [1, 5048.46] + - - [128, 256, 1, 3328, 128, 128, 3328, 256] + - [0, 827.524] + - - [4, 448, 1, 1792, 4, 4, 1792, 448] + - [5, 42.8792] + - - [5056, 1856, 1, 3328, 5056, 5056, 3328, 1856] + - [3, 6049.43] + - - [128, 448, 1, 768, 128, 128, 768, 448] + - [0, 1385.85] + - - [4, 256, 1, 256, 4, 4, 256, 256] + - [5, 19.7695] + - - [2368, 6784, 1, 3328, 2368, 2368, 3328, 6784] + - [3, 6392.01] + - - [64, 6784, 1, 1792, 64, 64, 1792, 6784] + - [3, 3929.41] + - - [4288, 64, 1, 256, 4288, 4288, 256, 64] + - [0, 2285.33] + - - [5056, 2944, 1, 256, 5056, 5056, 256, 2944] + - [3, 6006.64] + - - [128, 5888, 1, 1792, 128, 128, 1792, 5888] + - [0, 4547.82] + - - [256, 4, 1, 1792, 256, 256, 1792, 4] + - [5, 24.0272] + - - [64, 5056, 1, 768, 64, 64, 768, 5056] + - [0, 3253.97] + - - [5888, 3584, 1, 1792, 5888, 5888, 1792, 3584] + - [1, 6332.66] + - - [1856, 5056, 1, 768, 1856, 1856, 768, 5056] + - [3, 5947.2] + - - [1856, 448, 1, 768, 1856, 1856, 768, 448] + - [3, 4280.38] + - - [1024, 1856, 1, 256, 1024, 1024, 256, 1856] + - [1, 4658.24] + - - [704, 256, 1, 3328, 704, 704, 3328, 256] + - [0, 3216.89] + - - [704, 5056, 1, 768, 704, 704, 768, 5056] + - [3, 5862.3] + - - [6784, 256, 1, 256, 6784, 6784, 256, 256] + - [1, 4482.38] + - - [256, 5888, 1, 3328, 256, 256, 3328, 5888] + - [3, 5706.54] + - - [448, 1856, 1, 256, 448, 448, 256, 1856] + - [3, 3790.97] + - - [256, 128, 1, 256, 256, 256, 256, 128] + - [0, 534.17] + - - [2368, 2944, 1, 256, 2368, 2368, 256, 2944] + - [3, 5690.26] + - - [4288, 64, 1, 768, 4288, 4288, 768, 64] + - [0, 2845.24] + - - [1856, 5888, 1, 256, 1856, 1856, 256, 5888] + - [3, 6029.16] + - - [704, 2368, 1, 256, 704, 704, 256, 2368] + - [3, 4211.44] + - - [4288, 5888, 1, 1792, 4288, 4288, 1792, 5888] + - [1, 6377.02] + - - [3584, 704, 1, 3328, 3584, 3584, 3328, 704] + - [1, 5035.61] + - - [1024, 2944, 1, 768, 1024, 1024, 768, 2944] + - [1, 5767.71] + - - [1024, 1856, 1, 1792, 1024, 1024, 1792, 1856] + - [1, 5481.26] + - - [1408, 1408, 1, 1792, 1408, 1408, 1792, 1408] + - [1, 5765.64] + - - [2368, 4, 1, 256, 2368, 2368, 256, 4] + - [5, 141.098] + - - [4, 4288, 1, 1792, 4, 4, 1792, 4288] + - [5, 311.978] + - - [4, 5056, 1, 256, 4, 4, 256, 5056] + - [5, 247.844] + - - [4, 2944, 1, 3328, 4, 4, 3328, 2944] + - [5, 232.964] + - - [2944, 6784, 1, 3328, 2944, 2944, 3328, 6784] + - [3, 6329.78] + - - [128, 1408, 1, 768, 128, 128, 768, 1408] + - [0, 2797.19] + - - [4288, 1408, 1, 3328, 4288, 4288, 3328, 1408] + - [1, 6217.82] + - - [4288, 1024, 1, 3328, 4288, 4288, 3328, 1024] + - [3, 5569.45] + - - [2944, 704, 1, 3328, 2944, 2944, 3328, 704] + - [0, 5041.66] + - - [5056, 448, 1, 3328, 5056, 5056, 3328, 448] + - [3, 5309.86] + - - [3584, 1408, 1, 768, 3584, 3584, 768, 1408] + - [1, 6006.02] + - - [3584, 5056, 1, 768, 3584, 3584, 768, 5056] + - [3, 6220.89] + - - [2368, 3584, 1, 3328, 2368, 2368, 3328, 3584] + - [3, 6116.6] + - - [5056, 1408, 1, 256, 5056, 5056, 256, 1408] + - [3, 5816.11] + - - [5888, 6784, 1, 256, 5888, 5888, 256, 6784] + - [1, 6230.82] + - - [704, 2368, 1, 768, 704, 704, 768, 2368] + - [0, 4708.2] + - - [1408, 5056, 1, 3328, 1408, 1408, 3328, 5056] + - [1, 6312.72] + - - [2368, 3584, 1, 1792, 2368, 2368, 1792, 3584] + - [3, 6093.92] + - - [704, 2944, 1, 768, 704, 704, 768, 2944] + - [3, 5784.1] + - - [1024, 2944, 1, 1792, 1024, 1024, 1792, 2944] + - [1, 5953.86] + - - [2944, 1856, 1, 1792, 2944, 2944, 1792, 1856] + - [3, 5970.07] + - - [5888, 128, 1, 1792, 5888, 5888, 1792, 128] + - [0, 4559.19] + - - [2368, 704, 1, 3328, 2368, 2368, 3328, 704] + - [0, 4954.12] + - - [448, 704, 1, 1792, 448, 448, 1792, 704] + - [0, 3560.32] + - - [3584, 2368, 1, 3328, 3584, 3584, 3328, 2368] + - [3, 6098.8] + - - [1024, 5888, 1, 1792, 1024, 1024, 1792, 5888] + - [1, 6161.87] + - - [4288, 1408, 1, 256, 4288, 4288, 256, 1408] + - [3, 5707.08] + - - [4, 3584, 1, 3328, 4, 4, 3328, 3584] + - [5, 283.234] + - - [5056, 3584, 1, 3328, 5056, 5056, 3328, 3584] + - [3, 6391.72] + - - [1408, 1856, 1, 768, 1408, 1408, 768, 1856] + - [1, 5021.03] + - - [1856, 5056, 1, 3328, 1856, 1856, 3328, 5056] + - [3, 6059.26] + - - [4, 64, 1, 768, 4, 4, 768, 64] + - [5, 5.59945] + - - [1024, 1856, 1, 768, 1024, 1024, 768, 1856] + - [1, 5269.96] + - - [4, 4288, 1, 256, 4, 4, 256, 4288] + - [5, 224.53] + - - [2944, 128, 1, 768, 2944, 2944, 768, 128] + - [0, 3800.49] + - - [448, 64, 1, 1792, 448, 448, 1792, 64] + - [0, 706.325] + - - [704, 4, 1, 768, 704, 704, 768, 4] + - [5, 60.0738] + - - [1024, 4288, 1, 1792, 1024, 1024, 1792, 4288] + - [3, 5800.06] + - - [448, 1856, 1, 1792, 448, 448, 1792, 1856] + - [3, 4537.53] + - - [448, 256, 1, 1792, 448, 448, 1792, 256] + - [0, 2768.93] + - - [2368, 448, 1, 1792, 2368, 2368, 1792, 448] + - [0, 4418.7] + - - [5888, 704, 1, 3328, 5888, 5888, 3328, 704] + - [3, 5531.66] + - - [1856, 256, 1, 256, 1856, 1856, 256, 256] + - [3, 3284.05] + - - [4, 1856, 1, 256, 4, 4, 256, 1856] + - [5, 103.456] + - - [6784, 6784, 1, 768, 6784, 6784, 768, 6784] + - [1, 6552.11] + - - [3584, 4288, 1, 1792, 3584, 3584, 1792, 4288] + - [1, 6406.99] + - - [5056, 1856, 1, 256, 5056, 5056, 256, 1856] + - [3, 5630.23] + - - [256, 2368, 1, 3328, 256, 256, 3328, 2368] + - [0, 4418.4] + - - [2368, 5056, 1, 3328, 2368, 2368, 3328, 5056] + - [1, 6271.11] + - - [1856, 6784, 1, 3328, 1856, 1856, 3328, 6784] + - [3, 6214.14] + - - [2368, 4, 1, 1792, 2368, 2368, 1792, 4] + - [5, 175.319] + - - [256, 4288, 1, 768, 256, 256, 768, 4288] + - [0, 4333.95] + - - [5056, 64, 1, 768, 5056, 5056, 768, 64] + - [0, 3364.92] + - - [448, 4, 1, 768, 448, 448, 768, 4] + - [5, 41.3787] + - - [1408, 1024, 1, 256, 1408, 1408, 256, 1024] + - [3, 4635.08] + - - [5056, 2368, 1, 768, 5056, 5056, 768, 2368] + - [1, 6116.23] + - - [448, 5888, 1, 256, 448, 448, 256, 5888] + - [3, 4762.84] + - - [6784, 2944, 1, 3328, 6784, 6784, 3328, 2944] + - [3, 6329.02] + - - [6784, 448, 1, 3328, 6784, 6784, 3328, 448] + - [3, 5184.27] + - - [2944, 1408, 1, 1792, 2944, 2944, 1792, 1408] + - [1, 6262.62] + - - [1856, 64, 1, 3328, 1856, 1856, 3328, 64] + - [0, 2900.33] + - - [256, 256, 1, 256, 256, 256, 256, 256] + - [0, 1034.19] + - - [4288, 5056, 1, 1792, 4288, 4288, 1792, 5056] + - [3, 6337.89] + - - [1408, 1024, 1, 1792, 1408, 1408, 1792, 1024] + - [3, 5412.62] + - - [1856, 5888, 1, 1792, 1856, 1856, 1792, 5888] + - [3, 6353.75] + - - [2944, 3584, 1, 3328, 2944, 2944, 3328, 3584] + - [3, 6172.76] + - - [4, 448, 1, 3328, 4, 4, 3328, 448] + - [5, 39.311] + - - [3584, 256, 1, 768, 3584, 3584, 768, 256] + - [3, 4792.17] + - - [704, 64, 1, 1792, 704, 704, 1792, 64] + - [0, 1082.39] + - - [5056, 2368, 1, 3328, 5056, 5056, 3328, 2368] + - [1, 6260.17] + - - [128, 3584, 1, 256, 128, 128, 256, 3584] + - [3, 3170.81] + - - [256, 4, 1, 768, 256, 256, 768, 4] + - [5, 25.8942] + - - [1408, 1408, 1, 768, 1408, 1408, 768, 1408] + - [1, 5611.16] + - - [64, 4, 1, 1792, 64, 64, 1792, 4] + - [5, 6.01263] + - - [704, 6784, 1, 1792, 704, 704, 1792, 6784] + - [1, 5792.18] + - - [1408, 5888, 1, 3328, 1408, 1408, 3328, 5888] + - [1, 6469.98] + - - [6784, 448, 1, 1792, 6784, 6784, 1792, 448] + - [3, 5095.52] + - - [6784, 3584, 1, 768, 6784, 6784, 768, 3584] + - [1, 6325.56] + - - [64, 128, 1, 3328, 64, 64, 3328, 128] + - [0, 220.648] + - - [1408, 4288, 1, 1792, 1408, 1408, 1792, 4288] + - [1, 6165.29] + - - [2368, 6784, 1, 768, 2368, 2368, 768, 6784] + - [3, 6284.3] + - - [2944, 64, 1, 256, 2944, 2944, 256, 64] + - [0, 2342.28] + - - [6784, 704, 1, 3328, 6784, 6784, 3328, 704] + - [1, 5853.24] + - - [3584, 5888, 1, 3328, 3584, 3584, 3328, 5888] + - [1, 6362.14] + - - [2368, 256, 1, 256, 2368, 2368, 256, 256] + - [0, 3183.89] + - - [128, 1024, 1, 256, 128, 128, 256, 1024] + - [0, 1856.45] + - - [64, 3584, 1, 3328, 64, 64, 3328, 3584] + - [2, 3828.06] + - - [5888, 448, 1, 768, 5888, 5888, 768, 448] + - [1, 5002.44] + - - [4, 2368, 1, 3328, 4, 4, 3328, 2368] + - [5, 193.519] + - - [448, 64, 1, 768, 448, 448, 768, 64] + - [0, 725.037] + - - [128, 704, 1, 768, 128, 128, 768, 704] + - [0, 2062.37] + - - [2368, 2944, 1, 1792, 2368, 2368, 1792, 2944] + - [1, 6141.36] + - - [6784, 128, 1, 256, 6784, 6784, 256, 128] + - [3, 4077.37] + - - [3584, 2368, 1, 256, 3584, 3584, 256, 2368] + - [3, 5557.03] + - - [5056, 4, 1, 768, 5056, 5056, 768, 4] + - [5, 306.548] + - - [4288, 1856, 1, 1792, 4288, 4288, 1792, 1856] + - [1, 6147.13] + - - [6784, 5056, 1, 3328, 6784, 6784, 3328, 5056] + - [1, 6417.63] + - - [64, 1408, 1, 1792, 64, 64, 1792, 1408] + - [0, 2116.45] + - - [5888, 2944, 1, 1792, 5888, 5888, 1792, 2944] + - [1, 6411.54] + - - [2368, 4288, 1, 1792, 2368, 2368, 1792, 4288] + - [3, 6180.38] + - - [1408, 1856, 1, 3328, 1408, 1408, 3328, 1856] + - [1, 5222.18] + - - [1856, 4, 1, 3328, 1856, 1856, 3328, 4] + - [5, 144.077] + - - [64, 1024, 1, 3328, 64, 64, 3328, 1024] + - [0, 1308.01] + - - [5888, 128, 1, 768, 5888, 5888, 768, 128] + - [0, 4274.5] + - - [704, 1856, 1, 1792, 704, 704, 1792, 1856] + - [0, 4951.3] + - - [128, 3584, 1, 768, 128, 128, 768, 3584] + - [3, 3885.8] + - - [256, 2944, 1, 1792, 256, 256, 1792, 2944] + - [0, 4622.76] + - - [6784, 3584, 1, 3328, 6784, 6784, 3328, 3584] + - [1, 6414.59] + - - [4, 2944, 1, 1792, 4, 4, 1792, 2944] + - [5, 225.022] + - - [448, 1024, 1, 768, 448, 448, 768, 1024] + - [3, 3921.04] + - - [2944, 3584, 1, 256, 2944, 2944, 256, 3584] + - [3, 5839.91] + - - [64, 256, 1, 3328, 64, 64, 3328, 256] + - [0, 410.989] + - - [5888, 4288, 1, 3328, 5888, 5888, 3328, 4288] + - [1, 6399.78] + - - [5056, 2944, 1, 768, 5056, 5056, 768, 2944] + - [3, 6225.44] + - - [5056, 2944, 1, 3328, 5056, 5056, 3328, 2944] + - [3, 6328.72] + - - [5056, 6784, 1, 256, 5056, 5056, 256, 6784] + - [3, 6162.6] + - - [256, 2368, 1, 768, 256, 256, 768, 2368] + - [0, 4091.81] + - - [3584, 704, 1, 768, 3584, 3584, 768, 704] + - [1, 4832.67] + - - [704, 4288, 1, 256, 704, 704, 256, 4288] + - [3, 5290.33] + - - [1408, 2368, 1, 768, 1408, 1408, 768, 2368] + - [3, 5503.51] + - - [256, 2944, 1, 768, 256, 256, 768, 2944] + - [0, 4355.5] + - - [6784, 5888, 1, 256, 6784, 6784, 256, 5888] + - [1, 6241.52] + - - [448, 1408, 1, 256, 448, 448, 256, 1408] + - [0, 3639.29] + - - [2944, 2944, 1, 768, 2944, 2944, 768, 2944] + - [3, 6114.08] + - - [256, 704, 1, 1792, 256, 256, 1792, 704] + - [0, 3123.12] + - - [4288, 2944, 1, 256, 4288, 4288, 256, 2944] + - [3, 5909.12] + - - [1408, 5056, 1, 256, 1408, 1408, 256, 5056] + - [3, 5676.54] + - - [128, 5056, 1, 1792, 128, 128, 1792, 5056] + - [0, 4575.08] + - - [2368, 2944, 1, 3328, 2368, 2368, 3328, 2944] + - [1, 6185.87] + - - [1856, 4288, 1, 256, 1856, 1856, 256, 4288] + - [3, 5681.47] + - - [6784, 1856, 1, 768, 6784, 6784, 768, 1856] + - [3, 6060.7] + - - [448, 128, 1, 768, 448, 448, 768, 128] + - [0, 1443.02] + - - [6784, 1024, 1, 3328, 6784, 6784, 3328, 1024] + - [1, 5768.18] + - - [1856, 64, 1, 256, 1856, 1856, 256, 64] + - [0, 1900.48] + - - [256, 1024, 1, 3328, 256, 256, 3328, 1024] + - [0, 4547.22] + - - [5888, 5056, 1, 1792, 5888, 5888, 1792, 5056] + - [1, 6479.74] + - - [6784, 2944, 1, 256, 6784, 6784, 256, 2944] + - [3, 6033.38] + - - [256, 704, 1, 256, 256, 256, 256, 704] + - [4, 2208.64] + - - [4288, 4, 1, 256, 4288, 4288, 256, 4] + - [5, 235.217] + - - [128, 4, 1, 768, 128, 128, 768, 4] + - [5, 13.1394] + - - [1024, 4288, 1, 768, 1024, 1024, 768, 4288] + - [3, 5607.08] + - - [6784, 6784, 1, 3328, 6784, 6784, 3328, 6784] + - [1, 6657.53] + - - [5056, 5056, 1, 256, 5056, 5056, 256, 5056] + - [1, 6053.86] + - - [2368, 128, 1, 1792, 2368, 2368, 1792, 128] + - [0, 3385.28] + - - [448, 2368, 1, 256, 448, 448, 256, 2368] + - [0, 3695.81] + - - [3584, 3584, 1, 256, 3584, 3584, 256, 3584] + - [3, 5891.36] + - - [4288, 2368, 1, 256, 4288, 4288, 256, 2368] + - [3, 5785.87] + - - [128, 256, 1, 1792, 128, 128, 1792, 256] + - [0, 840.614] + - - [1856, 256, 1, 1792, 1856, 1856, 1792, 256] + - [3, 4332.53] + - - [2944, 4288, 1, 256, 2944, 2944, 256, 4288] + - [3, 5830.93] + - - [704, 1408, 1, 3328, 704, 704, 3328, 1408] + - [3, 5518.34] + - - [704, 64, 1, 256, 704, 704, 256, 64] + - [0, 704.559] + - - [1856, 128, 1, 1792, 1856, 1856, 1792, 128] + - [0, 4110.95] + - - [1024, 1024, 1, 768, 1024, 1024, 768, 1024] + - [3, 5485.05] + - - [1856, 2368, 1, 3328, 1856, 1856, 3328, 2368] + - [3, 5886.24] + - - [704, 1024, 1, 768, 704, 704, 768, 1024] + - [3, 3842.71] + - - [1408, 128, 1, 256, 1408, 1408, 256, 128] + - [0, 2341.52] + - - [448, 4288, 1, 256, 448, 448, 256, 4288] + - [3, 4779.62] + - - [4288, 256, 1, 768, 4288, 4288, 768, 256] + - [3, 4027.23] + - - [5888, 2368, 1, 256, 5888, 5888, 256, 2368] + - [1, 5781.05] + - - [128, 6784, 1, 3328, 128, 128, 3328, 6784] + - [3, 4829.46] + - - [2368, 1408, 1, 256, 2368, 2368, 256, 1408] + - [3, 5263.83] + - - [3584, 448, 1, 768, 3584, 3584, 768, 448] + - [1, 4485.8] + - - [3584, 1856, 1, 768, 3584, 3584, 768, 1856] + - [1, 5720.41] + - - [3584, 5888, 1, 1792, 3584, 3584, 1792, 5888] + - [1, 6334.79] + - - [128, 5888, 1, 256, 128, 128, 256, 5888] + - [0, 3577.76] + - - [64, 448, 1, 768, 64, 64, 768, 448] + - [0, 718.027] + - - [4288, 2368, 1, 1792, 4288, 4288, 1792, 2368] + - [3, 6183.8] + - - [3584, 5056, 1, 1792, 3584, 3584, 1792, 5056] + - [3, 6328.27] + - - [1024, 5056, 1, 256, 1024, 1024, 256, 5056] + - [1, 5537.74] + - - [6784, 448, 1, 256, 6784, 6784, 256, 448] + - [3, 4358.22] + - - [6784, 128, 1, 768, 6784, 6784, 768, 128] + - [3, 4602.68] + - - [448, 4288, 1, 3328, 448, 448, 3328, 4288] + - [3, 5587.57] + - - [1408, 4288, 1, 768, 1408, 1408, 768, 4288] + - [1, 6021.67] + - - [64, 1408, 1, 768, 64, 64, 768, 1408] + - [0, 1902.75] + - - [2368, 64, 1, 1792, 2368, 2368, 1792, 64] + - [0, 2664.43] + - - [3584, 5888, 1, 256, 3584, 3584, 256, 5888] + - [1, 6004.84] + - - [1856, 1408, 1, 1792, 1856, 1856, 1792, 1408] + - [3, 6106.26] + - - [128, 128, 1, 768, 128, 128, 768, 128] + - [0, 410.308] + - - [256, 6784, 1, 3328, 256, 256, 3328, 6784] + - [1, 5098.99] + - - [5056, 4288, 1, 3328, 5056, 5056, 3328, 4288] + - [3, 6375.07] + - - [128, 5888, 1, 768, 128, 128, 768, 5888] + - [0, 4258.21] + - - [2368, 2368, 1, 256, 2368, 2368, 256, 2368] + - [3, 5635.0] + - - [64, 5056, 1, 1792, 64, 64, 1792, 5056] + - [0, 3500.87] + - - [1408, 2368, 1, 1792, 1408, 1408, 1792, 2368] + - [3, 5631.34] + - - [6784, 5888, 1, 3328, 6784, 6784, 3328, 5888] + - [1, 6523.26] + - - [128, 1408, 1, 1792, 128, 128, 1792, 1408] + - [0, 3132.1] + - - [6784, 256, 1, 768, 6784, 6784, 768, 256] + - [3, 4904.85] + - - [128, 3584, 1, 1792, 128, 128, 1792, 3584] + - [3, 4263.49] + - - [4288, 704, 1, 768, 4288, 4288, 768, 704] + - [3, 4974.2] + - - [1408, 6784, 1, 1792, 1408, 1408, 1792, 6784] + - [3, 6135.45] + - - [4, 704, 1, 256, 4, 4, 256, 704] + - [5, 41.0627] + - - [2368, 1856, 1, 3328, 2368, 2368, 3328, 1856] + - [3, 5885.54] + - - [6784, 64, 1, 256, 6784, 6784, 256, 64] + - [0, 3025.12] + - - [5888, 1024, 1, 1792, 5888, 5888, 1792, 1024] + - [1, 6098.23] + - - [128, 64, 1, 768, 128, 128, 768, 64] + - [0, 206.145] + - - [1856, 5888, 1, 3328, 1856, 1856, 3328, 5888] + - [3, 6383.74] + - - [256, 1408, 1, 256, 256, 256, 256, 1408] + - [0, 3023.52] + - - [704, 704, 1, 1792, 704, 704, 1792, 704] + - [0, 4490.51] + - - [5056, 1856, 1, 768, 5056, 5056, 768, 1856] + - [3, 5921.86] + - - [1856, 5056, 1, 256, 1856, 1856, 256, 5056] + - [3, 5659.7] + - - [5888, 256, 1, 768, 5888, 5888, 768, 256] + - [3, 5396.26] + - - [704, 1024, 1, 256, 704, 704, 256, 1024] + - [3, 3445.85] + - - [128, 4, 1, 256, 128, 128, 256, 4] + - [5, 10.4703] + - - [5888, 4, 1, 256, 5888, 5888, 256, 4] + - [5, 290.695] + - - [3584, 1408, 1, 256, 3584, 3584, 256, 1408] + - [1, 5553.98] + - - [5056, 128, 1, 1792, 5056, 5056, 1792, 128] + - [0, 4442.65] + - - [4, 5056, 1, 768, 4, 4, 768, 5056] + - [5, 327.621] + - - [128, 704, 1, 3328, 128, 128, 3328, 704] + - [0, 2188.35] + - - [4, 1408, 1, 1792, 4, 4, 1792, 1408] + - [5, 113.918] + - - [3584, 64, 1, 3328, 3584, 3584, 3328, 64] + - [0, 4156.98] + - - [5888, 448, 1, 3328, 5888, 5888, 3328, 448] + - [1, 5258.9] + - - [448, 5056, 1, 256, 448, 448, 256, 5056] + - [3, 4604.68] + - - [64, 704, 1, 256, 64, 64, 256, 704] + - [0, 714.267] + - - [3584, 1408, 1, 3328, 3584, 3584, 3328, 1408] + - [1, 6196.89] + - - [128, 448, 1, 3328, 128, 128, 3328, 448] + - [0, 1417.88] + - - [4288, 448, 1, 3328, 4288, 4288, 3328, 448] + - [0, 4978.69] + - - [256, 1024, 1, 768, 256, 256, 768, 1024] + - [0, 3938.89] + - - [4, 1408, 1, 256, 4, 4, 256, 1408] + - [5, 81.4388] + - - [1024, 1024, 1, 1792, 1024, 1024, 1792, 1024] + - [3, 5676.11] + - - [1856, 1024, 1, 768, 1856, 1856, 768, 1024] + - [3, 5338.49] + - - [3584, 6784, 1, 1792, 3584, 3584, 1792, 6784] + - [1, 6396.67] + - - [448, 5888, 1, 768, 448, 448, 768, 5888] + - [3, 5059.81] + - - [2944, 1024, 1, 256, 2944, 2944, 256, 1024] + - [1, 5329.1] + - - [2368, 5056, 1, 256, 2368, 2368, 256, 5056] + - [1, 5779.62] + - - [256, 128, 1, 1792, 256, 256, 1792, 128] + - [0, 833.55] + - - [448, 448, 1, 1792, 448, 448, 1792, 448] + - [0, 3513.25] + - - [6784, 2368, 1, 256, 6784, 6784, 256, 2368] + - [1, 5849.5] + - - [256, 3584, 1, 3328, 256, 256, 3328, 3584] + - [3, 5083.32] + - - [5056, 1024, 1, 3328, 5056, 5056, 3328, 1024] + - [1, 6170.31] + - - [256, 64, 1, 256, 256, 256, 256, 64] + - [0, 260.929] + - - [64, 704, 1, 768, 64, 64, 768, 704] + - [0, 1099.14] + - - [1024, 2368, 1, 3328, 1024, 1024, 3328, 2368] + - [3, 5694.13] + - - [4288, 3584, 1, 3328, 4288, 4288, 3328, 3584] + - [1, 6459.64] + - - [704, 5888, 1, 3328, 704, 704, 3328, 5888] + - [3, 6224.02] + - - [6784, 64, 1, 3328, 6784, 6784, 3328, 64] + - [0, 4065.48] + - - [128, 2944, 1, 3328, 128, 128, 3328, 2944] + - [0, 4233.53] + - - [2368, 5888, 1, 256, 2368, 2368, 256, 5888] + - [3, 6028.41] + - - [4288, 448, 1, 256, 4288, 4288, 256, 448] + - [0, 3949.38] + - - [1024, 4288, 1, 3328, 1024, 1024, 3328, 4288] + - [3, 5857.19] + - - [3584, 2944, 1, 1792, 3584, 3584, 1792, 2944] + - [3, 6133.18] + - - [4288, 4288, 1, 768, 4288, 4288, 768, 4288] + - [3, 6192.74] + - - [5888, 1408, 1, 3328, 5888, 5888, 3328, 1408] + - [1, 6452.83] + - - [2368, 128, 1, 3328, 2368, 2368, 3328, 128] + - [0, 3461.5] + - - [2944, 1856, 1, 768, 2944, 2944, 768, 1856] + - [3, 5869.25] + - - [2944, 448, 1, 256, 2944, 2944, 256, 448] + - [3, 4048.0] + - - [1024, 448, 1, 768, 1024, 1024, 768, 448] + - [0, 3870.01] + - - [5056, 2368, 1, 256, 5056, 5056, 256, 2368] + - [1, 5758.71] + - - [704, 2368, 1, 3328, 704, 704, 3328, 2368] + - [0, 4980.47] + - - [64, 1856, 1, 3328, 64, 64, 3328, 1856] + - [0, 2794.03] + - - [1024, 3584, 1, 3328, 1024, 1024, 3328, 3584] + - [3, 6245.07] + - - [448, 704, 1, 256, 448, 448, 256, 704] + - [0, 2883.48] + - - [2944, 2944, 1, 1792, 2944, 2944, 1792, 2944] + - [3, 6196.11] + - - [1408, 128, 1, 3328, 1408, 1408, 3328, 128] + - [0, 3219.44] + - - [4, 1856, 1, 1792, 4, 4, 1792, 1856] + - [5, 148.55] + - - [1024, 1408, 1, 768, 1024, 1024, 768, 1408] + - [3, 5179.53] + - - [4, 1024, 1, 1792, 4, 4, 1792, 1024] + - [5, 80.8222] + - - [5888, 5888, 1, 1792, 5888, 5888, 1792, 5888] + - [1, 6468.86] + - - [128, 1408, 1, 3328, 128, 128, 3328, 1408] + - [0, 3225.86] + - - [1408, 3584, 1, 768, 1408, 1408, 768, 3584] + - [1, 6022.62] + - - [5888, 5888, 1, 768, 5888, 5888, 768, 5888] + - [1, 6393.13] + - - [704, 2944, 1, 1792, 704, 704, 1792, 2944] + - [3, 5958.23] + - - [5888, 256, 1, 3328, 5888, 5888, 3328, 256] + - [3, 5706.07] + - - [64, 2944, 1, 256, 64, 64, 256, 2944] + - [0, 2237.58] + - - [2944, 5056, 1, 768, 2944, 2944, 768, 5056] + - [3, 6197.94] + - - [6784, 704, 1, 1792, 6784, 6784, 1792, 704] + - [1, 5798.87] + - - [1024, 4, 1, 3328, 1024, 1024, 3328, 4] + - [5, 82.9671] + - - [64, 1856, 1, 768, 64, 64, 768, 1856] + - [0, 2382.04] + - - [1408, 64, 1, 768, 1408, 1408, 768, 64] + - [0, 1971.01] + - - [448, 2944, 1, 1792, 448, 448, 1792, 2944] + - [3, 4934.48] + - - [2944, 4, 1, 768, 2944, 2944, 768, 4] + - [5, 212.702] + - - [704, 128, 1, 768, 704, 704, 768, 128] + - [0, 2118.5] + - - [448, 4, 1, 3328, 448, 448, 3328, 4] + - [5, 43.331] + - - [2368, 6784, 1, 1792, 2368, 2368, 1792, 6784] + - [3, 6361.59] + - - [3584, 1024, 1, 1792, 3584, 3584, 1792, 1024] + - [3, 6153.08] + - - [6784, 4, 1, 1792, 6784, 6784, 1792, 4] + - [5, 165.814] + - - [3584, 64, 1, 256, 3584, 3584, 256, 64] + - [0, 2791.22] + - - [6784, 3584, 1, 256, 6784, 6784, 256, 3584] + - [3, 6096.84] + - - [64, 1408, 1, 256, 64, 64, 256, 1408] + - [0, 1371.91] + - - [1024, 448, 1, 1792, 1024, 1024, 1792, 448] + - [3, 4110.33] + - - [704, 1408, 1, 1792, 704, 704, 1792, 1408] + - [3, 5437.49] + - - [2944, 256, 1, 3328, 2944, 2944, 3328, 256] + - [2, 4402.52] + - - [1856, 1408, 1, 768, 1856, 1856, 768, 1408] + - [3, 5976.94] + - - [1024, 64, 1, 1792, 1024, 1024, 1792, 64] + - [0, 1542.23] + - - [1408, 2944, 1, 3328, 1408, 1408, 3328, 2944] + - [1, 6321.2] + - - [448, 3584, 1, 256, 448, 448, 256, 3584] + - [3, 4172.12] + - - [3584, 2944, 1, 768, 3584, 3584, 768, 2944] + - [3, 6049.17] + - - [4, 2368, 1, 768, 4, 4, 768, 2368] + - [5, 185.991] + - - [3584, 1856, 1, 256, 3584, 3584, 256, 1856] + - [3, 5300.54] + - - [448, 5888, 1, 3328, 448, 448, 3328, 5888] + - [1, 5259.36] + - - [4288, 4, 1, 1792, 4288, 4288, 1792, 4] + - [5, 161.076] + - - [2368, 6784, 1, 256, 2368, 2368, 256, 6784] + - [3, 6071.48] + - - [1408, 5888, 1, 256, 1408, 1408, 256, 5888] + - [1, 5939.17] + - - [4, 128, 1, 768, 4, 4, 768, 128] + - [5, 13.4728] + - - [3584, 704, 1, 256, 3584, 3584, 256, 704] + - [1, 4368.61] + - - [5056, 1408, 1, 1792, 5056, 5056, 1792, 1408] + - [1, 6262.13] + - - [704, 4, 1, 3328, 704, 704, 3328, 4] + - [5, 56.5301] + - - [6784, 5056, 1, 768, 6784, 6784, 768, 5056] + - [1, 6320.66] + - - [4288, 704, 1, 1792, 4288, 4288, 1792, 704] + - [3, 5089.05] + - - [4, 2368, 1, 256, 4, 4, 256, 2368] + - [5, 133.611] + - - [6784, 5888, 1, 768, 6784, 6784, 768, 5888] + - [1, 6440.21] + - - [704, 1856, 1, 3328, 704, 704, 3328, 1856] + - [0, 5031.5] + - - [5056, 256, 1, 768, 5056, 5056, 768, 256] + - [3, 4726.79] + - - [3584, 128, 1, 3328, 3584, 3584, 3328, 128] + - [3, 4265.37] + - - [1856, 6784, 1, 256, 1856, 1856, 256, 6784] + - [3, 5914.61] + - - [448, 3584, 1, 3328, 448, 448, 3328, 3584] + - [3, 4693.0] + - - [128, 128, 1, 256, 128, 128, 256, 128] + - [0, 299.582] + - - [2944, 128, 1, 256, 2944, 2944, 256, 128] + - [0, 3049.87] + - - [2368, 1856, 1, 256, 2368, 2368, 256, 1856] + - [3, 5353.82] + - - [4, 2944, 1, 768, 4, 4, 768, 2944] + - [5, 216.091] + - - [4, 1024, 1, 3328, 4, 4, 3328, 1024] + - [5, 80.7818] + - - [2944, 5888, 1, 256, 2944, 2944, 256, 5888] + - [1, 6043.15] + - - [6784, 1024, 1, 1792, 6784, 6784, 1792, 1024] + - [1, 5977.99] + - - [1856, 704, 1, 256, 1856, 1856, 256, 704] + - [3, 4120.07] + - - [3584, 256, 1, 256, 3584, 3584, 256, 256] + - [3, 3973.46] + - - [5888, 5056, 1, 256, 5888, 5888, 256, 5056] + - [1, 6119.17] + - - [448, 1408, 1, 1792, 448, 448, 1792, 1408] + - [0, 4530.74] + - - [256, 256, 1, 768, 256, 256, 768, 256] + - [0, 1629.38] + - - [128, 1024, 1, 1792, 128, 128, 1792, 1024] + - [0, 2828.56] + - - [128, 704, 1, 256, 128, 128, 256, 704] + - [0, 1336.54] + - - [5888, 256, 1, 1792, 5888, 5888, 1792, 256] + - [3, 5639.4] + - - [448, 2368, 1, 3328, 448, 448, 3328, 2368] + - [0, 4531.83] + - - [2944, 1024, 1, 1792, 2944, 2944, 1792, 1024] + - [1, 5958.72] + - - [2368, 448, 1, 3328, 2368, 2368, 3328, 448] + - [0, 4505.28] + - - [4288, 704, 1, 3328, 4288, 4288, 3328, 704] + - [3, 5143.48] + - - [5888, 4288, 1, 768, 5888, 5888, 768, 4288] + - [1, 6295.08] + - - [2944, 5888, 1, 1792, 2944, 2944, 1792, 5888] + - [1, 6413.89] + - - [64, 4288, 1, 3328, 64, 64, 3328, 4288] + - [0, 3069.3] + - - [5056, 256, 1, 256, 5056, 5056, 256, 256] + - [3, 4288.71] + - - [5056, 1024, 1, 1792, 5056, 5056, 1792, 1024] + - [1, 6279.32] + - - [1408, 704, 1, 256, 1408, 1408, 256, 704] + - [0, 3810.48] + - - [3584, 4, 1, 256, 3584, 3584, 256, 4] + - [5, 186.257] + - - [4, 5056, 1, 3328, 4, 4, 3328, 5056] + - [5, 366.225] + - - [5888, 3584, 1, 256, 5888, 5888, 256, 3584] + - [3, 6015.74] + - - [448, 3584, 1, 1792, 448, 448, 1792, 3584] + - [3, 4637.36] + - - [4288, 4, 1, 3328, 4288, 4288, 3328, 4] + - [5, 161.177] + - - [1408, 64, 1, 256, 1408, 1408, 256, 64] + - [0, 1428.53] + - - [64, 256, 1, 768, 64, 64, 768, 256] + - [0, 412.291] + - - [64, 6784, 1, 768, 64, 64, 768, 6784] + - [3, 3683.64] + - - [4, 4, 1, 3328, 4, 4, 3328, 4] + - [5, 0.375368] + - - [4288, 128, 1, 1792, 4288, 4288, 1792, 128] + - [0, 3847.52] + - - [2944, 5888, 1, 3328, 2944, 2944, 3328, 5888] + - [1, 6442.87] + - - [64, 3584, 1, 768, 64, 64, 768, 3584] + - [2, 3363.64] + - - [5888, 5056, 1, 3328, 5888, 5888, 3328, 5056] + - [1, 6510.39] + - - [128, 2368, 1, 3328, 128, 128, 3328, 2368] + - [0, 3455.35] + - - [2368, 1856, 1, 1792, 2368, 2368, 1792, 1856] + - [3, 5825.83] + - - [64, 64, 1, 3328, 64, 64, 3328, 64] + - [0, 118.954] + - - [5888, 1856, 1, 256, 5888, 5888, 256, 1856] + - [1, 5663.02] + - - [2944, 4, 1, 3328, 2944, 2944, 3328, 4] + - [5, 199.947] + - - [4, 448, 1, 256, 4, 4, 256, 448] + - [5, 27.6473] + - - [128, 5056, 1, 768, 128, 128, 768, 5056] + - [0, 4287.32] + - - [6784, 5888, 1, 1792, 6784, 6784, 1792, 5888] + - [1, 6504.35] + - - [6784, 2368, 1, 768, 6784, 6784, 768, 2368] + - [1, 6169.54] + - - [1408, 2944, 1, 1792, 1408, 1408, 1792, 2944] + - [1, 6266.93] + - - [5056, 704, 1, 1792, 5056, 5056, 1792, 704] + - [1, 5355.28] + - - [5888, 5888, 1, 256, 5888, 5888, 256, 5888] + - [1, 6193.09] + - - [2944, 1408, 1, 256, 2944, 2944, 256, 1408] + - [3, 5732.42] + - - [64, 4288, 1, 256, 64, 64, 256, 4288] + - [0, 2296.39] + - - [1856, 1408, 1, 3328, 1856, 1856, 3328, 1408] + - [3, 6161.06] + - - [1408, 1856, 1, 256, 1408, 1408, 256, 1856] + - [3, 4676.94] + - - [6784, 256, 1, 3328, 6784, 6784, 3328, 256] + - [1, 5096.66] + - - [256, 704, 1, 768, 256, 256, 768, 704] + - [0, 2879.06] + - - [448, 2944, 1, 3328, 448, 448, 3328, 2944] + - [3, 5009.69] + - - [4, 6784, 1, 3328, 4, 4, 3328, 6784] + - [5, 373.625] + - - [256, 5888, 1, 256, 256, 256, 256, 5888] + - [3, 4414.58] + - - [1856, 2368, 1, 1792, 1856, 1856, 1792, 2368] + - [3, 5840.23] + - - [3584, 128, 1, 256, 3584, 3584, 256, 128] + - [0, 3196.4] + - - [1408, 448, 1, 3328, 1408, 1408, 3328, 448] + - [0, 4621.52] + - - [704, 448, 1, 256, 704, 704, 256, 448] + - [0, 2914.34] + - - [1408, 256, 1, 256, 1408, 1408, 256, 256] + - [0, 2980.11] + - - [3584, 4, 1, 1792, 3584, 3584, 1792, 4] + - [5, 192.243] + - - [2368, 256, 1, 1792, 2368, 2368, 1792, 256] + - [2, 3490.01] + - - [256, 448, 1, 256, 256, 256, 256, 448] + - [0, 1604.69] + - - [1856, 1024, 1, 3328, 1856, 1856, 3328, 1024] + - [1, 5570.92] + - - [448, 6784, 1, 256, 448, 448, 256, 6784] + - [3, 5411.29] + - - [2944, 704, 1, 768, 2944, 2944, 768, 704] + - [0, 4692.71] + - - [3584, 4288, 1, 768, 3584, 3584, 768, 4288] + - [1, 6282.87] + - - [1408, 4, 1, 768, 1408, 1408, 768, 4] + - [5, 107.932] + - - [6784, 1408, 1, 256, 6784, 6784, 256, 1408] + - [3, 5811.7] + - - [64, 2368, 1, 256, 64, 64, 256, 2368] + - [0, 1932.67] + - - [4, 2944, 1, 256, 4, 4, 256, 2944] + - [5, 162.787] + - - [5056, 6784, 1, 768, 5056, 5056, 768, 6784] + - [3, 6347.05] + - - [1408, 1408, 1, 256, 1408, 1408, 256, 1408] + - [3, 5109.03] + - - [256, 5056, 1, 256, 256, 256, 256, 5056] + - [0, 3972.57] + - - [3584, 2944, 1, 256, 3584, 3584, 256, 2944] + - [3, 5801.81] + - - [448, 256, 1, 256, 448, 448, 256, 256] + - [0, 1730.78] + - - [1408, 5888, 1, 768, 1408, 1408, 768, 5888] + - [1, 6306.07] + - - [5056, 5056, 1, 768, 5056, 5056, 768, 5056] + - [1, 6365.0] + - - [256, 5056, 1, 1792, 256, 256, 1792, 5056] + - [0, 4859.06] + - - [6784, 4, 1, 3328, 6784, 6784, 3328, 4] + - [5, 168.413] + - - [4, 256, 1, 768, 4, 4, 768, 256] + - [5, 26.021] + - - [4, 4, 1, 1792, 4, 4, 1792, 4] + - [5, 0.347453] + - - [5888, 6784, 1, 3328, 5888, 5888, 3328, 6784] + - [1, 6521.7] + - - [704, 704, 1, 3328, 704, 704, 3328, 704] + - [0, 4591.03] + - - [2944, 256, 1, 1792, 2944, 2944, 1792, 256] + - [2, 4287.91] + - - [5888, 6784, 1, 768, 5888, 5888, 768, 6784] + - [1, 6417.16] + - - [704, 3584, 1, 1792, 704, 704, 1792, 3584] + - [3, 5891.15] + - - [1408, 4288, 1, 256, 1408, 1408, 256, 4288] + - [3, 5613.42] + - - [448, 4, 1, 256, 448, 448, 256, 4] + - [5, 31.1198] + - - [2368, 5888, 1, 768, 2368, 2368, 768, 5888] + - [3, 6249.96] + - - [5056, 6784, 1, 3328, 5056, 5056, 3328, 6784] + - [3, 6440.91] + - - [5888, 1408, 1, 256, 5888, 5888, 256, 1408] + - [3, 5879.45] + - - [128, 4288, 1, 1792, 128, 128, 1792, 4288] + - [0, 3908.68] + - - [256, 4288, 1, 1792, 256, 256, 1792, 4288] + - [0, 4560.46] + - - [4288, 2944, 1, 768, 4288, 4288, 768, 2944] + - [3, 6126.82] + - - [4288, 3584, 1, 1792, 4288, 4288, 1792, 3584] + - [1, 6425.92] + - - [2944, 1408, 1, 768, 2944, 2944, 768, 1408] + - [1, 6113.21] + - - [256, 4, 1, 3328, 256, 256, 3328, 4] + - [5, 24.9756] + - - [256, 4288, 1, 256, 256, 256, 256, 4288] + - [0, 3744.96] + - - [2944, 5888, 1, 768, 2944, 2944, 768, 5888] + - [1, 6325.85] + - - [1024, 3584, 1, 1792, 1024, 1024, 1792, 3584] + - [3, 6154.79] + - - [2368, 4, 1, 3328, 2368, 2368, 3328, 4] + - [5, 180.241] + - - [2944, 1856, 1, 3328, 2944, 2944, 3328, 1856] + - [3, 6012.04] + - - [704, 1856, 1, 256, 704, 704, 256, 1856] + - [3, 4135.14] + - - [1408, 5056, 1, 768, 1408, 1408, 768, 5056] + - [1, 6124.0] + - - [448, 5056, 1, 3328, 448, 448, 3328, 5056] + - [3, 5331.55] + - - [5888, 1856, 1, 3328, 5888, 5888, 3328, 1856] + - [1, 6224.78] + - - [1856, 704, 1, 768, 1856, 1856, 768, 704] + - [3, 4705.36] + - - [448, 1856, 1, 3328, 448, 448, 3328, 1856] + - [3, 4611.88] + - - [2368, 64, 1, 256, 2368, 2368, 256, 64] + - [0, 1890.8] + - - [64, 5888, 1, 256, 64, 64, 256, 5888] + - [0, 2724.5] + - - [3584, 64, 1, 768, 3584, 3584, 768, 64] + - [0, 3686.99] + - - [448, 6784, 1, 1792, 448, 448, 1792, 6784] + - [3, 5962.96] + - - [6784, 1024, 1, 256, 6784, 6784, 256, 1024] + - [3, 5621.07] + - - [64, 128, 1, 1792, 64, 64, 1792, 128] + - [0, 229.636] + - - [2368, 128, 1, 256, 2368, 2368, 256, 128] + - [0, 2658.62] + - - [4288, 5056, 1, 768, 4288, 4288, 768, 5056] + - [3, 6249.8] + - - [6784, 1408, 1, 768, 6784, 6784, 768, 1408] + - [3, 6048.46] + - - [5888, 1024, 1, 3328, 5888, 5888, 3328, 1024] + - [1, 5953.11] + - - [1856, 3584, 1, 256, 1856, 1856, 256, 3584] + - [3, 5797.83] + - - [128, 1856, 1, 768, 128, 128, 768, 1856] + - [0, 3748.95] + - - [64, 256, 1, 256, 64, 64, 256, 256] + - [0, 304.42] + - - [1856, 4288, 1, 1792, 1856, 1856, 1792, 4288] + - [1, 6154.0] + - - [3584, 448, 1, 1792, 3584, 3584, 1792, 448] + - [0, 4694.52] + - - [128, 448, 1, 1792, 128, 128, 1792, 448] + - [0, 1368.08] + - - [3584, 4, 1, 768, 3584, 3584, 768, 4] + - [5, 248.548] + - - [704, 5056, 1, 256, 704, 704, 256, 5056] + - [3, 5459.86] + - - [704, 6784, 1, 256, 704, 704, 256, 6784] + - [3, 5376.3] + - - [3584, 1856, 1, 3328, 3584, 3584, 3328, 1856] + - [1, 5895.75] + - - [448, 128, 1, 256, 448, 448, 256, 128] + - [0, 921.738] + - - [6784, 1856, 1, 1792, 6784, 6784, 1792, 1856] + - [3, 6144.65] + - - [1856, 256, 1, 3328, 1856, 1856, 3328, 256] + - [3, 4425.96] + - - [256, 1856, 1, 768, 256, 256, 768, 1856] + - [0, 4034.48] + - - [4, 448, 1, 768, 4, 4, 768, 448] + - [5, 38.7874] + - - [2368, 4288, 1, 256, 2368, 2368, 256, 4288] + - [3, 5806.94] + - - [4288, 4288, 1, 256, 4288, 4288, 256, 4288] + - [3, 5949.79] + - - [704, 256, 1, 256, 704, 704, 256, 256] + - [0, 2404.8] + - - [1408, 256, 1, 768, 1408, 1408, 768, 256] + - [0, 3663.75] + - - [128, 256, 1, 768, 128, 128, 768, 256] + - [0, 822.587] + - - [704, 1024, 1, 3328, 704, 704, 3328, 1024] + - [2, 4043.4] + - - [1408, 4288, 1, 3328, 1408, 1408, 3328, 4288] + - [1, 6220.1] + - - [5056, 5888, 1, 768, 5056, 5056, 768, 5888] + - [1, 6413.4] + - - [2368, 704, 1, 768, 2368, 2368, 768, 704] + - [0, 4625.04] + - - [4, 6784, 1, 768, 4, 4, 768, 6784] + - [5, 350.36] + - - [256, 6784, 1, 768, 256, 256, 768, 6784] + - [1, 4884.89] + - - [4, 128, 1, 3328, 4, 4, 3328, 128] + - [5, 13.0179] + - - [6784, 448, 1, 768, 6784, 6784, 768, 448] + - [3, 4929.6] + - - [5888, 2944, 1, 768, 5888, 5888, 768, 2944] + - [1, 6317.84] + - - [256, 64, 1, 768, 256, 256, 768, 64] + - [0, 411.3] + - - [5056, 1024, 1, 256, 5056, 5056, 256, 1024] + - [3, 5766.18] + - - [1856, 1856, 1, 256, 1856, 1856, 256, 1856] + - [3, 5225.95] + - - [5888, 2368, 1, 3328, 5888, 5888, 3328, 2368] + - [1, 6264.06] + - - [448, 4288, 1, 1792, 448, 448, 1792, 4288] + - [3, 5516.0] + - - [448, 2944, 1, 256, 448, 448, 256, 2944] + - [3, 4320.43] + - - [1408, 4, 1, 256, 1408, 1408, 256, 4] + - [5, 90.1092] + - - [256, 4, 1, 256, 256, 256, 256, 4] + - [5, 19.8812] + - - [5888, 1024, 1, 256, 5888, 5888, 256, 1024] + - [3, 5571.33] + - - [704, 5056, 1, 3328, 704, 704, 3328, 5056] + - [3, 6073.22] + - - [64, 64, 1, 1792, 64, 64, 1792, 64] + - [0, 116.029] + - - [5888, 2944, 1, 256, 5888, 5888, 256, 2944] + - [1, 6026.74] + - - [4288, 256, 1, 3328, 4288, 4288, 3328, 256] + - [3, 4189.39] + - - [2944, 704, 1, 256, 2944, 2944, 256, 704] + - [3, 4198.5] + - - [64, 448, 1, 256, 64, 64, 256, 448] + - [0, 483.358] + - - [1024, 128, 1, 256, 1024, 1024, 256, 128] + - [0, 2040.4] + - - [5888, 64, 1, 3328, 5888, 5888, 3328, 64] + - [0, 4265.53] + - - [4288, 6784, 1, 1792, 4288, 4288, 1792, 6784] + - [3, 6407.97] + - - [1856, 2944, 1, 3328, 1856, 1856, 3328, 2944] + - [3, 6029.61] + - - [704, 128, 1, 256, 704, 704, 256, 128] + - [0, 1475.92] + - - [5056, 448, 1, 1792, 5056, 5056, 1792, 448] + - [3, 5245.61] + - - [1408, 128, 1, 768, 1408, 1408, 768, 128] + - [0, 2892.44] + - - [64, 448, 1, 3328, 64, 64, 3328, 448] + - [0, 710.113] + - - [1408, 64, 1, 1792, 1408, 1408, 1792, 64] + - [0, 2130.93] + - - [256, 64, 1, 1792, 256, 256, 1792, 64] + - [0, 446.844] + - - [2368, 1408, 1, 1792, 2368, 2368, 1792, 1408] + - [3, 5668.14] + - - [1856, 4, 1, 768, 1856, 1856, 768, 4] + - [5, 139.693] + - - [5888, 1024, 1, 768, 5888, 5888, 768, 1024] + - [1, 5988.73] + - - [448, 6784, 1, 768, 448, 448, 768, 6784] + - [3, 5800.8] + - - [1856, 2944, 1, 1792, 1856, 1856, 1792, 2944] + - [3, 5985.75] + - - [1024, 5888, 1, 768, 1024, 1024, 768, 5888] + - [1, 6007.73] + - - [1856, 5888, 1, 768, 1856, 1856, 768, 5888] + - [3, 6271.8] + - - [128, 4288, 1, 768, 128, 128, 768, 4288] + - [0, 3673.65] + - - [5056, 1024, 1, 768, 5056, 5056, 768, 1024] + - [1, 6099.01] + - - [1024, 64, 1, 256, 1024, 1024, 256, 64] + - [0, 1011.1] + - - [5888, 704, 1, 1792, 5888, 5888, 1792, 704] + - [3, 5482.32] + - - [1024, 64, 1, 3328, 1024, 1024, 3328, 64] + - [0, 1620.43] + - - [6784, 2368, 1, 1792, 6784, 6784, 1792, 2368] + - [1, 6291.35] + - - [5056, 128, 1, 256, 5056, 5056, 256, 128] + - [0, 3456.78] + - - [1856, 256, 1, 768, 1856, 1856, 768, 256] + - [3, 4081.27] + - - [128, 2368, 1, 768, 128, 128, 768, 2368] + - [0, 3180.67] + - - [5056, 64, 1, 1792, 5056, 5056, 1792, 64] + - [0, 3602.37] + - - [1856, 4288, 1, 3328, 1856, 1856, 3328, 4288] + - [1, 6204.38] + - - [448, 1024, 1, 3328, 448, 448, 3328, 1024] + - [3, 4271.57] + - - [128, 5056, 1, 3328, 128, 128, 3328, 5056] + - [0, 4688.05] + - - [3584, 5888, 1, 768, 3584, 3584, 768, 5888] + - [1, 6267.51] + - - [5056, 4288, 1, 768, 5056, 5056, 768, 4288] + - [3, 6265.48] + - - [128, 64, 1, 3328, 128, 128, 3328, 64] + - [0, 223.597] + - - [4, 256, 1, 3328, 4, 4, 3328, 256] + - [5, 26.1097] + - - [256, 3584, 1, 1792, 256, 256, 1792, 3584] + - [3, 5013.71] + - - [1408, 4, 1, 1792, 1408, 1408, 1792, 4] + - [5, 110.41] + - - [704, 4288, 1, 3328, 704, 704, 3328, 4288] + - [3, 5973.01] + - - [6784, 704, 1, 768, 6784, 6784, 768, 704] + - [1, 5617.84] + - - [64, 1024, 1, 768, 64, 64, 768, 1024] + - [0, 1253.61] + - - [3584, 2944, 1, 3328, 3584, 3584, 3328, 2944] + - [3, 6164.97] + - - [704, 3584, 1, 3328, 704, 704, 3328, 3584] + - [3, 5946.11] + - - [1408, 448, 1, 768, 1408, 1408, 768, 448] + - [0, 4241.12] + - - [256, 1856, 1, 1792, 256, 256, 1792, 1856] + - [3, 4265.01] + - - [704, 448, 1, 1792, 704, 704, 1792, 448] + - [0, 3561.99] + - - [64, 2944, 1, 1792, 64, 64, 1792, 2944] + - [0, 3293.37] + - - [2368, 1024, 1, 768, 2368, 2368, 768, 1024] + - [3, 5533.8] + - - [2368, 3584, 1, 768, 2368, 2368, 768, 3584] + - [3, 6006.97] + - - [256, 5888, 1, 768, 256, 256, 768, 5888] + - [3, 5306.48] + - - [1408, 3584, 1, 256, 1408, 1408, 256, 3584] + - [3, 5629.26] + - - [704, 256, 1, 1792, 704, 704, 1792, 256] + - [0, 3111.97] + - - [1856, 3584, 1, 3328, 1856, 1856, 3328, 3584] + - [3, 6229.27] + - - [2368, 1024, 1, 256, 2368, 2368, 256, 1024] + - [3, 5128.53] + - - [1856, 448, 1, 256, 1856, 1856, 256, 448] + - [3, 3605.46] + - - [704, 256, 1, 768, 704, 704, 768, 256] + - [0, 2999.24] + - - [1408, 704, 1, 3328, 1408, 1408, 3328, 704] + - [0, 4735.03] + - - [1856, 4, 1, 256, 1856, 1856, 256, 4] + - [5, 117.155] + - - [1856, 2368, 1, 768, 1856, 1856, 768, 2368] + - [3, 5740.28] + - - [1024, 1408, 1, 1792, 1024, 1024, 1792, 1408] + - [3, 5381.73] + - - [448, 1024, 1, 1792, 448, 448, 1792, 1024] + - [3, 4158.06] + - - [2944, 2368, 1, 3328, 2944, 2944, 3328, 2368] + - [1, 6178.79] + - - [4288, 448, 1, 768, 4288, 4288, 768, 448] + - [0, 4642.59] + - - [2944, 4288, 1, 1792, 2944, 2944, 1792, 4288] + - [3, 6175.09] + - - [704, 448, 1, 3328, 704, 704, 3328, 448] + - [0, 3619.31] + - - [2944, 2368, 1, 768, 2944, 2944, 768, 2368] + - [1, 6003.62] + - - [5056, 448, 1, 768, 5056, 5056, 768, 448] + - [3, 5016.84] + - - [64, 128, 1, 256, 64, 64, 256, 128] + - [0, 151.397] + - - [5888, 2368, 1, 1792, 5888, 5888, 1792, 2368] + - [1, 6231.33] + - - [6784, 4288, 1, 3328, 6784, 6784, 3328, 4288] + - [1, 6371.21] +- null diff --git a/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Alik_Bljk_SB.yaml b/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Alik_Bljk_SB.yaml new file mode 100644 index 0000000000..c376a6732e --- /dev/null +++ b/tensilelite/Tensile/Source/lib/configs/lite_configs/navi10_Cijk_Alik_Bljk_SB.yaml @@ -0,0 +1,3407 @@ +- {MinimumRequiredVersion: 4.9.4} +- navi10 +- gfx1010 +- [NAVI] +- AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseInitialStrides: false +- - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: false + FractionalLoad: 0 + GlobalLoadVectorWidthA: 2 + GlobalLoadVectorWidthB: 2 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 8 + LSCB: 8 + LSPA: 64 + LSPB: 64 + LVCA: 4 + LVCB: 4 + LVPA: 32 + LVPB: 32 + LdcEqualsLdd: true + LdsNumElements: 1024 + LdsOffsetA: 0 + LdsOffsetB: 512 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 8 + MacroTile0: 64 + MacroTile1: 64 + MacroTileA: 64 + MacroTileB: 64 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 16 + NumGlobalWriteVectorsPerThread: 4 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: false + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 0 + SolutionNameMin: Cijk_Alik_Bljk_SB_MT64x64x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: false + ThreadTile: [4, 4] + ThreadTile0: 4 + ThreadTile1: 4 + ThreadTileA: 4 + ThreadTileB: 4 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 3 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 16 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 16 + LSCB: 16 + LSPA: 64 + LSPB: 64 + LVCA: 4 + LVCB: 4 + LVPA: 16 + LVPB: 16 + LdcEqualsLdd: true + LdsNumElements: 7168 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 2048 + LdsOffsetA: 0 + LdsOffsetA_Blk: 4096 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 5120 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 16 + MacroTile0: 64 + MacroTile1: 128 + MacroTileA: 64 + MacroTileB: 128 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 32 + NumGlobalWriteVectorsPerThread: 8 + NumLoadsA: 1 + NumLoadsB: 2 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 2 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 1 + SolutionNameMin: Cijk_Alik_Bljk_SB_MT64x128x16_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: true + ThreadTile: [4, 8] + ThreadTile0: 4 + ThreadTile1: 8 + ThreadTileA: 4 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 2 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 8 + LSCB: 8 + LSPA: 128 + LSPB: 128 + LVCA: 2 + LVCB: 2 + LVPA: 32 + LVPB: 32 + LdcEqualsLdd: true + LdsNumElements: 4096 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 1024 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 3072 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 8 + MacroTile0: 128 + MacroTile1: 128 + MacroTileA: 128 + MacroTileB: 128 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 64 + NumGlobalWriteVectorsPerThread: 16 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 2 + SolutionNameMin: Cijk_Alik_Bljk_SB_MT128x128x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: true + ThreadTile: [8, 8] + ThreadTile0: 8 + ThreadTile1: 8 + ThreadTileA: 8 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 3 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 1 + GlobalLoadVectorWidthB: 1 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 8 + LSCB: 8 + LSPA: 32 + LSPB: 32 + LVCA: 8 + LVCB: 8 + LVPA: 32 + LVPB: 32 + LdcEqualsLdd: true + LdsNumElements: 4096 + LdsNumElementsAlignedA: 256 + LdsNumElementsAlignedB: 256 + LdsOffsetA: 0 + LdsOffsetA_Blk: 512 + LdsOffsetB: 256 + LdsOffsetB_Blk: 768 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 2 + MacroTile0: 32 + MacroTile1: 32 + MacroTileA: 32 + MacroTileB: 32 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 4 + NumGlobalWriteVectorsPerThread: 1 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 3 + SolutionNameMin: Cijk_Alik_Bljk_SB_MT32x32x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: true + ThreadTile: [4, 4] + ThreadTile0: 4 + ThreadTile1: 4 + ThreadTileA: 4 + ThreadTileB: 4 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 3 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 16 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 2 + GlobalLoadVectorWidthB: 4 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 16 + LSCB: 16 + LSPA: 32 + LSPB: 64 + LVCA: 8 + LVCB: 4 + LVPA: 16 + LVPB: 16 + LdcEqualsLdd: true + LdsNumElements: 8192 + LdsNumElementsAlignedA: 512 + LdsNumElementsAlignedB: 1024 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 512 + LdsOffsetB_Blk: 2560 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 4 + MacroTile0: 32 + MacroTile1: 64 + MacroTileA: 32 + MacroTileB: 64 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 8 + NumGlobalWriteVectorsPerThread: 2 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 4 + SolutionNameMin: Cijk_Alik_Bljk_SB_MT32x64x16_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: true + ThreadTile: [4, 8] + ThreadTile0: 4 + ThreadTile1: 8 + ThreadTileA: 4 + ThreadTileB: 8 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 2 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 16 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 2 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 16 + LSCB: 16 + LSPA: 64 + LSPB: 32 + LVCA: 4 + LVCB: 8 + LVPA: 16 + LVPB: 16 + LdcEqualsLdd: true + LdsNumElements: 8192 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 512 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 3072 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 4 + MacroTile0: 64 + MacroTile1: 32 + MacroTileA: 64 + MacroTileB: 32 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 8 + NumGlobalWriteVectorsPerThread: 2 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 5 + SolutionNameMin: Cijk_Alik_Bljk_SB_MT64x32x16_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 8 + SubGroup1: 8 + SubGroupA: 8 + SubGroupB: 8 + SuppressNoLoadLoop: true + ThreadTile: [8, 4] + ThreadTile0: 8 + ThreadTile1: 4 + ThreadTileA: 8 + ThreadTileB: 4 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [8, 8, 4] + WorkGroupMapping: 8 + WorkGroupMappingType: B + _staggerStrideShift: 2 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BenchmarkFork: 0 + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 4 + GlobalLoadVectorWidthB: 2 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 4 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 4 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 8 + LSCB: 8 + LSPA: 128 + LSPB: 64 + LVCA: 2 + LVCB: 4 + LVPA: 32 + LVPB: 32 + LdcEqualsLdd: true + LdsNumElements: 3584 + LdsNumElementsAlignedA: 1024 + LdsNumElementsAlignedB: 512 + LdsOffsetA: 0 + LdsOffsetA_Blk: 2048 + LdsOffsetB: 1024 + LdsOffsetB_Blk: 3072 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 1 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 8 + MacroTile0: 128 + MacroTile1: 64 + MacroTileA: 128 + MacroTileB: 64 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 32 + NumGlobalWriteVectorsPerThread: 8 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 256 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 6 + SolutionNameMin: Cijk_Alik_Bljk_SB_MT128x64x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 16 + SubGroup1: 16 + SubGroupA: 16 + SubGroupB: 16 + SuppressNoLoadLoop: true + ThreadTile: [8, 4] + ThreadTile0: 8 + ThreadTile1: 4 + ThreadTileA: 8 + ThreadTileB: 4 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 4 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 3 + - AggressivePerfMode: 1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertMinApproxSize: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: false + AssignedProblemIndependentDerivedParameters: true + BufferLoad: true + BufferStore: true + CheckDimOverflow: 0 + CheckTensorDimAsserts: false + DepthU: 8 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DisableKernelPieces: 0 + EdgeType: ShiftPtr + ExpandPointerSwap: true + FractionalLoad: 0 + GlobalLoadVectorWidthA: 1 + GlobalLoadVectorWidthB: 1 + GlobalRead2A: true + GlobalRead2B: true + GlobalReadCoalesceGroupA: true + GlobalReadCoalesceGroupB: true + GlobalReadCoalesceVectorA: true + GlobalReadCoalesceVectorB: true + GlobalReadVectorWidth: 2 + GlobalSplitU: 1 + GlobalSplitUSummationAssignmentRoundRobin: true + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 1 + GuaranteeNoPartialA: true + GuaranteeNoPartialB: true + InnerUnroll: 1 + KernelLanguage: Assembly + LSCA: 8 + LSCB: 8 + LSPA: 8 + LSPB: 8 + LVCA: 8 + LVCB: 8 + LVPA: 8 + LVPB: 8 + LdcEqualsLdd: true + LdsNumElements: 409 + LdsNumElementsAlignedA: 64 + LdsNumElementsAlignedB: 64 + LdsOffsetA: 0 + LdsOffsetA_Blk: 128 + LdsOffsetB: 64 + LdsOffsetB_Blk: 192 + LdsPadA: 0 + LdsPadB: 0 + LocalDotLayout: 1 + LocalRead2A: true + LocalRead2B: true + LocalSplitU: 4 + LocalWrite2A: true + LocalWrite2B: true + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopDoWhile: false + LoopTail: true + LoopUnroll: 2 + MacroTile0: 8 + MacroTile1: 8 + MacroTileA: 8 + MacroTileB: 8 + MacroTileShapeMax: 64 + MacroTileShapeMin: 1 + MaxOccupancy: 40 + MinGlobalWriteVectorWidth: 1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NumElementsPerThread: 1 + NumGlobalWriteVectorsPerThread: 1 + NumLoadsA: 1 + NumLoadsB: 1 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 1 + NumThreads: 64 + PackBatchDims: 0 + PackFreeDims: 1 + PackGranularity: 2 + PackedC0Indices: [I] + PackedC1Indices: [J] + PerformanceSyncLocation: -1 + PerformanceWaitCount: -1 + PerformanceWaitLocation: -1 + PersistentKernel: 0 + PrefetchGlobalRead: true + PrefetchLocalRead: true + ProblemType: + AssignedDerivedParameters: true + Batched: true + ComplexConjugateA: false + ComplexConjugateB: false + DataType: 0 + DestDataType: 0 + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [3, 0, 2] + IndexAssignmentsB: [3, 1, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesSummation: 1 + NumIndiciesLD: 4 + OperationType: GEMM + SilentHighPrecisionAccumulate: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseInitialStrides: false + ReplacementKernel: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 1 + ScheduleLocalWrite: 1 + SolutionIndex: 7 + SolutionNameMin: Cijk_Alik_Bljk_SB_MT8x8x8_SE_ + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + SubGroup0: 4 + SubGroup1: 4 + SubGroupA: 4 + SubGroupB: 4 + SuppressNoLoadLoop: true + ThreadTile: [2, 2] + ThreadTile0: 2 + ThreadTile1: 2 + ThreadTileA: 2 + ThreadTileB: 2 + UnrollMemFence: false + UseSgprForGRO: 1 + Valid: true + VectorAtomicWidth: 1 + VectorStore: true + VectorWidth: 2 + WorkGroup: [4, 4, 4] + WorkGroupMapping: 1 + WorkGroupMappingType: B + _staggerStrideShift: 3 +- [2, 3, 0, 1] +- - - [6784, 448, 1, 256, 6784, 6784, 256, 256] + - [6, 5342.49] + - - [704, 5888, 1, 1792, 704, 704, 1792, 1792] + - [1, 5872.73] + - - [704, 448, 1, 3328, 704, 704, 3328, 3328] + - [5, 3167.09] + - - [64, 6784, 1, 1792, 64, 64, 1792, 1792] + - [4, 3264.94] + - - [1024, 1024, 1, 3328, 1024, 1024, 3328, 3328] + - [1, 5594.92] + - - [4, 1408, 1, 768, 4, 4, 768, 768] + - [7, 128.051] + - - [5056, 6784, 1, 768, 5056, 5056, 768, 768] + - [2, 6270.83] + - - [256, 1024, 1, 256, 256, 256, 256, 256] + - [5, 2970.27] + - - [704, 256, 1, 768, 704, 704, 768, 768] + - [5, 2684.64] + - - [1408, 1024, 1, 256, 1408, 1408, 256, 256] + - [6, 4880.19] + - - [3584, 5056, 1, 768, 3584, 3584, 768, 768] + - [2, 6085.47] + - - [1408, 128, 1, 3328, 1408, 1408, 3328, 3328] + - [5, 2947.54] + - - [2368, 2944, 1, 768, 2368, 2368, 768, 768] + - [6, 5954.65] + - - [3584, 1024, 1, 256, 3584, 3584, 256, 256] + - [6, 5644.53] + - - [448, 128, 1, 768, 448, 448, 768, 768] + - [5, 1300.92] + - - [256, 4, 1, 1792, 256, 256, 1792, 1792] + - [7, 27.1622] + - - [4288, 5888, 1, 256, 4288, 4288, 256, 256] + - [2, 5947.37] + - - [704, 1024, 1, 3328, 704, 704, 3328, 3328] + - [5, 4105.41] + - - [5888, 5888, 1, 768, 5888, 5888, 768, 768] + - [2, 6295.37] + - - [3584, 4, 1, 1792, 3584, 3584, 1792, 1792] + - [7, 200.005] + - - [1024, 2368, 1, 1792, 1024, 1024, 1792, 1792] + - [6, 5619.85] + - - [4288, 4, 1, 256, 4288, 4288, 256, 256] + - [7, 238.054] + - - [1856, 128, 1, 768, 1856, 1856, 768, 768] + - [5, 3488.7] + - - [4288, 5888, 1, 1792, 4288, 4288, 1792, 1792] + - [2, 6329.5] + - - [6784, 128, 1, 3328, 6784, 6784, 3328, 3328] + - [1, 4687.29] + - - [3584, 6784, 1, 1792, 3584, 3584, 1792, 1792] + - [2, 6317.04] + - - [704, 128, 1, 256, 704, 704, 256, 256] + - [4, 1428.53] + - - [448, 448, 1, 768, 448, 448, 768, 768] + - [4, 3020.12] + - - [4, 1408, 1, 1792, 4, 4, 1792, 1792] + - [7, 140.316] + - - [64, 4, 1, 3328, 64, 64, 3328, 3328] + - [7, 6.9959] + - - [1856, 5056, 1, 3328, 1856, 1856, 3328, 3328] + - [2, 5837.83] + - - [5056, 704, 1, 3328, 5056, 5056, 3328, 3328] + - [2, 5365.15] + - - [4288, 448, 1, 1792, 4288, 4288, 1792, 1792] + - [6, 5411.3] + - - [256, 6784, 1, 256, 256, 256, 256, 256] + - [1, 4235.62] + - - [704, 1024, 1, 768, 704, 704, 768, 768] + - [5, 3909.03] + - - [2944, 256, 1, 1792, 2944, 2944, 1792, 1792] + - [5, 4245.96] + - - [2944, 256, 1, 3328, 2944, 2944, 3328, 3328] + - [5, 4292.02] + - - [6784, 64, 1, 1792, 6784, 6784, 1792, 1792] + - [4, 3489.47] + - - [64, 64, 1, 768, 64, 64, 768, 768] + - [4, 87.9211] + - - [4288, 2944, 1, 768, 4288, 4288, 768, 768] + - [2, 5934.3] + - - [2368, 64, 1, 1792, 2368, 2368, 1792, 1792] + - [5, 2458.92] + - - [3584, 704, 1, 1792, 3584, 3584, 1792, 1792] + - [6, 5642.28] + - - [1408, 5056, 1, 3328, 1408, 1408, 3328, 3328] + - [2, 6256.77] + - - [1856, 1856, 1, 256, 1856, 1856, 256, 256] + - [6, 5377.0] + - - [1024, 4288, 1, 1792, 1024, 1024, 1792, 1792] + - [6, 5616.06] + - - [2944, 1408, 1, 3328, 2944, 2944, 3328, 3328] + - [2, 6244.25] + - - [448, 704, 1, 256, 448, 448, 256, 256] + - [5, 2691.3] + - - [3584, 4288, 1, 1792, 3584, 3584, 1792, 1792] + - [2, 6339.57] + - - [3584, 256, 1, 3328, 3584, 3584, 3328, 3328] + - [1, 4938.96] + - - [6784, 256, 1, 768, 6784, 6784, 768, 768] + - [6, 4876.95] + - - [4288, 1408, 1, 768, 4288, 4288, 768, 768] + - [2, 5935.45] + - - [2368, 6784, 1, 256, 2368, 2368, 256, 256] + - [6, 5879.23] + - - [448, 704, 1, 1792, 448, 448, 1792, 1792] + - [4, 3135.95] + - - [6784, 4, 1, 768, 6784, 6784, 768, 768] + - [7, 305.803] + - - [4288, 2368, 1, 256, 4288, 4288, 256, 256] + - [6, 5811.73] + - - [5056, 5888, 1, 256, 5056, 5056, 256, 256] + - [2, 6062.91] + - - [128, 1408, 1, 256, 128, 128, 256, 256] + - [5, 2133.03] + - - [2944, 6784, 1, 3328, 2944, 2944, 3328, 3328] + - [2, 6294.07] + - - [1024, 5888, 1, 768, 1024, 1024, 768, 768] + - [2, 5904.44] + - - [2944, 704, 1, 1792, 2944, 2944, 1792, 1792] + - [6, 5807.1] + - - [3584, 5056, 1, 256, 3584, 3584, 256, 256] + - [6, 5882.91] + - - [64, 256, 1, 768, 64, 64, 768, 768] + - [4, 377.48] + - - [3584, 128, 1, 768, 3584, 3584, 768, 768] + - [1, 3895.36] + - - [2944, 5056, 1, 256, 2944, 2944, 256, 256] + - [6, 5915.45] + - - [5056, 64, 1, 1792, 5056, 5056, 1792, 1792] + - [4, 3090.38] + - - [5888, 6784, 1, 3328, 5888, 5888, 3328, 3328] + - [2, 6474.61] + - - [1856, 3584, 1, 3328, 1856, 1856, 3328, 3328] + - [1, 5903.89] + - - [128, 128, 1, 256, 128, 128, 256, 256] + - [5, 287.419] + - - [5888, 5056, 1, 768, 5888, 5888, 768, 768] + - [2, 6273.39] + - - [1408, 4288, 1, 256, 1408, 1408, 256, 256] + - [6, 5716.46] + - - [4288, 256, 1, 256, 4288, 4288, 256, 256] + - [6, 3704.73] + - - [448, 1408, 1, 256, 448, 448, 256, 256] + - [4, 3400.81] + - - [5888, 1408, 1, 256, 5888, 5888, 256, 256] + - [2, 5757.75] + - - [1856, 704, 1, 1792, 1856, 1856, 1792, 1792] + - [1, 4704.43] + - - [5056, 256, 1, 256, 5056, 5056, 256, 256] + - [6, 4359.75] + - - [5056, 256, 1, 768, 5056, 5056, 768, 768] + - [6, 4760.32] + - - [2368, 64, 1, 3328, 2368, 2368, 3328, 3328] + - [5, 2469.62] + - - [448, 2368, 1, 3328, 448, 448, 3328, 3328] + - [6, 4054.21] + - - [2944, 4288, 1, 3328, 2944, 2944, 3328, 3328] + - [2, 6065.46] + - - [704, 1024, 1, 1792, 704, 704, 1792, 1792] + - [5, 4089.93] + - - [1856, 4, 1, 3328, 1856, 1856, 3328, 3328] + - [7, 194.369] + - - [128, 3584, 1, 768, 128, 128, 768, 768] + - [1, 3739.17] + - - [4288, 1856, 1, 3328, 4288, 4288, 3328, 3328] + - [2, 6145.93] + - - [448, 6784, 1, 256, 448, 448, 256, 256] + - [1, 5272.74] + - - [5056, 1024, 1, 768, 5056, 5056, 768, 768] + - [6, 6133.85] + - - [6784, 704, 1, 256, 6784, 6784, 256, 256] + - [6, 5236.42] + - - [5888, 128, 1, 3328, 5888, 5888, 3328, 3328] + - [5, 4209.31] + - - [1856, 256, 1, 256, 1856, 1856, 256, 256] + - [1, 3303.9] + - - [5056, 2944, 1, 768, 5056, 5056, 768, 768] + - [2, 6074.96] + - - [704, 3584, 1, 1792, 704, 704, 1792, 1792] + - [1, 5631.34] + - - [2368, 1856, 1, 768, 2368, 2368, 768, 768] + - [6, 5718.66] + - - [4, 64, 1, 256, 4, 4, 256, 256] + - [7, 5.23492] + - - [5888, 448, 1, 1792, 5888, 5888, 1792, 1792] + - [6, 5185.06] + - - [1408, 704, 1, 3328, 1408, 1408, 3328, 3328] + - [6, 5328.18] + - - [2944, 3584, 1, 3328, 2944, 2944, 3328, 3328] + - [2, 5986.14] + - - [704, 64, 1, 256, 704, 704, 256, 256] + - [5, 794.458] + - - [704, 2368, 1, 3328, 704, 704, 3328, 3328] + - [2, 4815.55] + - - [2368, 1024, 1, 1792, 2368, 2368, 1792, 1792] + - [6, 5649.62] + - - [4, 64, 1, 1792, 4, 4, 1792, 1792] + - [7, 6.92728] + - - [128, 3584, 1, 1792, 128, 128, 1792, 1792] + - [1, 4098.17] + - - [4, 1024, 1, 1792, 4, 4, 1792, 1792] + - [7, 106.317] + - - [2368, 5056, 1, 1792, 2368, 2368, 1792, 1792] + - [2, 6158.75] + - - [2368, 4288, 1, 768, 2368, 2368, 768, 768] + - [1, 5801.49] + - - [5056, 3584, 1, 3328, 5056, 5056, 3328, 3328] + - [2, 6326.0] + - - [448, 704, 1, 768, 448, 448, 768, 768] + - [5, 3076.12] + - - [6784, 2944, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 6290.24] + - - [6784, 5888, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 6483.07] + - - [2368, 1856, 1, 1792, 2368, 2368, 1792, 1792] + - [1, 5558.7] + - - [448, 6784, 1, 768, 448, 448, 768, 768] + - [1, 5548.5] + - - [256, 3584, 1, 3328, 256, 256, 3328, 3328] + - [1, 4946.07] + - - [5056, 704, 1, 1792, 5056, 5056, 1792, 1792] + - [6, 5646.76] + - - [704, 5888, 1, 768, 704, 704, 768, 768] + - [1, 5784.87] + - - [1408, 1856, 1, 768, 1408, 1408, 768, 768] + - [6, 5132.3] + - - [704, 1856, 1, 1792, 704, 704, 1792, 1792] + - [6, 4830.97] + - - [1024, 1856, 1, 256, 1024, 1024, 256, 256] + - [2, 4628.7] + - - [448, 4288, 1, 1792, 448, 448, 1792, 1792] + - [1, 5289.95] + - - [2368, 1024, 1, 256, 2368, 2368, 256, 256] + - [6, 5185.64] + - - [4, 2368, 1, 768, 4, 4, 768, 768] + - [7, 184.939] + - - [2944, 1408, 1, 768, 2944, 2944, 768, 768] + - [2, 5998.02] + - - [1024, 4, 1, 3328, 1024, 1024, 3328, 3328] + - [7, 112.276] + - - [704, 5056, 1, 3328, 704, 704, 3328, 3328] + - [1, 5783.08] + - - [1856, 1024, 1, 3328, 1856, 1856, 3328, 3328] + - [2, 5500.93] + - - [448, 5888, 1, 3328, 448, 448, 3328, 3328] + - [2, 5198.68] + - - [128, 128, 1, 768, 128, 128, 768, 768] + - [5, 382.576] + - - [4288, 2944, 1, 256, 4288, 4288, 256, 256] + - [6, 5866.21] + - - [256, 2368, 1, 1792, 256, 256, 1792, 1792] + - [4, 3840.01] + - - [5888, 1856, 1, 3328, 5888, 5888, 3328, 3328] + - [2, 6177.23] + - - [704, 5056, 1, 768, 704, 704, 768, 768] + - [1, 5619.48] + - - [4, 3584, 1, 768, 4, 4, 768, 768] + - [7, 239.729] + - - [704, 256, 1, 256, 704, 704, 256, 256] + - [5, 2133.03] + - - [5888, 704, 1, 1792, 5888, 5888, 1792, 1792] + - [6, 5691.73] + - - [448, 64, 1, 1792, 448, 448, 1792, 1792] + - [4, 583.359] + - - [4, 704, 1, 3328, 4, 4, 3328, 3328] + - [7, 76.9552] + - - [3584, 2368, 1, 256, 3584, 3584, 256, 256] + - [6, 5615.55] + - - [6784, 64, 1, 256, 6784, 6784, 256, 256] + - [6, 2896.66] + - - [1856, 6784, 1, 256, 1856, 1856, 256, 256] + - [6, 5869.92] + - - [3584, 704, 1, 3328, 3584, 3584, 3328, 3328] + - [2, 4978.98] + - - [5888, 2368, 1, 256, 5888, 5888, 256, 256] + - [6, 5743.15] + - - [1856, 5056, 1, 256, 1856, 1856, 256, 256] + - [6, 5743.9] + - - [6784, 1408, 1, 768, 6784, 6784, 768, 768] + - [6, 5869.91] + - - [4, 6784, 1, 768, 4, 4, 768, 768] + - [7, 244.43] + - - [2368, 4, 1, 3328, 2368, 2368, 3328, 3328] + - [7, 201.109] + - - [128, 3584, 1, 256, 128, 128, 256, 256] + - [4, 2903.67] + - - [2368, 1024, 1, 768, 2368, 2368, 768, 768] + - [1, 5329.66] + - - [1024, 256, 1, 3328, 1024, 1024, 3328, 3328] + - [5, 4260.96] + - - [3584, 3584, 1, 3328, 3584, 3584, 3328, 3328] + - [2, 6162.77] + - - [704, 5056, 1, 1792, 704, 704, 1792, 1792] + - [1, 5718.38] + - - [704, 6784, 1, 768, 704, 704, 768, 768] + - [2, 5591.83] + - - [6784, 2944, 1, 768, 6784, 6784, 768, 768] + - [2, 6177.55] + - - [1408, 128, 1, 768, 1408, 1408, 768, 768] + - [5, 2676.96] + - - [6784, 2944, 1, 256, 6784, 6784, 256, 256] + - [2, 5931.75] + - - [2944, 5056, 1, 3328, 2944, 2944, 3328, 3328] + - [2, 6213.81] + - - [64, 4, 1, 1792, 64, 64, 1792, 1792] + - [7, 6.8582] + - - [128, 2368, 1, 768, 128, 128, 768, 768] + - [4, 2734.98] + - - [64, 704, 1, 256, 64, 64, 256, 256] + - [5, 759.537] + - - [704, 5888, 1, 3328, 704, 704, 3328, 3328] + - [1, 5919.54] + - - [1408, 4288, 1, 3328, 1408, 1408, 3328, 3328] + - [2, 6159.92] + - - [1024, 128, 1, 1792, 1024, 1024, 1792, 1792] + - [5, 2644.55] + - - [5888, 64, 1, 3328, 5888, 5888, 3328, 3328] + - [5, 3552.96] + - - [3584, 1408, 1, 768, 3584, 3584, 768, 768] + - [6, 5989.51] + - - [4, 1024, 1, 768, 4, 4, 768, 768] + - [7, 96.514] + - - [6784, 1024, 1, 256, 6784, 6784, 256, 256] + - [2, 5515.18] + - - [5056, 2368, 1, 256, 5056, 5056, 256, 256] + - [6, 5740.33] + - - [1024, 64, 1, 256, 1024, 1024, 256, 256] + - [4, 1038.93] + - - [448, 704, 1, 3328, 448, 448, 3328, 3328] + - [5, 3204.33] + - - [2944, 448, 1, 1792, 2944, 2944, 1792, 1792] + - [6, 4951.32] + - - [1856, 64, 1, 256, 1856, 1856, 256, 256] + - [4, 1849.12] + - - [1024, 1408, 1, 768, 1024, 1024, 768, 768] + - [6, 5101.75] + - - [1856, 1024, 1, 768, 1856, 1856, 768, 768] + - [6, 5351.54] + - - [448, 1856, 1, 768, 448, 448, 768, 768] + - [1, 4227.88] + - - [704, 448, 1, 1792, 704, 704, 1792, 1792] + - [4, 3143.7] + - - [448, 2944, 1, 768, 448, 448, 768, 768] + - [6, 4743.02] + - - [128, 704, 1, 3328, 128, 128, 3328, 3328] + - [5, 1684.04] + - - [5888, 1408, 1, 1792, 5888, 5888, 1792, 1792] + - [2, 6338.62] + - - [6784, 1856, 1, 256, 6784, 6784, 256, 256] + - [6, 5637.22] + - - [2368, 2368, 1, 768, 2368, 2368, 768, 768] + - [1, 5741.29] + - - [2368, 128, 1, 256, 2368, 2368, 256, 256] + - [5, 2413.59] + - - [3584, 6784, 1, 3328, 3584, 3584, 3328, 3328] + - [2, 6369.06] + - - [2368, 3584, 1, 256, 2368, 2368, 256, 256] + - [6, 5789.58] + - - [64, 3584, 1, 1792, 64, 64, 1792, 1792] + - [4, 3310.81] + - - [704, 2368, 1, 256, 704, 704, 256, 256] + - [6, 4208.35] + - - [4288, 4288, 1, 3328, 4288, 4288, 3328, 3328] + - [2, 6086.38] + - - [3584, 1856, 1, 256, 3584, 3584, 256, 256] + - [6, 5690.19] + - - [128, 128, 1, 3328, 128, 128, 3328, 3328] + - [4, 341.094] + - - [1856, 128, 1, 256, 1856, 1856, 256, 256] + - [4, 2755.04] + - - [3584, 448, 1, 256, 3584, 3584, 256, 256] + - [6, 4229.35] + - - [2944, 1856, 1, 3328, 2944, 2944, 3328, 3328] + - [1, 5707.45] + - - [1024, 5056, 1, 256, 1024, 1024, 256, 256] + - [6, 5561.84] + - - [448, 64, 1, 256, 448, 448, 256, 256] + - [4, 516.068] + - - [2368, 5888, 1, 768, 2368, 2368, 768, 768] + - [2, 6067.37] + - - [1856, 2368, 1, 1792, 1856, 1856, 1792, 1792] + - [1, 5556.38] + - - [5888, 4, 1, 1792, 5888, 5888, 1792, 1792] + - [7, 167.329] + - - [6784, 448, 1, 768, 6784, 6784, 768, 768] + - [6, 5810.43] + - - [64, 1408, 1, 3328, 64, 64, 3328, 3328] + - [4, 1727.89] + - - [5888, 2944, 1, 3328, 5888, 5888, 3328, 3328] + - [2, 6392.14] + - - [448, 2944, 1, 3328, 448, 448, 3328, 3328] + - [6, 4978.12] + - - [64, 1024, 1, 3328, 64, 64, 3328, 3328] + - [4, 1285.73] + - - [448, 1408, 1, 3328, 448, 448, 3328, 3328] + - [5, 4079.34] + - - [704, 128, 1, 3328, 704, 704, 3328, 3328] + - [5, 2010.15] + - - [2944, 6784, 1, 768, 2944, 2944, 768, 768] + - [2, 6166.33] + - - [128, 2368, 1, 1792, 128, 128, 1792, 1792] + - [4, 2901.66] + - - [3584, 6784, 1, 256, 3584, 3584, 256, 256] + - [6, 5904.47] + - - [2944, 1024, 1, 1792, 2944, 2944, 1792, 1792] + - [2, 5897.52] + - - [5888, 2368, 1, 3328, 5888, 5888, 3328, 3328] + - [2, 6222.12] + - - [4288, 2368, 1, 3328, 4288, 4288, 3328, 3328] + - [1, 5886.96] + - - [448, 256, 1, 1792, 448, 448, 1792, 1792] + - [5, 2389.71] + - - [256, 128, 1, 768, 256, 256, 768, 768] + - [5, 749.954] + - - [2944, 2368, 1, 256, 2944, 2944, 256, 256] + - [6, 5668.82] + - - [128, 64, 1, 768, 128, 128, 768, 768] + - [5, 193.467] + - - [64, 704, 1, 768, 64, 64, 768, 768] + - [4, 961.168] + - - [4288, 704, 1, 256, 4288, 4288, 256, 256] + - [6, 5462.07] + - - [448, 3584, 1, 256, 448, 448, 256, 256] + - [1, 4107.28] + - - [128, 6784, 1, 1792, 128, 128, 1792, 1792] + - [1, 4565.67] + - - [1024, 1408, 1, 1792, 1024, 1024, 1792, 1792] + - [6, 5186.46] + - - [64, 64, 1, 3328, 64, 64, 3328, 3328] + - [4, 106.802] + - - [2368, 256, 1, 3328, 2368, 2368, 3328, 3328] + - [5, 4010.46] + - - [3584, 3584, 1, 256, 3584, 3584, 256, 256] + - [6, 5810.39] + - - [1408, 5888, 1, 768, 1408, 1408, 768, 768] + - [2, 6201.42] + - - [1024, 4, 1, 768, 1024, 1024, 768, 768] + - [7, 95.2154] + - - [256, 448, 1, 768, 256, 256, 768, 768] + - [5, 2461.82] + - - [2368, 5056, 1, 256, 2368, 2368, 256, 256] + - [6, 5827.64] + - - [448, 4, 1, 1792, 448, 448, 1792, 1792] + - [7, 50.3495] + - - [2368, 4288, 1, 256, 2368, 2368, 256, 256] + - [6, 5833.96] + - - [1408, 1024, 1, 3328, 1408, 1408, 3328, 3328] + - [1, 5259.01] + - - [448, 2944, 1, 1792, 448, 448, 1792, 1792] + - [6, 4935.23] + - - [4, 6784, 1, 1792, 4, 4, 1792, 1792] + - [7, 168.886] + - - [128, 704, 1, 256, 128, 128, 256, 256] + - [4, 1384.09] + - - [256, 128, 1, 1792, 256, 256, 1792, 1792] + - [5, 748.539] + - - [4, 704, 1, 256, 4, 4, 256, 256] + - [7, 48.9041] + - - [448, 6784, 1, 1792, 448, 448, 1792, 1792] + - [1, 5691.09] + - - [4288, 5888, 1, 3328, 4288, 4288, 3328, 3328] + - [2, 6367.76] + - - [1408, 3584, 1, 256, 1408, 1408, 256, 256] + - [6, 5680.61] + - - [704, 1856, 1, 768, 704, 704, 768, 768] + - [6, 4807.21] + - - [64, 128, 1, 768, 64, 64, 768, 768] + - [4, 190.005] + - - [128, 4, 1, 3328, 128, 128, 3328, 3328] + - [7, 13.9494] + - - [128, 1856, 1, 3328, 128, 128, 3328, 3328] + - [4, 3795.59] + - - [1408, 1408, 1, 3328, 1408, 1408, 3328, 3328] + - [2, 5742.85] + - - [4288, 2944, 1, 3328, 4288, 4288, 3328, 3328] + - [2, 6074.23] + - - [4, 4288, 1, 256, 4, 4, 256, 256] + - [7, 174.853] + - - [6784, 3584, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 6353.41] + - - [3584, 448, 1, 3328, 3584, 3584, 3328, 3328] + - [2, 4620.55] + - - [128, 2944, 1, 768, 128, 128, 768, 768] + - [4, 3291.39] + - - [1408, 448, 1, 1792, 1408, 1408, 1792, 1792] + - [5, 4108.65] + - - [448, 5056, 1, 256, 448, 448, 256, 256] + - [1, 4575.07] + - - [5888, 5056, 1, 256, 5888, 5888, 256, 256] + - [2, 5913.07] + - - [5056, 5056, 1, 3328, 5056, 5056, 3328, 3328] + - [2, 6437.04] + - - [2944, 704, 1, 768, 2944, 2944, 768, 768] + - [6, 5863.01] + - - [4288, 128, 1, 1792, 4288, 4288, 1792, 1792] + - [5, 3584.71] + - - [5888, 2944, 1, 768, 5888, 5888, 768, 768] + - [2, 6214.28] + - - [4288, 4, 1, 768, 4288, 4288, 768, 768] + - [7, 280.928] + - - [5056, 448, 1, 1792, 5056, 5056, 1792, 1792] + - [6, 5245.1] + - - [448, 64, 1, 768, 448, 448, 768, 768] + - [4, 669.507] + - - [128, 704, 1, 768, 128, 128, 768, 768] + - [4, 1793.18] + - - [2944, 64, 1, 3328, 2944, 2944, 3328, 3328] + - [5, 3068.12] + - - [5056, 1856, 1, 256, 5056, 5056, 256, 256] + - [6, 5662.66] + - - [5056, 704, 1, 256, 5056, 5056, 256, 256] + - [6, 5558.53] + - - [256, 4, 1, 3328, 256, 256, 3328, 3328] + - [7, 28.3108] + - - [448, 1408, 1, 768, 448, 448, 768, 768] + - [5, 3937.27] + - - [3584, 2368, 1, 768, 3584, 3584, 768, 768] + - [6, 6019.3] + - - [704, 448, 1, 768, 704, 704, 768, 768] + - [5, 3010.96] + - - [1024, 704, 1, 3328, 1024, 1024, 3328, 3328] + - [5, 4068.79] + - - [5056, 3584, 1, 1792, 5056, 5056, 1792, 1792] + - [2, 6283.27] + - - [704, 3584, 1, 256, 704, 704, 256, 256] + - [1, 5177.99] + - - [4, 1408, 1, 3328, 4, 4, 3328, 3328] + - [7, 146.259] + - - [2368, 1856, 1, 3328, 2368, 2368, 3328, 3328] + - [1, 5585.84] + - - [2944, 1408, 1, 1792, 2944, 2944, 1792, 1792] + - [2, 6171.27] + - - [1024, 64, 1, 1792, 1024, 1024, 1792, 1792] + - [4, 1295.27] + - - [1024, 6784, 1, 768, 1024, 1024, 768, 768] + - [2, 5876.53] + - - [5888, 256, 1, 768, 5888, 5888, 768, 768] + - [6, 5462.27] + - - [704, 1024, 1, 256, 704, 704, 256, 256] + - [1, 3408.15] + - - [1024, 1408, 1, 3328, 1024, 1024, 3328, 3328] + - [6, 5420.06] + - - [704, 1408, 1, 768, 704, 704, 768, 768] + - [1, 5123.04] + - - [448, 1024, 1, 768, 448, 448, 768, 768] + - [1, 3869.99] + - - [5056, 2368, 1, 1792, 5056, 5056, 1792, 1792] + - [2, 6175.43] + - - [1856, 5888, 1, 3328, 1856, 1856, 3328, 3328] + - [2, 6191.25] + - - [1024, 64, 1, 768, 1024, 1024, 768, 768] + - [5, 1260.59] + - - [1024, 2944, 1, 3328, 1024, 1024, 3328, 3328] + - [2, 5960.96] + - - [128, 4, 1, 256, 128, 128, 256, 256] + - [7, 11.7186] + - - [1024, 256, 1, 1792, 1024, 1024, 1792, 1792] + - [4, 4107.26] + - - [5888, 1024, 1, 1792, 5888, 5888, 1792, 1792] + - [2, 6101.8] + - - [2944, 4288, 1, 768, 2944, 2944, 768, 768] + - [6, 5971.39] + - - [2368, 128, 1, 1792, 2368, 2368, 1792, 1792] + - [5, 3016.24] + - - [128, 5888, 1, 1792, 128, 128, 1792, 1792] + - [4, 4076.05] + - - [3584, 1024, 1, 1792, 3584, 3584, 1792, 1792] + - [1, 5912.54] + - - [1024, 1024, 1, 1792, 1024, 1024, 1792, 1792] + - [1, 5469.27] + - - [2368, 2944, 1, 1792, 2368, 2368, 1792, 1792] + - [2, 6084.63] + - - [1408, 5888, 1, 256, 1408, 1408, 256, 256] + - [6, 5917.1] + - - [4, 128, 1, 768, 4, 4, 768, 768] + - [7, 15.7051] + - - [1856, 128, 1, 1792, 1856, 1856, 1792, 1792] + - [5, 3793.47] + - - [448, 448, 1, 3328, 448, 448, 3328, 3328] + - [5, 3292.07] + - - [5056, 5888, 1, 1792, 5056, 5056, 1792, 1792] + - [2, 6415.66] + - - [704, 704, 1, 256, 704, 704, 256, 256] + - [5, 3300.2] + - - [4288, 2368, 1, 1792, 4288, 4288, 1792, 1792] + - [1, 5852.31] + - - [128, 448, 1, 256, 128, 128, 256, 256] + - [5, 939.224] + - - [2944, 5056, 1, 1792, 2944, 2944, 1792, 1792] + - [2, 6162.15] + - - [128, 5888, 1, 768, 128, 128, 768, 768] + - [4, 3837.82] + - - [1408, 6784, 1, 1792, 1408, 1408, 1792, 1792] + - [2, 5911.15] + - - [64, 6784, 1, 768, 64, 64, 768, 768] + - [5, 2902.27] + - - [4, 256, 1, 768, 4, 4, 768, 768] + - [7, 30.7723] + - - [2944, 5888, 1, 1792, 2944, 2944, 1792, 1792] + - [2, 6353.35] + - - [3584, 4, 1, 3328, 3584, 3584, 3328, 3328] + - [7, 170.705] + - - [4288, 2944, 1, 1792, 4288, 4288, 1792, 1792] + - [2, 6027.13] + - - [1408, 2944, 1, 256, 1408, 1408, 256, 256] + - [6, 5732.4] + - - [704, 448, 1, 256, 704, 704, 256, 256] + - [5, 2678.04] + - - [1024, 448, 1, 3328, 1024, 1024, 3328, 3328] + - [1, 4188.23] + - - [2944, 6784, 1, 256, 2944, 2944, 256, 256] + - [6, 5999.28] + - - [256, 64, 1, 256, 256, 256, 256, 256] + - [4, 285.969] + - - [4288, 256, 1, 768, 4288, 4288, 768, 768] + - [6, 4037.23] + - - [2944, 2944, 1, 3328, 2944, 2944, 3328, 3328] + - [2, 5978.7] + - - [1856, 5056, 1, 1792, 1856, 1856, 1792, 1792] + - [2, 5795.51] + - - [1408, 4, 1, 256, 1408, 1408, 256, 256] + - [7, 97.8083] + - - [1408, 3584, 1, 768, 1408, 1408, 768, 768] + - [6, 5950.0] + - - [1024, 3584, 1, 3328, 1024, 1024, 3328, 3328] + - [1, 5935.28] + - - [1856, 2944, 1, 256, 1856, 1856, 256, 256] + - [6, 5611.69] + - - [2944, 256, 1, 256, 2944, 2944, 256, 256] + - [1, 3597.51] + - - [128, 5056, 1, 3328, 128, 128, 3328, 3328] + - [5, 4046.09] + - - [1024, 704, 1, 768, 1024, 1024, 768, 768] + - [5, 3759.59] + - - [2944, 448, 1, 768, 2944, 2944, 768, 768] + - [6, 4861.05] + - - [64, 6784, 1, 256, 64, 64, 256, 256] + - [4, 2143.54] + - - [5056, 4, 1, 3328, 5056, 5056, 3328, 3328] + - [7, 165.17] + - - [5056, 1856, 1, 3328, 5056, 5056, 3328, 3328] + - [2, 5839.53] + - - [5888, 2368, 1, 768, 5888, 5888, 768, 768] + - [2, 6028.4] + - - [128, 256, 1, 256, 128, 128, 256, 256] + - [4, 569.066] + - - [1856, 6784, 1, 3328, 1856, 1856, 3328, 3328] + - [2, 6057.68] + - - [2944, 1856, 1, 256, 2944, 2944, 256, 256] + - [6, 5581.83] + - - [4288, 6784, 1, 256, 4288, 4288, 256, 256] + - [6, 5986.53] + - - [2368, 704, 1, 256, 2368, 2368, 256, 256] + - [6, 4387.84] + - - [2944, 448, 1, 3328, 2944, 2944, 3328, 3328] + - [1, 4802.99] + - - [4288, 4288, 1, 1792, 4288, 4288, 1792, 1792] + - [2, 6045.52] + - - [1856, 448, 1, 768, 1856, 1856, 768, 768] + - [6, 4257.11] + - - [2944, 2944, 1, 1792, 2944, 2944, 1792, 1792] + - [2, 5921.37] + - - [1024, 5888, 1, 3328, 1024, 1024, 3328, 3328] + - [2, 6134.84] + - - [5056, 1024, 1, 256, 5056, 5056, 256, 256] + - [6, 5710.95] + - - [1024, 128, 1, 256, 1024, 1024, 256, 256] + - [4, 2059.0] + - - [2944, 1024, 1, 3328, 2944, 2944, 3328, 3328] + - [2, 5960.7] + - - [128, 1408, 1, 768, 128, 128, 768, 768] + - [5, 2624.32] + - - [256, 4288, 1, 768, 256, 256, 768, 768] + - [5, 3966.87] + - - [1408, 704, 1, 768, 1408, 1408, 768, 768] + - [6, 5107.78] + - - [6784, 64, 1, 3328, 6784, 6784, 3328, 3328] + - [5, 3615.6] + - - [2368, 4, 1, 768, 2368, 2368, 768, 768] + - [7, 188.127] + - - [1856, 2368, 1, 3328, 1856, 1856, 3328, 3328] + - [1, 5589.32] + - - [1408, 4, 1, 768, 1408, 1408, 768, 768] + - [7, 132.406] + - - [6784, 2368, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 6263.04] + - - [6784, 5056, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 6333.38] + - - [704, 4, 1, 768, 704, 704, 768, 768] + - [7, 66.5043] + - - [64, 3584, 1, 768, 64, 64, 768, 768] + - [4, 2758.83] + - - [4288, 704, 1, 3328, 4288, 4288, 3328, 3328] + - [1, 4901.8] + - - [128, 2944, 1, 1792, 128, 128, 1792, 1792] + - [4, 3526.53] + - - [3584, 1024, 1, 3328, 3584, 3584, 3328, 3328] + - [1, 5952.0] + - - [448, 256, 1, 768, 448, 448, 768, 768] + - [5, 2456.74] + - - [1024, 2944, 1, 768, 1024, 1024, 768, 768] + - [2, 5709.75] + - - [4, 1856, 1, 256, 4, 4, 256, 256] + - [7, 113.023] + - - [704, 2944, 1, 3328, 704, 704, 3328, 3328] + - [1, 5785.29] + - - [64, 704, 1, 1792, 64, 64, 1792, 1792] + - [4, 879.007] + - - [1856, 64, 1, 1792, 1856, 1856, 1792, 1792] + - [4, 2326.76] + - - [64, 704, 1, 3328, 64, 64, 3328, 3328] + - [4, 896.071] + - - [6784, 1408, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 5948.07] + - - [4, 64, 1, 3328, 4, 4, 3328, 3328] + - [7, 7.13924] + - - [3584, 1408, 1, 256, 3584, 3584, 256, 256] + - [6, 5557.51] + - - [6784, 2368, 1, 768, 6784, 6784, 768, 768] + - [2, 6105.97] + - - [1024, 1024, 1, 256, 1024, 1024, 256, 256] + - [6, 4616.3] + - - [6784, 5888, 1, 256, 6784, 6784, 256, 256] + - [2, 6122.21] + - - [2368, 448, 1, 768, 2368, 2368, 768, 768] + - [6, 3922.51] + - - [1856, 3584, 1, 768, 1856, 1856, 768, 768] + - [1, 5822.79] + - - [64, 4, 1, 256, 64, 64, 256, 256] + - [7, 4.86118] + - - [2944, 3584, 1, 256, 2944, 2944, 256, 256] + - [6, 5789.83] + - - [448, 1856, 1, 256, 448, 448, 256, 256] + - [1, 3674.61] + - - [5888, 4288, 1, 3328, 5888, 5888, 3328, 3328] + - [2, 6352.14] + - - [4288, 6784, 1, 1792, 4288, 4288, 1792, 1792] + - [2, 6305.7] + - - [128, 448, 1, 1792, 128, 128, 1792, 1792] + - [5, 1141.76] + - - [5888, 3584, 1, 1792, 5888, 5888, 1792, 1792] + - [2, 6281.23] + - - [64, 1408, 1, 256, 64, 64, 256, 256] + - [4, 1348.14] + - - [256, 704, 1, 1792, 256, 256, 1792, 1792] + - [4, 2840.31] + - - [5056, 1408, 1, 256, 5056, 5056, 256, 256] + - [6, 5737.44] + - - [2368, 704, 1, 1792, 2368, 2368, 1792, 1792] + - [6, 4797.05] + - - [5056, 5888, 1, 3328, 5056, 5056, 3328, 3328] + - [2, 6478.15] + - - [704, 704, 1, 3328, 704, 704, 3328, 3328] + - [5, 4172.09] + - - [1408, 64, 1, 1792, 1408, 1408, 1792, 1792] + - [5, 1831.88] + - - [1408, 1408, 1, 256, 1408, 1408, 256, 256] + - [6, 5174.62] + - - [64, 1408, 1, 1792, 64, 64, 1792, 1792] + - [4, 1640.28] + - - [2944, 1856, 1, 768, 2944, 2944, 768, 768] + - [6, 5912.74] + - - [4288, 5056, 1, 1792, 4288, 4288, 1792, 1792] + - [2, 6167.07] + - - [4288, 1856, 1, 1792, 4288, 4288, 1792, 1792] + - [2, 6078.99] + - - [1408, 5888, 1, 3328, 1408, 1408, 3328, 3328] + - [2, 6374.74] + - - [2368, 4, 1, 256, 2368, 2368, 256, 256] + - [7, 139.294] + - - [128, 3584, 1, 3328, 128, 128, 3328, 3328] + - [1, 4204.46] + - - [6784, 1856, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 6051.92] + - - [1856, 704, 1, 256, 1856, 1856, 256, 256] + - [6, 4379.79] + - - [256, 1856, 1, 3328, 256, 256, 3328, 3328] + - [1, 4365.31] + - - [2944, 1024, 1, 256, 2944, 2944, 256, 256] + - [6, 5291.21] + - - [5888, 3584, 1, 768, 5888, 5888, 768, 768] + - [2, 6192.47] + - - [448, 3584, 1, 1792, 448, 448, 1792, 1792] + - [6, 4619.15] + - - [2368, 704, 1, 768, 2368, 2368, 768, 768] + - [6, 4736.58] + - - [4, 128, 1, 1792, 4, 4, 1792, 1792] + - [7, 14.2532] + - - [3584, 5888, 1, 3328, 3584, 3584, 3328, 3328] + - [2, 6310.93] + - - [6784, 1856, 1, 768, 6784, 6784, 768, 768] + - [6, 5928.57] + - - [3584, 448, 1, 1792, 3584, 3584, 1792, 1792] + - [6, 4588.6] + - - [5888, 3584, 1, 3328, 5888, 5888, 3328, 3328] + - [2, 6321.79] + - - [64, 1408, 1, 768, 64, 64, 768, 768] + - [4, 1428.53] + - - [64, 1856, 1, 256, 64, 64, 256, 256] + - [5, 1584.98] + - - [1024, 448, 1, 768, 1024, 1024, 768, 768] + - [1, 3686.99] + - - [1856, 4288, 1, 3328, 1856, 1856, 3328, 3328] + - [2, 6153.65] + - - [5056, 4, 1, 1792, 5056, 5056, 1792, 1792] + - [7, 167.493] + - - [1024, 6784, 1, 256, 1024, 1024, 256, 256] + - [2, 5437.73] + - - [1408, 256, 1, 1792, 1408, 1408, 1792, 1792] + - [5, 3556.17] + - - [448, 1024, 1, 1792, 448, 448, 1792, 1792] + - [1, 4128.67] + - - [5056, 3584, 1, 768, 5056, 5056, 768, 768] + - [2, 6188.1] + - - [1856, 1408, 1, 1792, 1856, 1856, 1792, 1792] + - [1, 5841.08] + - - [256, 448, 1, 256, 256, 256, 256, 256] + - [4, 1761.57] + - - [4288, 4, 1, 1792, 4288, 4288, 1792, 1792] + - [7, 165.245] + - - [2944, 2368, 1, 3328, 2944, 2944, 3328, 3328] + - [2, 6128.61] + - - [1856, 4, 1, 768, 1856, 1856, 768, 768] + - [7, 169.165] + - - [128, 1856, 1, 256, 128, 128, 256, 256] + - [4, 2648.44] + - - [2944, 1856, 1, 1792, 2944, 2944, 1792, 1792] + - [1, 5677.49] + - - [704, 64, 1, 1792, 704, 704, 1792, 1792] + - [5, 902.293] + - - [1856, 4, 1, 1792, 1856, 1856, 1792, 1792] + - [7, 179.06] + - - [256, 256, 1, 3328, 256, 256, 3328, 3328] + - [4, 1448.27] + - - [4, 4, 1, 256, 4, 4, 256, 256] + - [7, 0.280683] + - - [2944, 64, 1, 1792, 2944, 2944, 1792, 1792] + - [5, 2986.92] + - - [4288, 448, 1, 768, 4288, 4288, 768, 768] + - [6, 5423.93] + - - [5888, 5888, 1, 256, 5888, 5888, 256, 256] + - [2, 6059.65] + - - [64, 128, 1, 3328, 64, 64, 3328, 3328] + - [5, 192.692] + - - [6784, 1408, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 5907.38] + - - [4, 256, 1, 256, 4, 4, 256, 256] + - [7, 22.3978] + - - [704, 1408, 1, 1792, 704, 704, 1792, 1792] + - [1, 5282.94] + - - [1024, 256, 1, 256, 1024, 1024, 256, 256] + - [5, 2913.03] + - - [2368, 6784, 1, 1792, 2368, 2368, 1792, 1792] + - [2, 6248.35] + - - [1856, 256, 1, 768, 1856, 1856, 768, 768] + - [1, 4004.94] + - - [1408, 1024, 1, 1792, 1408, 1408, 1792, 1792] + - [6, 5351.17] + - - [1024, 3584, 1, 1792, 1024, 1024, 1792, 1792] + - [1, 5845.58] + - - [704, 5056, 1, 256, 704, 704, 256, 256] + - [1, 5256.85] + - - [4, 448, 1, 3328, 4, 4, 3328, 3328] + - [7, 50.1301] + - - [5056, 448, 1, 256, 5056, 5056, 256, 256] + - [6, 4901.65] + - - [1856, 448, 1, 3328, 1856, 1856, 3328, 3328] + - [6, 4505.08] + - - [5888, 3584, 1, 256, 5888, 5888, 256, 256] + - [2, 5873.77] + - - [2368, 128, 1, 3328, 2368, 2368, 3328, 3328] + - [5, 2982.37] + - - [4, 2368, 1, 3328, 4, 4, 3328, 3328] + - [7, 201.015] + - - [5056, 128, 1, 768, 5056, 5056, 768, 768] + - [5, 3914.63] + - - [64, 256, 1, 1792, 64, 64, 1792, 1792] + - [5, 397.944] + - - [4288, 64, 1, 768, 4288, 4288, 768, 768] + - [5, 2476.27] + - - [2368, 6784, 1, 768, 2368, 2368, 768, 768] + - [2, 6139.51] + - - [2944, 2944, 1, 768, 2944, 2944, 768, 768] + - [6, 5920.71] + - - [5888, 4288, 1, 1792, 5888, 5888, 1792, 1792] + - [2, 6307.34] + - - [128, 1408, 1, 1792, 128, 128, 1792, 1792] + - [5, 2785.86] + - - [256, 1856, 1, 256, 256, 256, 256, 256] + - [6, 3264.49] + - - [3584, 2368, 1, 3328, 3584, 3584, 3328, 3328] + - [2, 5841.65] + - - [5888, 448, 1, 256, 5888, 5888, 256, 256] + - [6, 4743.02] + - - [2368, 2368, 1, 256, 2368, 2368, 256, 256] + - [6, 5750.37] + - - [3584, 256, 1, 1792, 3584, 3584, 1792, 1792] + - [1, 4885.68] + - - [256, 4288, 1, 3328, 256, 256, 3328, 3328] + - [5, 4173.78] + - - [448, 128, 1, 256, 448, 448, 256, 256] + - [4, 888.677] + - - [2944, 128, 1, 768, 2944, 2944, 768, 768] + - [5, 3488.3] + - - [4, 5056, 1, 256, 4, 4, 256, 256] + - [7, 185.884] + - - [1856, 1024, 1, 256, 1856, 1856, 256, 256] + - [6, 4894.27] + - - [256, 3584, 1, 256, 256, 256, 256, 256] + - [1, 4210.88] + - - [5888, 256, 1, 256, 5888, 5888, 256, 256] + - [6, 4994.45] + - - [256, 4288, 1, 256, 256, 256, 256, 256] + - [0, 3502.9] + - - [1408, 128, 1, 1792, 1408, 1408, 1792, 1792] + - [5, 2939.9] + - - [4, 1024, 1, 256, 4, 4, 256, 256] + - [7, 70.0756] + - - [2368, 4, 1, 1792, 2368, 2368, 1792, 1792] + - [7, 208.31] + - - [4288, 4288, 1, 256, 4288, 4288, 256, 256] + - [6, 5962.06] + - - [2944, 5888, 1, 256, 2944, 2944, 256, 256] + - [6, 5937.13] + - - [5888, 1408, 1, 768, 5888, 5888, 768, 768] + - [2, 6192.93] + - - [4288, 6784, 1, 768, 4288, 4288, 768, 768] + - [2, 6225.13] + - - [256, 704, 1, 256, 256, 256, 256, 256] + - [5, 2155.15] + - - [448, 5888, 1, 256, 448, 448, 256, 256] + - [1, 4636.88] + - - [256, 6784, 1, 3328, 256, 256, 3328, 3328] + - [2, 5004.79] + - - [4, 2368, 1, 1792, 4, 4, 1792, 1792] + - [7, 199.599] + - - [5888, 4, 1, 3328, 5888, 5888, 3328, 3328] + - [7, 165.305] + - - [2944, 4288, 1, 256, 2944, 2944, 256, 256] + - [6, 5814.61] + - - [64, 2368, 1, 768, 64, 64, 768, 768] + - [4, 2035.3] + - - [6784, 1408, 1, 256, 6784, 6784, 256, 256] + - [6, 5566.68] + - - [1408, 2368, 1, 768, 1408, 1408, 768, 768] + - [6, 5581.71] + - - [64, 6784, 1, 3328, 64, 64, 3328, 3328] + - [1, 3620.3] + - - [448, 448, 1, 256, 448, 448, 256, 256] + - [4, 2375.41] + - - [6784, 2368, 1, 256, 6784, 6784, 256, 256] + - [6, 5753.7] + - - [64, 5888, 1, 768, 64, 64, 768, 768] + - [1, 2924.32] + - - [4, 3584, 1, 3328, 4, 4, 3328, 3328] + - [7, 177.969] + - - [3584, 64, 1, 3328, 3584, 3584, 3328, 3328] + - [5, 3743.23] + - - [2944, 2368, 1, 1792, 2944, 2944, 1792, 1792] + - [2, 6072.36] + - - [1856, 704, 1, 3328, 1856, 1856, 3328, 3328] + - [1, 4760.56] + - - [704, 4288, 1, 3328, 704, 704, 3328, 3328] + - [1, 5692.7] + - - [4, 448, 1, 256, 4, 4, 256, 256] + - [7, 33.1169] + - - [4288, 6784, 1, 3328, 4288, 4288, 3328, 3328] + - [2, 6333.35] + - - [128, 6784, 1, 3328, 128, 128, 3328, 3328] + - [1, 4647.64] + - - [6784, 256, 1, 256, 6784, 6784, 256, 256] + - [6, 4519.52] + - - [1408, 704, 1, 256, 1408, 1408, 256, 256] + - [6, 4573.57] + - - [64, 2368, 1, 256, 64, 64, 256, 256] + - [4, 1706.01] + - - [64, 5056, 1, 3328, 64, 64, 3328, 3328] + - [4, 2990.66] + - - [5056, 64, 1, 256, 5056, 5056, 256, 256] + - [4, 2410.1] + - - [1856, 4288, 1, 256, 1856, 1856, 256, 256] + - [6, 5744.35] + - - [4, 1856, 1, 768, 4, 4, 768, 768] + - [7, 159.359] + - - [704, 4288, 1, 1792, 704, 704, 1792, 1792] + - [1, 5637.9] + - - [6784, 4, 1, 1792, 6784, 6784, 1792, 1792] + - [7, 166.318] + - - [64, 5888, 1, 1792, 64, 64, 1792, 1792] + - [4, 3170.81] + - - [2368, 5056, 1, 3328, 2368, 2368, 3328, 3328] + - [2, 6222.28] + - - [1856, 4288, 1, 768, 1856, 1856, 768, 768] + - [2, 5958.83] + - - [4, 4288, 1, 768, 4, 4, 768, 768] + - [7, 241.285] + - - [4, 5888, 1, 256, 4, 4, 256, 256] + - [7, 190.617] + - - [6784, 128, 1, 256, 6784, 6784, 256, 256] + - [6, 4011.99] + - - [4, 128, 1, 3328, 4, 4, 3328, 3328] + - [7, 14.3497] + - - [4, 4288, 1, 3328, 4, 4, 3328, 3328] + - [7, 168.25] + - - [5888, 5056, 1, 3328, 5888, 5888, 3328, 3328] + - [2, 6462.47] + - - [4288, 1024, 1, 1792, 4288, 4288, 1792, 1792] + - [1, 5567.2] + - - [3584, 64, 1, 768, 3584, 3584, 768, 768] + - [5, 3378.0] + - - [256, 2944, 1, 3328, 256, 256, 3328, 3328] + - [5, 4238.83] + - - [1408, 2368, 1, 1792, 1408, 1408, 1792, 1792] + - [6, 5622.3] + - - [1856, 5888, 1, 256, 1856, 1856, 256, 256] + - [6, 5776.48] + - - [2368, 5888, 1, 256, 2368, 2368, 256, 256] + - [6, 5848.4] + - - [4, 5056, 1, 768, 4, 4, 768, 768] + - [7, 265.414] + - - [5056, 3584, 1, 256, 5056, 5056, 256, 256] + - [6, 5981.24] + - - [3584, 5888, 1, 1792, 3584, 3584, 1792, 1792] + - [2, 6257.9] + - - [128, 5056, 1, 1792, 128, 128, 1792, 1792] + - [4, 4021.54] + - - [4, 3584, 1, 1792, 4, 4, 1792, 1792] + - [7, 183.302] + - - [5056, 2944, 1, 3328, 5056, 5056, 3328, 3328] + - [2, 6221.96] + - - [3584, 2368, 1, 1792, 3584, 3584, 1792, 1792] + - [2, 5779.31] + - - [448, 5056, 1, 3328, 448, 448, 3328, 3328] + - [1, 5094.27] + - - [1856, 3584, 1, 1792, 1856, 1856, 1792, 1792] + - [1, 5880.67] + - - [5056, 1408, 1, 3328, 5056, 5056, 3328, 3328] + - [2, 6266.33] + - - [5056, 704, 1, 768, 5056, 5056, 768, 768] + - [6, 5881.91] + - - [1024, 6784, 1, 1792, 1024, 1024, 1792, 1792] + - [2, 6042.25] + - - [64, 128, 1, 256, 64, 64, 256, 256] + - [4, 142.267] + - - [5888, 1024, 1, 256, 5888, 5888, 256, 256] + - [2, 5534.32] + - - [1856, 2944, 1, 768, 1856, 1856, 768, 768] + - [6, 5946.88] + - - [5888, 6784, 1, 1792, 5888, 5888, 1792, 1792] + - [2, 6445.6] + - - [64, 64, 1, 1792, 64, 64, 1792, 1792] + - [5, 104.744] + - - [704, 64, 1, 768, 704, 704, 768, 768] + - [5, 957.233] + - - [5056, 2944, 1, 1792, 5056, 5056, 1792, 1792] + - [2, 6180.77] + - - [1856, 128, 1, 3328, 1856, 1856, 3328, 3328] + - [4, 3840.66] + - - [5888, 2944, 1, 256, 5888, 5888, 256, 256] + - [2, 5884.07] + - - [5056, 1408, 1, 768, 5056, 5056, 768, 768] + - [2, 6044.25] + - - [2944, 4288, 1, 1792, 2944, 2944, 1792, 1792] + - [2, 6009.46] + - - [4, 5056, 1, 3328, 4, 4, 3328, 3328] + - [7, 171.564] + - - [256, 5056, 1, 1792, 256, 256, 1792, 1792] + - [1, 4643.59] + - - [1024, 5888, 1, 1792, 1024, 1024, 1792, 1792] + - [2, 6012.75] + - - [128, 5056, 1, 256, 128, 128, 256, 256] + - [4, 3227.36] + - - [64, 2944, 1, 1792, 64, 64, 1792, 1792] + - [4, 2745.8] + - - [64, 128, 1, 1792, 64, 64, 1792, 1792] + - [5, 206.863] + - - [1408, 6784, 1, 256, 1408, 1408, 256, 256] + - [6, 5841.52] + - - [2368, 2944, 1, 256, 2368, 2368, 256, 256] + - [6, 5729.49] + - - [704, 4, 1, 256, 704, 704, 256, 256] + - [7, 48.9041] + - - [64, 4, 1, 768, 64, 64, 768, 768] + - [7, 6.36488] + - - [5056, 1408, 1, 1792, 5056, 5056, 1792, 1792] + - [2, 6202.58] + - - [1024, 1856, 1, 768, 1024, 1024, 768, 768] + - [2, 5211.4] + - - [5056, 64, 1, 768, 5056, 5056, 768, 768] + - [4, 2879.69] + - - [3584, 2944, 1, 256, 3584, 3584, 256, 256] + - [6, 5663.92] + - - [4, 1408, 1, 256, 4, 4, 256, 256] + - [7, 95.4102] + - - [5056, 64, 1, 3328, 5056, 5056, 3328, 3328] + - [4, 3119.65] + - - [448, 3584, 1, 3328, 448, 448, 3328, 3328] + - [2, 4635.39] + - - [2944, 2368, 1, 768, 2944, 2944, 768, 768] + - [6, 5967.93] + - - [2944, 3584, 1, 1792, 2944, 2944, 1792, 1792] + - [2, 5946.56] + - - [2944, 704, 1, 256, 2944, 2944, 256, 256] + - [6, 5270.51] + - - [64, 64, 1, 256, 64, 64, 256, 256] + - [3, 53.8228] + - - [64, 448, 1, 256, 64, 64, 256, 256] + - [5, 469.612] + - - [2368, 2944, 1, 3328, 2368, 2368, 3328, 3328] + - [2, 6139.54] + - - [4288, 1024, 1, 3328, 4288, 4288, 3328, 3328] + - [1, 5596.31] + - - [2368, 2368, 1, 1792, 2368, 2368, 1792, 1792] + - [1, 5827.95] + - - [704, 1408, 1, 256, 704, 704, 256, 256] + - [1, 4673.43] + - - [448, 4, 1, 768, 448, 448, 768, 768] + - [7, 47.8839] + - - [1024, 5888, 1, 256, 1024, 1024, 256, 256] + - [2, 5414.96] + - - [128, 5056, 1, 768, 128, 128, 768, 768] + - [4, 3733.82] + - - [3584, 6784, 1, 768, 3584, 3584, 768, 768] + - [2, 6186.46] + - - [64, 1024, 1, 1792, 64, 64, 1792, 1792] + - [4, 1225.2] + - - [256, 256, 1, 256, 256, 256, 256, 256] + - [5, 1083.66] + - - [1856, 1408, 1, 768, 1856, 1856, 768, 768] + - [1, 5741.36] + - - [704, 704, 1, 1792, 704, 704, 1792, 1792] + - [5, 4068.42] + - - [1408, 2944, 1, 3328, 1408, 1408, 3328, 3328] + - [2, 6253.27] + - - [5888, 128, 1, 256, 5888, 5888, 256, 256] + - [6, 3500.8] + - - [1856, 1856, 1, 768, 1856, 1856, 768, 768] + - [6, 5655.44] + - - [256, 704, 1, 3328, 256, 256, 3328, 3328] + - [5, 2942.19] + - - [3584, 1856, 1, 768, 3584, 3584, 768, 768] + - [6, 6036.34] + - - [3584, 128, 1, 3328, 3584, 3584, 3328, 3328] + - [1, 4234.7] + - - [5888, 128, 1, 1792, 5888, 5888, 1792, 1792] + - [5, 4239.05] + - - [4288, 64, 1, 256, 4288, 4288, 256, 256] + - [5, 2126.48] + - - [5056, 4, 1, 256, 5056, 5056, 256, 256] + - [7, 258.854] + - - [64, 4288, 1, 1792, 64, 64, 1792, 1792] + - [5, 2436.29] + - - [2944, 4, 1, 1792, 2944, 2944, 1792, 1792] + - [7, 223.608] + - - [5888, 1856, 1, 256, 5888, 5888, 256, 256] + - [6, 5725.68] + - - [2944, 4, 1, 768, 2944, 2944, 768, 768] + - [7, 227.355] + - - [704, 6784, 1, 3328, 704, 704, 3328, 3328] + - [2, 5764.85] + - - [1408, 2944, 1, 1792, 1408, 1408, 1792, 1792] + - [2, 6184.59] + - - [1408, 128, 1, 256, 1408, 1408, 256, 256] + - [5, 2177.73] + - - [4288, 128, 1, 768, 4288, 4288, 768, 768] + - [5, 3365.17] + - - [704, 704, 1, 768, 704, 704, 768, 768] + - [5, 3910.51] + - - [4288, 1408, 1, 256, 4288, 4288, 256, 256] + - [6, 5676.03] + - - [2944, 5888, 1, 768, 2944, 2944, 768, 768] + - [2, 6244.08] + - - [4288, 64, 1, 1792, 4288, 4288, 1792, 1792] + - [5, 2621.99] + - - [5888, 1856, 1, 1792, 5888, 5888, 1792, 1792] + - [2, 6133.65] + - - [2368, 256, 1, 256, 2368, 2368, 256, 256] + - [5, 3203.34] + - - [256, 2944, 1, 256, 256, 256, 256, 256] + - [1, 3482.07] + - - [1408, 256, 1, 3328, 1408, 1408, 3328, 3328] + - [4, 3673.73] + - - [128, 1856, 1, 1792, 128, 128, 1792, 1792] + - [4, 3619.08] + - - [2944, 128, 1, 3328, 2944, 2944, 3328, 3328] + - [5, 3852.95] + - - [1024, 2368, 1, 768, 1024, 1024, 768, 768] + - [6, 5445.09] + - - [3584, 704, 1, 256, 3584, 3584, 256, 256] + - [6, 5271.89] + - - [1024, 128, 1, 768, 1024, 1024, 768, 768] + - [5, 2535.31] + - - [1408, 4, 1, 3328, 1408, 1408, 3328, 3328] + - [7, 154.662] + - - [3584, 1856, 1, 1792, 3584, 3584, 1792, 1792] + - [2, 5784.91] + - - [704, 1856, 1, 3328, 704, 704, 3328, 3328] + - [1, 4755.55] + - - [4288, 704, 1, 1792, 4288, 4288, 1792, 1792] + - [6, 5438.07] + - - [1024, 704, 1, 256, 1024, 1024, 256, 256] + - [6, 3312.95] + - - [256, 1024, 1, 768, 256, 256, 768, 768] + - [5, 3759.14] + - - [2944, 3584, 1, 768, 2944, 2944, 768, 768] + - [6, 5973.52] + - - [704, 64, 1, 3328, 704, 704, 3328, 3328] + - [4, 918.016] + - - [64, 1024, 1, 256, 64, 64, 256, 256] + - [4, 1058.33] + - - [4288, 128, 1, 256, 4288, 4288, 256, 256] + - [4, 2860.97] + - - [5056, 1856, 1, 768, 5056, 5056, 768, 768] + - [6, 5860.15] + - - [2368, 1856, 1, 256, 2368, 2368, 256, 256] + - [6, 5451.81] + - - [2368, 1024, 1, 3328, 2368, 2368, 3328, 3328] + - [1, 5463.39] + - - [128, 6784, 1, 768, 128, 128, 768, 768] + - [1, 4297.3] + - - [4, 2944, 1, 3328, 4, 4, 3328, 3328] + - [7, 219.163] + - - [1856, 5888, 1, 768, 1856, 1856, 768, 768] + - [2, 6018.59] + - - [704, 2944, 1, 768, 704, 704, 768, 768] + - [1, 5600.19] + - - [128, 4288, 1, 768, 128, 128, 768, 768] + - [4, 3287.39] + - - [1856, 448, 1, 256, 1856, 1856, 256, 256] + - [6, 3806.02] + - - [4, 64, 1, 768, 4, 4, 768, 768] + - [7, 6.60235] + - - [3584, 5888, 1, 768, 3584, 3584, 768, 768] + - [2, 6124.86] + - - [448, 1856, 1, 3328, 448, 448, 3328, 3328] + - [6, 4482.92] + - - [64, 2944, 1, 768, 64, 64, 768, 768] + - [5, 2438.75] + - - [5888, 704, 1, 768, 5888, 5888, 768, 768] + - [6, 5763.93] + - - [5888, 704, 1, 3328, 5888, 5888, 3328, 3328] + - [1, 5260.4] + - - [128, 128, 1, 1792, 128, 128, 1792, 1792] + - [4, 413.296] + - - [6784, 3584, 1, 256, 6784, 6784, 256, 256] + - [2, 5981.39] + - - [256, 256, 1, 768, 256, 256, 768, 768] + - [4, 1464.36] + - - [1408, 5056, 1, 768, 1408, 1408, 768, 768] + - [2, 6008.82] + - - [4, 5056, 1, 1792, 4, 4, 1792, 1792] + - [7, 163.902] + - - [5888, 448, 1, 3328, 5888, 5888, 3328, 3328] + - [2, 5185.71] + - - [6784, 1024, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 6120.54] + - - [2368, 3584, 1, 1792, 2368, 2368, 1792, 1792] + - [2, 5815.66] + - - [64, 2944, 1, 3328, 64, 64, 3328, 3328] + - [5, 2975.38] + - - [4, 448, 1, 1792, 4, 4, 1792, 1792] + - [7, 51.4247] + - - [4, 256, 1, 3328, 4, 4, 3328, 3328] + - [7, 29.007] + - - [3584, 1408, 1, 1792, 3584, 3584, 1792, 1792] + - [2, 6074.96] + - - [1024, 6784, 1, 3328, 1024, 1024, 3328, 3328] + - [2, 6111.19] + - - [2944, 2944, 1, 256, 2944, 2944, 256, 256] + - [6, 5795.83] + - - [4288, 256, 1, 1792, 4288, 4288, 1792, 1792] + - [5, 4126.74] + - - [1856, 6784, 1, 1792, 1856, 1856, 1792, 1792] + - [2, 6010.54] + - - [3584, 4288, 1, 768, 3584, 3584, 768, 768] + - [6, 6180.98] + - - [1024, 448, 1, 256, 1024, 1024, 256, 256] + - [6, 3202.85] + - - [2368, 4288, 1, 3328, 2368, 2368, 3328, 3328] + - [1, 5886.88] + - - [128, 4288, 1, 256, 128, 128, 256, 256] + - [4, 2891.49] + - - [6784, 5888, 1, 768, 6784, 6784, 768, 768] + - [2, 6383.12] + - - [1856, 1024, 1, 1792, 1856, 1856, 1792, 1792] + - [2, 5417.95] + - - [1856, 3584, 1, 256, 1856, 1856, 256, 256] + - [1, 5556.77] + - - [64, 2944, 1, 256, 64, 64, 256, 256] + - [4, 2009.69] + - - [256, 2944, 1, 768, 256, 256, 768, 768] + - [5, 3970.42] + - - [64, 3584, 1, 3328, 64, 64, 3328, 3328] + - [4, 3470.91] + - - [1408, 5056, 1, 256, 1408, 1408, 256, 256] + - [6, 5820.22] + - - [4288, 1856, 1, 768, 4288, 4288, 768, 768] + - [2, 5924.6] + - - [448, 4, 1, 3328, 448, 448, 3328, 3328] + - [7, 50.0678] + - - [448, 2368, 1, 256, 448, 448, 256, 256] + - [0, 3515.11] + - - [256, 2368, 1, 768, 256, 256, 768, 768] + - [4, 3688.38] + - - [4288, 3584, 1, 1792, 4288, 4288, 1792, 1792] + - [2, 6367.58] + - - [128, 2944, 1, 3328, 128, 128, 3328, 3328] + - [4, 3682.81] + - - [3584, 4, 1, 768, 3584, 3584, 768, 768] + - [7, 258.941] + - - [1856, 1408, 1, 3328, 1856, 1856, 3328, 3328] + - [1, 5877.01] + - - [256, 128, 1, 256, 256, 256, 256, 256] + - [4, 560.623] + - - [256, 5056, 1, 256, 256, 256, 256, 256] + - [0, 3983.17] + - - [5888, 256, 1, 3328, 5888, 5888, 3328, 3328] + - [1, 5498.5] + - - [5888, 4, 1, 768, 5888, 5888, 768, 768] + - [7, 278.111] + - - [256, 2944, 1, 1792, 256, 256, 1792, 1792] + - [5, 4138.03] + - - [5888, 5888, 1, 3328, 5888, 5888, 3328, 3328] + - [2, 6438.44] + - - [6784, 704, 1, 768, 6784, 6784, 768, 768] + - [6, 5615.29] + - - [5888, 448, 1, 768, 5888, 5888, 768, 768] + - [6, 5115.65] + - - [128, 6784, 1, 256, 128, 128, 256, 256] + - [1, 3572.55] + - - [128, 2368, 1, 256, 128, 128, 256, 256] + - [4, 2476.33] + - - [1024, 128, 1, 3328, 1024, 1024, 3328, 3328] + - [5, 2858.57] + - - [64, 3584, 1, 256, 64, 64, 256, 256] + - [4, 2258.38] + - - [1024, 5056, 1, 768, 1024, 1024, 768, 768] + - [6, 6030.49] + - - [2944, 6784, 1, 1792, 2944, 2944, 1792, 1792] + - [2, 6253.02] + - - [448, 5056, 1, 1792, 448, 448, 1792, 1792] + - [6, 5300.41] + - - [2944, 448, 1, 256, 2944, 2944, 256, 256] + - [6, 4438.21] + - - [256, 5056, 1, 3328, 256, 256, 3328, 3328] + - [1, 4718.85] + - - [256, 1024, 1, 3328, 256, 256, 3328, 3328] + - [4, 4170.43] + - - [1856, 4, 1, 256, 1856, 1856, 256, 256] + - [7, 127.012] + - - [448, 4288, 1, 256, 448, 448, 256, 256] + - [1, 4708.44] + - - [3584, 64, 1, 256, 3584, 3584, 256, 256] + - [5, 2696.25] + - - [5056, 5056, 1, 768, 5056, 5056, 768, 768] + - [2, 6263.3] + - - [4288, 1024, 1, 768, 4288, 4288, 768, 768] + - [6, 5646.71] + - - [4, 1024, 1, 3328, 4, 4, 3328, 3328] + - [7, 109.082] + - - [256, 5056, 1, 768, 256, 256, 768, 768] + - [1, 4477.58] + - - [5056, 1024, 1, 3328, 5056, 5056, 3328, 3328] + - [2, 6299.98] + - - [1024, 5056, 1, 3328, 1024, 1024, 3328, 3328] + - [2, 6292.14] + - - [2368, 448, 1, 1792, 2368, 2368, 1792, 1792] + - [5, 4047.27] + - - [64, 2368, 1, 3328, 64, 64, 3328, 3328] + - [5, 2411.03] + - - [6784, 4288, 1, 768, 6784, 6784, 768, 768] + - [2, 6220.77] + - - [448, 5056, 1, 768, 448, 448, 768, 768] + - [6, 5105.18] + - - [2368, 3584, 1, 768, 2368, 2368, 768, 768] + - [6, 5977.18] + - - [1856, 2368, 1, 256, 1856, 1856, 256, 256] + - [6, 5457.69] + - - [6784, 2944, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 6249.06] + - - [256, 2368, 1, 3328, 256, 256, 3328, 3328] + - [5, 3887.94] + - - [5056, 4288, 1, 1792, 5056, 5056, 1792, 1792] + - [2, 6165.33] + - - [2368, 256, 1, 1792, 2368, 2368, 1792, 1792] + - [5, 3907.54] + - - [4288, 256, 1, 3328, 4288, 4288, 3328, 3328] + - [4, 4194.73] + - - [1024, 1024, 1, 768, 1024, 1024, 768, 768] + - [6, 5194.15] + - - [1024, 3584, 1, 256, 1024, 1024, 256, 256] + - [6, 5540.95] + - - [6784, 6784, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 6616.49] + - - [6784, 4, 1, 3328, 6784, 6784, 3328, 3328] + - [7, 160.857] + - - [704, 2944, 1, 256, 704, 704, 256, 256] + - [1, 5243.5] + - - [5888, 5888, 1, 1792, 5888, 5888, 1792, 1792] + - [2, 6400.52] + - - [1408, 4288, 1, 1792, 1408, 1408, 1792, 1792] + - [2, 6100.13] + - - [2368, 704, 1, 3328, 2368, 2368, 3328, 3328] + - [2, 4816.79] + - - [3584, 5056, 1, 3328, 3584, 3584, 3328, 3328] + - [2, 6305.96] + - - [3584, 1024, 1, 768, 3584, 3584, 768, 768] + - [6, 6130.14] + - - [4288, 1024, 1, 256, 4288, 4288, 256, 256] + - [6, 5450.65] + - - [256, 128, 1, 3328, 256, 256, 3328, 3328] + - [5, 692.457] + - - [1856, 2944, 1, 1792, 1856, 1856, 1792, 1792] + - [1, 5691.67] + - - [2368, 128, 1, 768, 2368, 2368, 768, 768] + - [5, 2869.85] + - - [4, 6784, 1, 3328, 4, 4, 3328, 3328] + - [7, 171.179] + - - [256, 3584, 1, 1792, 256, 256, 1792, 1792] + - [1, 4890.0] + - - [704, 1856, 1, 256, 704, 704, 256, 256] + - [6, 4379.82] + - - [1024, 1856, 1, 1792, 1024, 1024, 1792, 1792] + - [2, 5413.49] + - - [1024, 1408, 1, 256, 1024, 1024, 256, 256] + - [6, 4749.94] + - - [1024, 4288, 1, 768, 1024, 1024, 768, 768] + - [1, 5353.85] + - - [256, 1024, 1, 1792, 256, 256, 1792, 1792] + - [5, 4083.47] + - - [5888, 1408, 1, 3328, 5888, 5888, 3328, 3328] + - [2, 6402.69] + - - [448, 2944, 1, 256, 448, 448, 256, 256] + - [1, 4283.89] + - - [256, 64, 1, 3328, 256, 256, 3328, 3328] + - [5, 335.498] + - - [5056, 448, 1, 768, 5056, 5056, 768, 768] + - [6, 5204.75] + - - [128, 4288, 1, 1792, 128, 128, 1792, 1792] + - [4, 3486.81] + - - [64, 4288, 1, 768, 64, 64, 768, 768] + - [4, 2226.31] + - - [4288, 448, 1, 3328, 4288, 4288, 3328, 3328] + - [5, 4492.97] + - - [128, 64, 1, 256, 128, 128, 256, 256] + - [4, 146.69] + - - [128, 1408, 1, 3328, 128, 128, 3328, 3328] + - [5, 2890.71] + - - [1856, 2368, 1, 768, 1856, 1856, 768, 768] + - [6, 5628.25] + - - [4, 4288, 1, 1792, 4, 4, 1792, 1792] + - [7, 159.773] + - - [2368, 5888, 1, 1792, 2368, 2368, 1792, 1792] + - [2, 6191.4] + - - [6784, 64, 1, 768, 6784, 6784, 768, 768] + - [4, 3326.99] + - - [3584, 5056, 1, 1792, 3584, 3584, 1792, 1792] + - [2, 6241.05] + - - [128, 1024, 1, 3328, 128, 128, 3328, 3328] + - [5, 2595.27] + - - [4288, 2368, 1, 768, 4288, 4288, 768, 768] + - [6, 5944.98] + - - [2944, 128, 1, 256, 2944, 2944, 256, 256] + - [6, 2718.78] + - - [6784, 6784, 1, 768, 6784, 6784, 768, 768] + - [2, 6390.3] + - - [1856, 1856, 1, 1792, 1856, 1856, 1792, 1792] + - [1, 5554.42] + - - [1024, 4, 1, 256, 1024, 1024, 256, 256] + - [7, 71.1333] + - - [1024, 4288, 1, 256, 1024, 1024, 256, 256] + - [6, 5283.64] + - - [5056, 2368, 1, 768, 5056, 5056, 768, 768] + - [2, 6055.65] + - - [448, 3584, 1, 768, 448, 448, 768, 768] + - [6, 4448.62] + - - [2944, 64, 1, 256, 2944, 2944, 256, 256] + - [5, 2253.11] + - - [4, 256, 1, 1792, 4, 4, 1792, 1792] + - [7, 27.2519] + - - [2368, 256, 1, 768, 2368, 2368, 768, 768] + - [5, 3799.89] + - - [256, 5888, 1, 768, 256, 256, 768, 768] + - [1, 5230.11] + - - [1024, 2368, 1, 3328, 1024, 1024, 3328, 3328] + - [1, 5429.36] + - - [3584, 3584, 1, 768, 3584, 3584, 768, 768] + - [6, 5990.74] + - - [4, 704, 1, 768, 4, 4, 768, 768] + - [7, 66.6571] + - - [3584, 4288, 1, 256, 3584, 3584, 256, 256] + - [6, 5824.22] + - - [5056, 4, 1, 768, 5056, 5056, 768, 768] + - [7, 318.661] + - - [2368, 5056, 1, 768, 2368, 2368, 768, 768] + - [2, 6030.64] + - - [6784, 256, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 5050.14] + - - [128, 5888, 1, 256, 128, 128, 256, 256] + - [4, 3301.13] + - - [64, 448, 1, 3328, 64, 64, 3328, 3328] + - [4, 581.553] + - - [256, 5888, 1, 3328, 256, 256, 3328, 3328] + - [1, 5498.94] + - - [1856, 704, 1, 768, 1856, 1856, 768, 768] + - [6, 4733.31] + - - [4, 1856, 1, 3328, 4, 4, 3328, 3328] + - [7, 185.092] + - - [3584, 2944, 1, 768, 3584, 3584, 768, 768] + - [6, 6082.8] + - - [448, 1408, 1, 1792, 448, 448, 1792, 1792] + - [5, 4034.78] + - - [5056, 128, 1, 3328, 5056, 5056, 3328, 3328] + - [4, 4196.77] + - - [5888, 6784, 1, 768, 5888, 5888, 768, 768] + - [2, 6352.77] + - - [4288, 64, 1, 3328, 4288, 4288, 3328, 3328] + - [5, 2820.07] + - - [448, 1024, 1, 3328, 448, 448, 3328, 3328] + - [1, 4219.96] + - - [1856, 256, 1, 3328, 1856, 1856, 3328, 3328] + - [1, 4369.78] + - - [5888, 704, 1, 256, 5888, 5888, 256, 256] + - [6, 5497.04] + - - [1856, 2944, 1, 3328, 1856, 1856, 3328, 3328] + - [1, 5718.08] + - - [704, 4288, 1, 256, 704, 704, 256, 256] + - [1, 5144.25] + - - [128, 2368, 1, 3328, 128, 128, 3328, 3328] + - [5, 2893.65] + - - [448, 2368, 1, 768, 448, 448, 768, 768] + - [5, 3896.11] + - - [5888, 64, 1, 1792, 5888, 5888, 1792, 1792] + - [4, 3522.45] + - - [1024, 64, 1, 3328, 1024, 1024, 3328, 3328] + - [4, 1339.55] + - - [4288, 5056, 1, 768, 4288, 4288, 768, 768] + - [2, 6077.92] + - - [64, 1024, 1, 768, 64, 64, 768, 768] + - [5, 1183.72] + - - [1408, 64, 1, 768, 1408, 1408, 768, 768] + - [5, 1730.11] + - - [1024, 3584, 1, 768, 1024, 1024, 768, 768] + - [6, 6010.07] + - - [1856, 1408, 1, 256, 1856, 1856, 256, 256] + - [1, 5407.87] + - - [5888, 6784, 1, 256, 5888, 5888, 256, 256] + - [2, 6065.45] + - - [2368, 64, 1, 768, 2368, 2368, 768, 768] + - [5, 2283.78] + - - [6784, 128, 1, 768, 6784, 6784, 768, 768] + - [1, 4483.49] + - - [256, 2368, 1, 256, 256, 256, 256, 256] + - [5, 3218.12] + - - [2368, 6784, 1, 3328, 2368, 2368, 3328, 3328] + - [2, 6281.46] + - - [1408, 64, 1, 3328, 1408, 1408, 3328, 3328] + - [5, 1890.03] + - - [1856, 1856, 1, 3328, 1856, 1856, 3328, 3328] + - [1, 5595.93] + - - [64, 4288, 1, 256, 64, 64, 256, 256] + - [4, 1827.38] + - - [128, 256, 1, 768, 128, 128, 768, 768] + - [5, 732.182] + - - [448, 256, 1, 3328, 448, 448, 3328, 3328] + - [5, 2497.61] + - - [2944, 128, 1, 1792, 2944, 2944, 1792, 1792] + - [5, 3678.81] + - - [2368, 1408, 1, 768, 2368, 2368, 768, 768] + - [6, 5586.21] + - - [1408, 5888, 1, 1792, 1408, 1408, 1792, 1792] + - [2, 6354.89] + - - [6784, 1024, 1, 768, 6784, 6784, 768, 768] + - [2, 5930.24] + - - [704, 256, 1, 1792, 704, 704, 1792, 1792] + - [5, 2889.25] + - - [1856, 64, 1, 3328, 1856, 1856, 3328, 3328] + - [4, 2377.1] + - - [3584, 1408, 1, 3328, 3584, 3584, 3328, 3328] + - [2, 6145.32] + - - [5056, 2944, 1, 256, 5056, 5056, 256, 256] + - [6, 5920.9] + - - [1856, 448, 1, 1792, 1856, 1856, 1792, 1792] + - [6, 4425.71] + - - [5888, 128, 1, 768, 5888, 5888, 768, 768] + - [5, 4007.07] + - - [5888, 1024, 1, 768, 5888, 5888, 768, 768] + - [2, 5952.79] + - - [3584, 2944, 1, 1792, 3584, 3584, 1792, 1792] + - [2, 5918.99] + - - [128, 448, 1, 768, 128, 128, 768, 768] + - [4, 1298.11] + - - [256, 5888, 1, 256, 256, 256, 256, 256] + - [1, 4605.82] + - - [2944, 4, 1, 3328, 2944, 2944, 3328, 3328] + - [7, 222.48] + - - [4288, 3584, 1, 768, 4288, 4288, 768, 768] + - [2, 6248.12] + - - [448, 4, 1, 256, 448, 448, 256, 256] + - [7, 33.1169] + - - [5056, 128, 1, 256, 5056, 5056, 256, 256] + - [4, 3284.21] + - - [448, 1024, 1, 256, 448, 448, 256, 256] + - [1, 3248.79] + - - [64, 5888, 1, 256, 64, 64, 256, 256] + - [1, 2063.86] + - - [1408, 448, 1, 768, 1408, 1408, 768, 768] + - [4, 3911.36] + - - [6784, 3584, 1, 768, 6784, 6784, 768, 768] + - [2, 6278.91] + - - [4, 128, 1, 256, 4, 4, 256, 256] + - [7, 11.718] + - - [3584, 256, 1, 256, 3584, 3584, 256, 256] + - [1, 4302.32] + - - [3584, 128, 1, 1792, 3584, 3584, 1792, 1792] + - [1, 4150.27] + - - [704, 4288, 1, 768, 704, 704, 768, 768] + - [1, 5528.63] + - - [704, 4, 1, 3328, 704, 704, 3328, 3328] + - [7, 78.4826] + - - [4, 704, 1, 1792, 4, 4, 1792, 1792] + - [7, 75.4408] + - - [2368, 2368, 1, 3328, 2368, 2368, 3328, 3328] + - [1, 5856.11] + - - [1408, 1856, 1, 256, 1408, 1408, 256, 256] + - [6, 5547.4] + - - [4, 2944, 1, 256, 4, 4, 256, 256] + - [7, 154.155] + - - [448, 4288, 1, 3328, 448, 448, 3328, 3328] + - [1, 5350.95] + - - [704, 128, 1, 1792, 704, 704, 1792, 1792] + - [4, 1860.01] + - - [3584, 704, 1, 768, 3584, 3584, 768, 768] + - [6, 5773.38] + - - [256, 64, 1, 768, 256, 256, 768, 768] + - [5, 375.806] + - - [5056, 4288, 1, 768, 5056, 5056, 768, 768] + - [2, 6084.01] + - - [2368, 3584, 1, 3328, 2368, 2368, 3328, 3328] + - [2, 5855.28] + - - [5056, 2368, 1, 3328, 5056, 5056, 3328, 3328] + - [2, 6224.15] + - - [128, 256, 1, 1792, 128, 128, 1792, 1792] + - [5, 717.378] + - - [704, 6784, 1, 1792, 704, 704, 1792, 1792] + - [2, 5756.68] + - - [128, 2944, 1, 256, 128, 128, 256, 256] + - [4, 2913.41] + - - [1408, 2368, 1, 3328, 1408, 1408, 3328, 3328] + - [1, 5419.8] + - - [256, 6784, 1, 768, 256, 256, 768, 768] + - [1, 4658.65] + - - [2944, 4, 1, 256, 2944, 2944, 256, 256] + - [7, 166.791] + - - [4, 3584, 1, 256, 4, 4, 256, 256] + - [7, 172.027] + - - [256, 1856, 1, 1792, 256, 256, 1792, 1792] + - [1, 4268.18] + - - [2368, 4288, 1, 1792, 2368, 2368, 1792, 1792] + - [1, 5862.36] + - - [704, 128, 1, 768, 704, 704, 768, 768] + - [5, 1930.27] + - - [6784, 2368, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 6235.22] + - - [128, 5888, 1, 3328, 128, 128, 3328, 3328] + - [4, 4185.9] + - - [5056, 256, 1, 3328, 5056, 5056, 3328, 3328] + - [6, 4832.58] + - - [6784, 5888, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 6449.64] + - - [5888, 4288, 1, 768, 5888, 5888, 768, 768] + - [2, 6188.66] + - - [4, 2944, 1, 768, 4, 4, 768, 768] + - [7, 204.851] + - - [5056, 6784, 1, 3328, 5056, 5056, 3328, 3328] + - [2, 6383.72] + - - [64, 1856, 1, 768, 64, 64, 768, 768] + - [5, 1819.08] + - - [704, 2944, 1, 1792, 704, 704, 1792, 1792] + - [1, 5728.8] + - - [5056, 1856, 1, 1792, 5056, 5056, 1792, 1792] + - [2, 5788.13] + - - [1408, 448, 1, 256, 1408, 1408, 256, 256] + - [5, 3364.11] + - - [4288, 128, 1, 3328, 4288, 4288, 3328, 3328] + - [5, 3651.52] + - - [1024, 2944, 1, 256, 1024, 1024, 256, 256] + - [6, 5243.27] + - - [5056, 5056, 1, 1792, 5056, 5056, 1792, 1792] + - [2, 6389.1] + - - [6784, 128, 1, 1792, 6784, 6784, 1792, 1792] + - [1, 4642.35] + - - [256, 4, 1, 256, 256, 256, 256, 256] + - [7, 21.9809] + - - [256, 1856, 1, 768, 256, 256, 768, 768] + - [1, 4014.73] + - - [1408, 3584, 1, 1792, 1408, 1408, 1792, 1792] + - [2, 6089.2] + - - [2368, 1408, 1, 3328, 2368, 2368, 3328, 3328] + - [1, 5428.45] + - - [64, 448, 1, 1792, 64, 64, 1792, 1792] + - [4, 575.615] + - - [3584, 4, 1, 256, 3584, 3584, 256, 256] + - [7, 195.053] + - - [6784, 3584, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 6388.02] + - - [3584, 4288, 1, 3328, 3584, 3584, 3328, 3328] + - [2, 6397.39] + - - [1408, 1408, 1, 1792, 1408, 1408, 1792, 1792] + - [2, 5676.91] + - - [1408, 1408, 1, 768, 1408, 1408, 768, 768] + - [6, 5600.45] + - - [64, 1856, 1, 1792, 64, 64, 1792, 1792] + - [4, 2114.46] + - - [256, 5888, 1, 1792, 256, 256, 1792, 1792] + - [1, 5428.66] + - - [128, 64, 1, 1792, 128, 128, 1792, 1792] + - [5, 209.488] + - - [6784, 5056, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 6374.82] + - - [704, 3584, 1, 3328, 704, 704, 3328, 3328] + - [1, 5681.46] + - - [1408, 256, 1, 256, 1408, 1408, 256, 256] + - [5, 2903.68] + - - [2368, 1408, 1, 256, 2368, 2368, 256, 256] + - [6, 5283.12] + - - [704, 4, 1, 1792, 704, 704, 1792, 1792] + - [7, 75.7761] + - - [128, 64, 1, 3328, 128, 128, 3328, 3328] + - [4, 192.995] + - - [5056, 448, 1, 3328, 5056, 5056, 3328, 3328] + - [1, 5082.57] + - - [4, 2368, 1, 256, 4, 4, 256, 256] + - [7, 138.12] + - - [2368, 448, 1, 3328, 2368, 2368, 3328, 3328] + - [6, 4056.62] + - - [2368, 5888, 1, 3328, 2368, 2368, 3328, 3328] + - [2, 6231.65] + - - [5888, 256, 1, 1792, 5888, 5888, 1792, 1792] + - [6, 5471.02] + - - [448, 1856, 1, 1792, 448, 448, 1792, 1792] + - [6, 4410.18] + - - [4288, 5888, 1, 768, 4288, 4288, 768, 768] + - [2, 6243.72] + - - [448, 256, 1, 256, 448, 448, 256, 256] + - [4, 1777.35] + - - [4288, 4288, 1, 768, 4288, 4288, 768, 768] + - [6, 5986.59] + - - [256, 6784, 1, 1792, 256, 256, 1792, 1792] + - [2, 4935.77] + - - [256, 256, 1, 1792, 256, 256, 1792, 1792] + - [5, 1382.22] + - - [6784, 1856, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 6004.62] + - - [1856, 5056, 1, 768, 1856, 1856, 768, 768] + - [6, 5957.39] + - - [1408, 64, 1, 256, 1408, 1408, 256, 256] + - [4, 1428.53] + - - [6784, 5056, 1, 768, 6784, 6784, 768, 768] + - [2, 6264.31] + - - [704, 2368, 1, 768, 704, 704, 768, 768] + - [6, 4647.42] + - - [4288, 3584, 1, 3328, 4288, 4288, 3328, 3328] + - [2, 6415.87] + - - [2368, 1408, 1, 1792, 2368, 2368, 1792, 1792] + - [1, 5404.91] + - - [4288, 1408, 1, 3328, 4288, 4288, 3328, 3328] + - [2, 6170.29] + - - [5056, 128, 1, 1792, 5056, 5056, 1792, 1792] + - [4, 3933.64] + - - [1408, 1856, 1, 3328, 1408, 1408, 3328, 3328] + - [2, 5170.2] + - - [6784, 4288, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 6297.74] + - - [2944, 256, 1, 768, 2944, 2944, 768, 768] + - [5, 4065.45] + - - [256, 3584, 1, 768, 256, 256, 768, 768] + - [1, 4702.15] + - - [5056, 5056, 1, 256, 5056, 5056, 256, 256] + - [6, 5945.48] + - - [5056, 6784, 1, 256, 5056, 5056, 256, 256] + - [2, 6005.24] + - - [1024, 448, 1, 1792, 1024, 1024, 1792, 1792] + - [1, 4057.72] + - - [4, 5888, 1, 3328, 4, 4, 3328, 3328] + - [7, 169.597] + - - [3584, 3584, 1, 1792, 3584, 3584, 1792, 1792] + - [2, 6115.12] + - - [5888, 64, 1, 256, 5888, 5888, 256, 256] + - [4, 2718.78] + - - [5888, 4, 1, 256, 5888, 5888, 256, 256] + - [7, 290.688] + - - [6784, 4, 1, 256, 6784, 6784, 256, 256] + - [7, 310.528] + - - [2944, 64, 1, 768, 2944, 2944, 768, 768] + - [5, 2790.62] + - - [256, 4288, 1, 1792, 256, 256, 1792, 1792] + - [5, 4090.51] + - - [448, 2368, 1, 1792, 448, 448, 1792, 1792] + - [4, 4011.22] + - - [5888, 1856, 1, 768, 5888, 5888, 768, 768] + - [2, 5987.73] + - - [1408, 4, 1, 1792, 1408, 1408, 1792, 1792] + - [7, 148.578] + - - [128, 448, 1, 3328, 128, 128, 3328, 3328] + - [4, 1187.78] + - - [64, 5056, 1, 768, 64, 64, 768, 768] + - [1, 2558.99] + - - [1024, 1856, 1, 3328, 1024, 1024, 3328, 3328] + - [2, 5495.97] + - - [128, 1024, 1, 256, 128, 128, 256, 256] + - [4, 2058.93] + - - [1024, 256, 1, 768, 1024, 1024, 768, 768] + - [5, 3743.59] + - - [4288, 1856, 1, 256, 4288, 4288, 256, 256] + - [6, 5737.16] + - - [6784, 256, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 5008.14] + - - [704, 6784, 1, 256, 704, 704, 256, 256] + - [1, 5151.44] + - - [4288, 448, 1, 256, 4288, 4288, 256, 256] + - [6, 4932.3] + - - [128, 704, 1, 1792, 128, 128, 1792, 1792] + - [4, 1644.0] + - - [1024, 2944, 1, 1792, 1024, 1024, 1792, 1792] + - [2, 5853.96] + - - [128, 4, 1, 1792, 128, 128, 1792, 1792] + - [7, 14.1554] + - - [64, 5888, 1, 3328, 64, 64, 3328, 3328] + - [4, 3356.44] + - - [64, 4288, 1, 3328, 64, 64, 3328, 3328] + - [4, 2485.76] + - - [2944, 1408, 1, 256, 2944, 2944, 256, 256] + - [6, 5680.13] + - - [1408, 3584, 1, 3328, 1408, 1408, 3328, 3328] + - [2, 6149.32] + - - [64, 256, 1, 256, 64, 64, 256, 256] + - [4, 288.884] + - - [1856, 256, 1, 1792, 1856, 1856, 1792, 1792] + - [1, 4287.27] + - - [3584, 1856, 1, 3328, 3584, 3584, 3328, 3328] + - [2, 5843.51] + - - [5056, 256, 1, 1792, 5056, 5056, 1792, 1792] + - [6, 4826.85] + - - [448, 128, 1, 1792, 448, 448, 1792, 1792] + - [5, 1214.73] + - - [6784, 448, 1, 1792, 6784, 6784, 1792, 1792] + - [6, 5466.0] + - - [3584, 128, 1, 256, 3584, 3584, 256, 256] + - [6, 3268.86] + - - [4, 4, 1, 1792, 4, 4, 1792, 1792] + - [7, 0.420719] + - - [1408, 2368, 1, 256, 1408, 1408, 256, 256] + - [6, 5292.83] + - - [4288, 4, 1, 3328, 4288, 4288, 3328, 3328] + - [7, 158.72] + - - [4, 4, 1, 3328, 4, 4, 3328, 3328] + - [7, 0.44372] + - - [256, 64, 1, 1792, 256, 256, 1792, 1792] + - [4, 401.572] + - - [1408, 256, 1, 768, 1408, 1408, 768, 768] + - [4, 3425.32] + - - [1856, 64, 1, 768, 1856, 1856, 768, 768] + - [4, 2156.8] + - - [128, 1856, 1, 768, 128, 128, 768, 768] + - [4, 3314.99] + - - [5056, 4288, 1, 3328, 5056, 5056, 3328, 3328] + - [2, 6207.11] + - - [3584, 5888, 1, 256, 3584, 3584, 256, 256] + - [6, 5853.49] + - - [256, 1408, 1, 768, 256, 256, 768, 768] + - [4, 3406.6] + - - [1856, 4288, 1, 1792, 1856, 1856, 1792, 1792] + - [2, 6100.37] + - - [128, 256, 1, 3328, 128, 128, 3328, 3328] + - [4, 738.296] + - - [1408, 6784, 1, 3328, 1408, 1408, 3328, 3328] + - [2, 5948.56] + - - [448, 5888, 1, 1792, 448, 448, 1792, 1792] + - [2, 5128.39] + - - [448, 6784, 1, 3328, 448, 448, 3328, 3328] + - [1, 5741.2] + - - [5888, 2944, 1, 1792, 5888, 5888, 1792, 1792] + - [2, 6347.19] + - - [704, 5888, 1, 256, 704, 704, 256, 256] + - [1, 5478.12] + - - [5888, 2368, 1, 1792, 5888, 5888, 1792, 1792] + - [2, 6169.88] + - - [4, 5888, 1, 768, 4, 4, 768, 768] + - [7, 240.335] + - - [6784, 704, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 5737.52] + - - [128, 1024, 1, 1792, 128, 128, 1792, 1792] + - [4, 2208.09] + - - [1408, 4288, 1, 768, 1408, 1408, 768, 768] + - [6, 6046.09] + - - [5888, 64, 1, 768, 5888, 5888, 768, 768] + - [4, 3310.93] + - - [704, 256, 1, 3328, 704, 704, 3328, 3328] + - [5, 2992.19] + - - [256, 1408, 1, 1792, 256, 256, 1792, 1792] + - [4, 3483.71] + - - [5888, 4288, 1, 256, 5888, 5888, 256, 256] + - [2, 5840.68] + - - [1856, 5888, 1, 1792, 1856, 1856, 1792, 1792] + - [2, 6140.49] + - - [1408, 2944, 1, 768, 1408, 1408, 768, 768] + - [2, 5995.51] + - - [5888, 5056, 1, 1792, 5888, 5888, 1792, 1792] + - [2, 6413.6] + - - [1856, 6784, 1, 768, 1856, 1856, 768, 768] + - [6, 5944.5] + - - [2944, 5056, 1, 768, 2944, 2944, 768, 768] + - [6, 6047.59] + - - [1408, 5056, 1, 1792, 1408, 1408, 1792, 1792] + - [2, 6190.77] + - - [1024, 5056, 1, 1792, 1024, 1024, 1792, 1792] + - [2, 6212.04] + - - [1408, 6784, 1, 768, 1408, 1408, 768, 768] + - [6, 6038.86] + - - [4288, 3584, 1, 256, 4288, 4288, 256, 256] + - [6, 5914.38] + - - [128, 1024, 1, 768, 128, 128, 768, 768] + - [4, 2283.88] + - - [3584, 256, 1, 768, 3584, 3584, 768, 768] + - [6, 4741.99] + - - [4, 6784, 1, 256, 4, 4, 256, 256] + - [7, 203.427] + - - [6784, 448, 1, 3328, 6784, 6784, 3328, 3328] + - [6, 4984.23] + - - [2368, 64, 1, 256, 2368, 2368, 256, 256] + - [5, 1844.2] + - - [6784, 6784, 1, 256, 6784, 6784, 256, 256] + - [2, 6189.21] + - - [256, 704, 1, 768, 256, 256, 768, 768] + - [5, 2616.98] + - - [128, 4288, 1, 3328, 128, 128, 3328, 3328] + - [4, 3565.46] + - - [6784, 6784, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 6517.92] + - - [64, 256, 1, 3328, 64, 64, 3328, 3328] + - [4, 330.823] + - - [256, 1408, 1, 3328, 256, 256, 3328, 3328] + - [4, 3530.37] + - - [6784, 5056, 1, 256, 6784, 6784, 256, 256] + - [2, 5923.61] + - - [64, 1856, 1, 3328, 64, 64, 3328, 3328] + - [4, 2162.31] + - - [448, 128, 1, 3328, 448, 448, 3328, 3328] + - [5, 1212.37] + - - [4, 2944, 1, 1792, 4, 4, 1792, 1792] + - [7, 222.733] + - - [4288, 5056, 1, 3328, 4288, 4288, 3328, 3328] + - [2, 6203.12] + - - [448, 4288, 1, 768, 448, 448, 768, 768] + - [1, 5101.61] + - - [3584, 448, 1, 768, 3584, 3584, 768, 768] + - [6, 4493.05] + - - [448, 5888, 1, 768, 448, 448, 768, 768] + - [6, 4988.74] + - - [6784, 1024, 1, 1792, 6784, 6784, 1792, 1792] + - [2, 6059.24] + - - [128, 4, 1, 768, 128, 128, 768, 768] + - [7, 15.5213] + - - [256, 4, 1, 768, 256, 256, 768, 768] + - [7, 30.075] + - - [5056, 6784, 1, 1792, 5056, 5056, 1792, 1792] + - [2, 6342.39] + - - [1408, 704, 1, 1792, 1408, 1408, 1792, 1792] + - [6, 5120.49] + - - [6784, 704, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 5809.33] + - - [256, 1408, 1, 256, 256, 256, 256, 256] + - [5, 2917.32] + - - [4, 5888, 1, 1792, 4, 4, 1792, 1792] + - [7, 166.206] + - - [64, 448, 1, 768, 64, 64, 768, 768] + - [5, 650.472] + - - [5056, 5888, 1, 768, 5056, 5056, 768, 768] + - [2, 6347.71] + - - [4, 448, 1, 768, 4, 4, 768, 768] + - [7, 47.5169] + - - [256, 448, 1, 3328, 256, 256, 3328, 3328] + - [4, 2424.74] + - - [2944, 1024, 1, 768, 2944, 2944, 768, 768] + - [6, 5845.37] + - - [3584, 64, 1, 1792, 3584, 3584, 1792, 1792] + - [5, 3596.18] + - - [2944, 5888, 1, 3328, 2944, 2944, 3328, 3328] + - [2, 6395.98] + - - [4288, 5056, 1, 256, 4288, 4288, 256, 256] + - [6, 6005.98] + - - [64, 5056, 1, 256, 64, 64, 256, 256] + - [1, 1876.32] + - - [5888, 1024, 1, 3328, 5888, 5888, 3328, 3328] + - [2, 6149.33] + - - [4, 1856, 1, 1792, 4, 4, 1792, 1792] + - [7, 172.027] + - - [3584, 2944, 1, 3328, 3584, 3584, 3328, 3328] + - [2, 5974.67] + - - [5056, 1024, 1, 1792, 5056, 5056, 1792, 1792] + - [2, 6230.27] + - - [1024, 4288, 1, 3328, 1024, 1024, 3328, 3328] + - [1, 5561.6] + - - [2944, 704, 1, 3328, 2944, 2944, 3328, 3328] + - [1, 4646.19] + - - [1408, 1856, 1, 1792, 1408, 1408, 1792, 1792] + - [6, 5802.48] + - - [5056, 4288, 1, 256, 5056, 5056, 256, 256] + - [6, 5968.66] + - - [448, 64, 1, 3328, 448, 448, 3328, 3328] + - [5, 606.185] + - - [1024, 2368, 1, 256, 1024, 1024, 256, 256] + - [6, 4997.04] + - - [448, 448, 1, 1792, 448, 448, 1792, 1792] + - [5, 3223.99] + - - [4, 4, 1, 768, 4, 4, 768, 768] + - [7, 0.403613] + - - [2368, 448, 1, 256, 2368, 2368, 256, 256] + - [6, 3604.96] + - - [1408, 1024, 1, 768, 1408, 1408, 768, 768] + - [1, 5100.0] + - - [64, 5056, 1, 1792, 64, 64, 1792, 1792] + - [1, 2795.69] + - - [1408, 448, 1, 3328, 1408, 1408, 3328, 3328] + - [4, 4110.69] + - - [6784, 4288, 1, 3328, 6784, 6784, 3328, 3328] + - [2, 6332.74] + - - [4288, 704, 1, 768, 4288, 4288, 768, 768] + - [6, 5789.42] + - - [1024, 704, 1, 1792, 1024, 1024, 1792, 1792] + - [5, 4031.32] + - - [6784, 4288, 1, 256, 6784, 6784, 256, 256] + - [2, 5879.74] + - - [704, 1408, 1, 3328, 704, 704, 3328, 3328] + - [1, 5333.92] + - - [64, 2368, 1, 1792, 64, 64, 1792, 1792] + - [5, 2365.32] + - - [4288, 1408, 1, 1792, 4288, 4288, 1792, 1792] + - [2, 6114.95] + - - [256, 448, 1, 1792, 256, 256, 1792, 1792] + - [5, 2292.94] + - - [704, 2368, 1, 1792, 704, 704, 1792, 1792] + - [6, 4762.49] + - - [1024, 4, 1, 1792, 1024, 1024, 1792, 1792] + - [7, 106.431] + - - [704, 3584, 1, 768, 704, 704, 768, 768] + - [1, 5499.1] +- null diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPU.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPU.hpp new file mode 100644 index 0000000000..4b91ba9dcd --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPU.hpp @@ -0,0 +1,116 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +namespace Tensile +{ + /** + * \ingroup Hardware + * Represents a particular AMD GPU in terms of processor model and number of + * compute units. + * + * See subclass in `hip` directory which can create an instance + * automatically. + */ + struct TENSILE_API AMDGPU : public Hardware + { + static std::string Type() + { + return "AMDGPU"; + } + virtual std::string type() const; + + enum class Processor : int + { + // matching enum used in hipGcnArch + // only including supported types + //gfx000 = 0, + //gfx701 = 1, + //gfx801 = 2, + //gfx802 = 3, + gfx803 = 803, + gfx900 = 900, + gfx906 = 906, + gfx908 = 908, + gfx90a = 910, + gfx1010 = 1010, + gfx1011 = 1011, + gfx1012 = 1012, + gfx1030 = 1030, + gfx1100 = 1100, + gfx1101 = 1101, + gfx1102 = 1102 + }; + + AMDGPU(); + AMDGPU(Processor p, int computeUnitCount, std::string const& deviceName); + ~AMDGPU(); + + Processor processor = Processor::gfx900; + int wavefrontSize = 64; + int simdPerCu = 4; + int computeUnitCount = 0; + std::string deviceName; + + virtual bool runsKernelTargeting(Processor p) const; + virtual size_t id() const + { + return (size_t)processor; + } + virtual std::string description() const; + + bool operator==(AMDGPU const& rhs) const + { + return processor == rhs.processor && computeUnitCount == rhs.computeUnitCount; + } + }; + + inline bool operator<(AMDGPU::Processor l, AMDGPU::Processor r) + { + return static_cast(l) < static_cast(r); + } + + inline bool operator>(AMDGPU::Processor l, AMDGPU::Processor r) + { + return static_cast(l) > static_cast(r); + } + + inline bool operator<=(AMDGPU::Processor l, AMDGPU::Processor r) + { + return static_cast(l) <= static_cast(r); + } + + inline bool operator>=(AMDGPU::Processor l, AMDGPU::Processor r) + { + return static_cast(l) >= static_cast(r); + } + + TENSILE_API std::ostream& operator<<(std::ostream& stream, AMDGPU::Processor p); + TENSILE_API std::ostream& operator<<(std::ostream& stream, AMDGPU g); +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPUPredicates.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPUPredicates.hpp new file mode 100644 index 0000000000..0fe1b92807 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPUPredicates.hpp @@ -0,0 +1,130 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include + +namespace Tensile +{ + namespace Predicates + { + /** + * \addtogroup Predicates + * @{ + */ + /** + * @brief GPU Predicates + */ + namespace GPU + { + struct ProcessorEqual : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + AMDGPU::Processor value; + + ProcessorEqual() = default; + ProcessorEqual(AMDGPU::Processor p) + : value(p) + { + } + + static std::string Type() + { + return "Processor"; + } + + virtual bool operator()(AMDGPU const& gpu) const + { + return gpu.processor == value; + } + }; + + struct CUCountEqual : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + int value; + + CUCountEqual() = default; + CUCountEqual(int val) + : value(val) + { + } + + static std::string Type() + { + return "CUCount"; + } + + virtual bool operator()(AMDGPU const& gpu) const + { + return gpu.computeUnitCount == value; + } + }; + + struct RunsKernelTargeting : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + AMDGPU::Processor value; + + RunsKernelTargeting() = default; + RunsKernelTargeting(AMDGPU::Processor p) + : value(p) + { + } + + static std::string Type() + { + return "TargetProcessor"; + } + + virtual bool operator()(AMDGPU const& gpu) const + { + return gpu.runsKernelTargeting(value); + } + }; + } // namespace GPU + + /** + * @} + */ + } // namespace Predicates +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPU_Detail.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPU_Detail.hpp new file mode 100644 index 0000000000..01a13c81d5 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPU_Detail.hpp @@ -0,0 +1,42 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +namespace std +{ + template <> + struct hash + { + inline size_t operator()(Tensile::AMDGPU const& gpu) const + { + return Tensile::hash_combine(static_cast(gpu.processor), gpu.computeUnitCount); + } + }; +} // namespace std diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Activation.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Activation.hpp new file mode 100644 index 0000000000..0409d29b6c --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Activation.hpp @@ -0,0 +1,73 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \defgroup ActivationTypes Activation Type Info + * + * @brief Definitions and metadata on supported activation types. + */ + + /** + * \ingroup ActivationTypes + * @{ + */ + + /** + * Data Type + */ + enum class ActivationType : uint32_t + { + Abs = 0, + Clippedrelu, + Gelu, + Leakyrelu, + Relu, + Sigmoid, + Tanh, + None, + All, + Exp, // Verification use only. + Count + }; + + std::string ToString(ActivationType d); + std::ostream& operator<<(std::ostream& stream, const ActivationType& t); + std::istream& operator>>(std::istream& stream, ActivationType& t); + + int getAdditionalArgNum(ActivationType d); +} diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ArithmeticUnitTypes.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ArithmeticUnitTypes.hpp new file mode 100644 index 0000000000..95d2c3edff --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ArithmeticUnitTypes.hpp @@ -0,0 +1,147 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \defgroup ArithmeticUnits Arithmetic unit type Info + * + * @brief Definitions and metadata on supported arithmetic unit types. + */ + + /** + * \ingroup ArithmeticUnits + * @{ + */ + + /** + * Arithmetic Unit + */ + enum class ArithmeticUnit : int + { + Any, + MFMA, + VALU, + Count + }; + + std::string ToString(ArithmeticUnit d); + std::string TypeAbbrev(ArithmeticUnit d); + std::ostream& operator<<(std::ostream& stream, ArithmeticUnit const& t); + std::istream& operator>>(std::istream& stream, ArithmeticUnit& t); + + /** + * \ingroup ArithmeticUnits + * \brief Runtime accessible arithmetic unit type metadata + */ + struct ArithmeticUnitTypeInfo + { + static ArithmeticUnitTypeInfo const& Get(int index); + static ArithmeticUnitTypeInfo const& Get(ArithmeticUnit t); + static ArithmeticUnitTypeInfo const& Get(std::string const& str); + + ArithmeticUnit m_arithmeticUnit; + std::string name; + + private: + static void registerAllTypeInfo(); + static void registerAllTypeInfoOnce(); + + template + static void registerTypeInfo(); + + static void addInfoObject(ArithmeticUnitTypeInfo const& info); + + static std::map data; + static std::map typeNames; + }; + + /** + * \ingroup ArithmeticUnits + * \brief Compile-time accessible arithmetic unit type metadata. + */ + template + struct ArithmeticUnitInfo; + + template + struct BaseArithmeticUnitInfo + { + constexpr static ArithmeticUnit Enum = T_Enum; + + static inline std::string Name() + { + return ToString(Enum); + } + static inline std::string Abbrev() + { + return TypeAbbrev(Enum); + } + }; + + template + constexpr ArithmeticUnit BaseArithmeticUnitInfo::Enum; + + template <> + struct ArithmeticUnitInfo + : public BaseArithmeticUnitInfo + { + }; + template <> + struct ArithmeticUnitInfo + : public BaseArithmeticUnitInfo + { + }; + template <> + struct ArithmeticUnitInfo + : public BaseArithmeticUnitInfo + { + }; + + /** + * @} + */ +} // namespace Tensile + +namespace std +{ + template <> + struct hash + { + inline size_t operator()(Tensile::ArithmeticUnit const& val) const + { + return hash()(static_cast(val)); + } + }; +} // namespace std diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/CachingLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/CachingLibrary.hpp new file mode 100644 index 0000000000..d256d38595 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/CachingLibrary.hpp @@ -0,0 +1,244 @@ + +/******************************************************************************* + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + *******************************************************************************/ + +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include +#include + +namespace Tensile +{ + template + struct MultiLevelMap + { + using type = typename MultiLevelMap, Keys...>::type; + }; + + template + struct MultiLevelMap + { + using type = std::unordered_map; + }; + + /** + * Thread-safe multi-valued cache. + * + * Note that due to a quirk with templates, the order of the keys in find() and add() is *opposite* of that in the type. + * + * e.g. + * + * CacheMap myCache + * myCache.find("foo", 1.4); // great + * myCache.find(1.4, "foo"); // error! + */ + template + class CacheMap + { + using Map = typename MultiLevelMap::type; + + public: + CacheMap(Value const& nullValue) + : m_nullValue(nullValue) + , m_lookupEfficiency(Debug::Instance().printLookupEfficiency()) + , m_lookups(0) + , m_hits(0) + + { + } + + ~CacheMap() + { + if(m_lookupEfficiency) + std::cout << "CacheMap: " << m_hits << "/" << m_lookups << " cache hits" + << std::endl; + } + + template + Value find(Ks const&... keys) + { + std::shared_lock lock(m_mutex); + + auto rv = find_impl(m_map, keys...); + + if(m_lookupEfficiency) + { + m_lookups++; + if(rv != m_nullValue) + m_hits++; + } + + return rv; + } + + template + void add(Value const& value, Ks const&... ks) + { + std::lock_guard lock(m_mutex); + + add_impl(m_map, value, ks...); + } + + private: + template + Value find_impl(SubMap const& map, K const& key) + { + auto iter = map.find(key); + + if(iter == map.end()) + return m_nullValue; + + return iter->second; + } + + template + Value find_impl(SubMap const& map, K const& key, Ks const&... ks) + { + auto iter = map.find(key); + + if(iter == map.end()) + return m_nullValue; + + return find_impl(iter->second, ks...); + } + + template + void add_impl(SubMap& map, Value const& value, K const& key) + { + map.emplace(key, value); + } + + template + void add_impl(SubMap& map, Value const& value, K const& key, Ks const&... ks) + { + add_impl(map[key], value, ks...); + } + + Map m_map; + std::shared_timed_mutex m_mutex; + Value m_nullValue; + + bool m_lookupEfficiency; + std::atomic m_lookups; + std::atomic m_hits; + }; + + template + class CachingLibrary : public SolutionLibrary + { + public: + using Library = SolutionLibrary; + using Cache = CacheMap, double>, AMDGPU, MyProblem>; + + CachingLibrary(std::shared_ptr subLibrary) + : m_subLibrary(subLibrary) + , m_cache(std::make_tuple(nullptr, std::numeric_limits::max())) + { + } + + virtual std::shared_ptr findBestSolution(MyProblem const& problem, + Hardware const& hardware, + double* fitness + = nullptr) const override + { + try + { + double cachedFitness = std::numeric_limits::max(); + fitness = (fitness) ? fitness : &cachedFitness; + + auto const& amdgpu = dynamic_cast(hardware); + std::shared_ptr solution; + std::tie(solution, *fitness) = m_cache.find(problem, amdgpu); + + if(solution) + return solution; + + solution = m_subLibrary->findBestSolution(problem, hardware, fitness); + if(solution) + m_cache.add(std::make_tuple(solution, *fitness), problem, amdgpu); + + return solution; + } + catch(std::bad_cast const& exc) + { + return m_subLibrary->findBestSolution(problem, hardware, fitness); + } + } + + virtual SolutionSet findAllSolutions(MyProblem const& problem, + Hardware const& hardware) const override + { + return m_subLibrary->findAllSolutions(problem, hardware); + } + + std::shared_ptr findSolutionInCache(MyProblem const& problem, + Hardware const& hardware) const + { + auto const& amdgpu = dynamic_cast(hardware); + + return std::get>(m_cache.find(problem, amdgpu)); + } + + virtual std::string type() const override + { + return "Caching Library"; + } + virtual std::string description() const override + { + return "Caching Library"; + } + + std::shared_ptr library() const + { + return m_subLibrary; + } + + private: + std::shared_ptr m_subLibrary; + mutable Cache m_cache; + }; + +#if 0 + struct ContractionCachingLibrary: public CachingLibrary + { + using Super = CachingLibrary; + using Library = typename Super::Library; + using Key = typename Super::Key; + + ContractionCachingLibrary(std::shared_ptr subLibrary) + : CachingLibrary(subLibrary) + {} + + }; +#endif + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Comparison.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Comparison.hpp new file mode 100644 index 0000000000..d9d2041292 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Comparison.hpp @@ -0,0 +1,233 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +namespace Tensile +{ + /** + * \ingroup Utilities + * \defgroup Comparison Comparison + */ + + /** + * \addtogroup Comparison + * @{ + */ + + /** + * Lexicographically compares two lists of values, one pair of values at a + * time. + * + * \return -1: The first list is lesser. + * 1: The second list is lesser. + * 0: The lists are equal. + */ + inline int LexicographicCompare() + { + return 0; + } + + template + inline int LexicographicCompare(A const& lhs, A const& rhs) + { + if(lhs < rhs) + return -1; + if(lhs > rhs) + return 1; + return 0; + } + + template > + inline int LexicographicCompare(A const& lhs, A const& rhs, Args const&... rest) + { + if(lhs < rhs) + return -1; + if(lhs > rhs) + return 1; + return LexicographicCompare(rest...); + } + + /** + * @brief Traits class which enables comparison operators to be implemented + * based on a `compare()` function. + * + * Specializations must implement a `compare(lhs, rhs)` function which returns: + * - -1: lhs is lesser + * - 0: values are equal + * - 1: rhs is lesser + */ + template + struct Comparison + { + enum + { + implemented = false + }; + }; + + template ::implemented>::type> + inline bool operator==(T const& lhs, U const& rhs) + { + return Comparison::compare(lhs, rhs) == 0; + } + + template ::implemented>::type> + inline bool operator!=(T const& lhs, U const& rhs) + { + return Comparison::compare(lhs, rhs) != 0; + } + + template ::implemented>::type> + inline bool operator<(T const& lhs, U const& rhs) + { + return Comparison::compare(lhs, rhs) < 0; + } + + template ::implemented>::type> + inline bool operator<=(T const& lhs, U const& rhs) + { + return Comparison::compare(lhs, rhs) <= 0; + } + + template ::implemented>::type> + inline bool operator>(T const& lhs, U const& rhs) + { + return Comparison::compare(lhs, rhs) > 0; + } + + template ::implemented>::type> + inline bool operator>=(T const& lhs, U const& rhs) + { + return Comparison::compare(lhs, rhs) >= 0; + } + + /** + * Combines a number of already-hashed values. + */ + template + inline size_t combine_hashes(size_t a, Ts... rest) + { + return combine_hashes(a, combine_hashes(rest...)); + } + + template <> + inline size_t combine_hashes(size_t a, size_t b) + { + return b ^ (a + 0x9b9773e99e3779b9 + (b << 6) + (b >> 2)); + } + + template + inline size_t hash_combine(T const& val) + { + return std::hash()(val); + } + + template + inline size_t hash_combine(T const& val, Ts const&... more) + { + size_t mine = std::hash()(val); + size_t rest = hash_combine(more...); + + return combine_hashes(mine, rest); + } + + template + inline size_t hash_combine_iter(Iter begin, Iter end) + { + size_t rv = 0; + while(begin != end) + { + rv = combine_hashes(hash_combine(*begin), rv); + begin++; + } + + return rv; + } + + template + struct tuple_hash + { + using MyTuple = std::tuple; + using TypeN = typename std::tuple_element::type; + static size_t apply(std::tuple const& tup) + { + size_t mine = std::hash()(std::get(tup)); + size_t rest = tuple_hash::apply(tup); + + return combine_hashes(mine, rest); + } + }; + + template + struct tuple_hash<0, Types...> + { + using MyTuple = std::tuple; + using Type0 = typename std::tuple_element<0, MyTuple>::type; + static size_t apply(std::tuple const& tup) + { + return std::hash()(std::get<0>(tup)); + } + }; + + template + size_t hash_tuple(std::tuple const& tup) + { + return tuple_hash::apply(tup); + } + + /** + * @} + */ +} // namespace Tensile + +namespace std +{ + template + struct hash> + { + inline size_t operator()(tuple const& tup) const + { + return Tensile::hash_tuple(tup); + } + }; +} // namespace std diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionLibrary.hpp new file mode 100644 index 0000000000..09f241ace9 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionLibrary.hpp @@ -0,0 +1,54 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Tensile +{ + using ContractionLibrary = SolutionLibrary; + using MasterContractionLibrary = MasterSolutionLibrary; + using SingleContractionLibrary = SingleSolutionLibrary; + using ContractionHardwareSelectionLibrary + = HardwareSelectionLibrary; + using ContractionProblemSelectionLibrary + = ProblemSelectionLibrary; + using ContractionProblemMatchingLibrary + = ProblemMatchingLibrary; + using ContractionProblemMapLibrary = ProblemMapLibrary; + using ContractionProblemPredicate = ProblemPredicate; + using ContractionGranularitySelectionLibrary + = GranularitySelectionLibrary; +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem.hpp new file mode 100644 index 0000000000..0111a16452 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem.hpp @@ -0,0 +1,1035 @@ +/** + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +namespace Tensile +{ + /** + * \addtogroup Problem + * @{ + */ + + /** + * Describes a tensor contraction in by using TensorDescriptor objects for + * each input or output tensor as well as indices describing transposes, + * summations, etc. This is decoupled from any particular pointers, which + * are provided in ContractionInputs objects. + */ + class TENSILE_API ContractionProblem : public Problem + { + public: + using Solution = ContractionSolution; + using Inputs = ContractionInputs; + + ContractionProblem() = default; + + /** + * Zero-padding description + */ + struct ZeroPad + { + ZeroPad(int32_t ai = -1, int32_t bi = -1, int64_t ps = 0, int64_t pe = 0) + : anchorIndex(ai) + , anchorPos(-1) + , boundIndex(bi) + , padStart(ps) + , padEnd(pe){}; + + int32_t anchorIndex; + int32_t anchorPos; //! position of anchorIndex in A or B tensor + int32_t boundIndex; + int32_t boundPos; //! position of anchroIndex in A or B tensor + int64_t padStart; + int64_t padEnd; + + bool valid() const + { + return anchorIndex != -1; + }; + std::string description() const; + }; + using ZeroPads = std::vector; + + /** + * Represents a pair of free indices in a tensor contraction. + */ + struct FreeIndex + { + bool isA; //< True=index is in A; False=index is in B + size_t i; //< Dimension in A or B (depending on isA) + size_t c; //< Dimension of C which corresponds for this index + size_t d; //< Dimension of D which corresponds for this index + }; + using FreeIndices = std::vector; + + /** + * Represents a batched index in a tensor contraction. + */ + struct BatchIndex + { + size_t a, b, c, d; + }; + using BatchIndices = std::vector; + + /** + * Represents a bound (or summed) index in a tensor contraction. + */ + struct BoundIndex + { + BoundIndex(size_t xa = 0, size_t xb = 0, bool aMirror = false, bool bMirror = false) + : a(xa) + , b(xb) + , aMirror(aMirror) + , bMirror(bMirror){}; + size_t a, b; //! positions in a or b tensor + ZeroPad aZeroPad; + ZeroPad bZeroPad; + bool aMirror, bMirror; + }; + using BoundIndices = std::vector; + + virtual std::string description() const; + + /** + * Create a ContractionProblem representing a batched GEMM, specifying + * strides between matrices. + */ + static ContractionProblem GEMM_Strides(bool transA, + bool transB, + DataType aType, + DataType bType, + DataType cType, + DataType dType, + size_t m, + size_t n, + size_t k, + size_t batchSize, + size_t lda, + size_t aStride, + size_t ldb, + size_t bStride, + size_t ldc, + size_t cStride, + size_t ldd, + size_t dStride, + double beta); + + /** + * Create a ContractionProblem representing a batched GEMM, specifying + * strides between matrices. + */ + static ContractionProblem GEMM_Strides(bool transA, + bool transB, + DataType aType, + DataType bType, + DataType cType, + DataType dType, + size_t m, + size_t n, + size_t k, + size_t batchSize, + size_t lda, + size_t aStride, + size_t aOffset, + size_t ldb, + size_t bStride, + size_t bOffset, + size_t ldc, + size_t cStride, + size_t cOffset, + size_t ldd, + size_t dStride, + size_t dOffset, + double beta); + + /** + * Create a ContractionProblem representing a batched SGEMM, with + * leading dimensions, but no strides. + */ + static ContractionProblem GEMM(bool transA, + bool transB, + size_t m, + size_t n, + size_t k, + size_t lda, + size_t ldb, + size_t ldc, + double beta, + bool unused, + size_t batchCount); + + /** + * Create a ContractionProblem representing a batched SGEMM, with + * leading dimensions, but no strides. + */ + static ContractionProblem GEMM(bool transA, + bool transB, + size_t m, + size_t n, + size_t k, + size_t lda, + size_t offsetA, + size_t ldb, + size_t offsetB, + size_t ldc, + size_t offsetC, + double beta, + bool unused, + size_t batchCount); + + /** + * Create a ContractionProblem representing a batched GEMM based on the + * dimensions of each of the tensors. + */ + static ContractionProblem GEMM(bool transA, + bool transB, + TensorDescriptor const& a, + TensorOps const& aOps, + TensorDescriptor const& b, + TensorOps const& bOps, + TensorDescriptor const& c, + TensorOps const& cOps, + TensorDescriptor const& d, + TensorOps const& dOps, + double beta); + + /** + * Converts an identifier such as `Contraction_l_AlikC_Bjlk_Cijk_Dijk` + * into a set of indices and operations. + */ + static void IdentifierToIndices(std::string const& identifier, + FreeIndices& freeIndices, + BatchIndices& batchIndices, + BoundIndices& boundIndices, + TensorOps& aOps, + TensorOps& bOps, + TensorOps& cOps, + TensorOps& dOps); + + /** + * Create a ContractionProblem from a definition of each index, the + * size of each index, the strides of each tensor, and any operations. + * + * @param freeIndices Free indices + * @param batchIndices Batch indices + * @param boundIndices Bound indices + * @param indexSizes Size of each index, in the order of appearance in + * the D tensor. + * + * @param aType Data type of A + * @param aStrides Strides of A + * @param aOps Operations to apply to A as it is read + * + * @param bType Data type of B + * @param bStrides Strides of B + * @param bOps Operations to apply to B as it is read + * + * @param cType Data type of C + * @param cStrides Strides of C + * @param cOps Operations to apply to C as it is read + * + * @param dType Data type of D + * @param dStrides Strides of D + * @param dOps Operations to apply to D as it is read + * + * @param beta Representative value of beta. Is only used to possibly + * select a more efficient kernel if we know that + * `beta == 0` or `beta == 1`. + */ + static ContractionProblem FromIndexSizes(FreeIndices const& freeIndices, + BatchIndices const& batchIndices, + BoundIndices const& boundIndices, + std::vector const& indexSizes, + DataType aType, + std::vector const& aStrides, + TensorOps const& aOps, + DataType bType, + std::vector const& bStrides, + TensorOps const& bOps, + DataType cType, + std::vector const& cStrides, + TensorOps const& cOps, + DataType dType, + std::vector const& dStrides, + TensorOps const& dOps, + double beta); + + /** + * Create a ContractionProblem from a definition of each index, the + * size of each index, the strides of each tensor, and any operations. + * + * @param freeIndices Free indices + * @param batchIndices Batch indices + * @param boundIndices Bound indices + * @param indexSizes Size of each index, in the order of appearance in + * the D tensor. + * + * @param aType Data type of A + * @param aStrides Strides of A + * @param aOps Operations to apply to A as it is read + * @param aOffset start offset of buffer A + * + * @param bType Data type of B + * @param bStrides Strides of B + * @param bOps Operations to apply to B as it is read + * @param bOffset start offset of buffer B + * + * @param cType Data type of C + * @param cStrides Strides of C + * @param cOps Operations to apply to C as it is read + * @param cOffset start offset of buffer C + * + * @param dType Data type of D + * @param dStrides Strides of D + * @param dOps Operations to apply to D as it is read + * @param dOffset start offset of buffer D + * + * @param beta Representative value of beta. Is only used to possibly + * select a more efficient kernel if we know that + * `beta == 0` or `beta == 1`. + */ + static ContractionProblem FromIndexSizes(FreeIndices const& freeIndices, + BatchIndices const& batchIndices, + BoundIndices const& boundIndices, + std::vector const& indexSizes, + DataType aType, + std::vector const& aStrides, + TensorOps const& aOps, + size_t aOffset, + DataType bType, + std::vector const& bStrides, + TensorOps const& bOps, + size_t bOffset, + DataType cType, + std::vector const& cStrides, + TensorOps const& cOps, + size_t cOffset, + DataType dType, + std::vector const& dStrides, + TensorOps const& dOps, + size_t dOffset, + double beta); + + /** + * Create a ContractionProblem based on an operation identifier such as + * `Contraction_l_AlikC_Bjlk_Cijk_Dijk` and individual index sizes. + * + * @param operationIdentifier String that represents this exact + * operation in terms of transposes, data + * types, and operations. + * @param indexSizes Size of each index, in the order of appearance in + * the D tensor. + * + * @param aType Data type of A + * @param aStrides Strides of A + * + * @param bType Data type of B + * @param bStrides Strides of B + * + * @param cType Data type of C + * @param cStrides Strides of C + * + * @param dType Data type of D + * @param dStrides Strides of D + * + * @param beta Representative value of beta. Is only used to possibly + * select a more efficient kernel if we know that + * `beta == 0` or `beta == 1`. + */ + static ContractionProblem FromIndexSizes(std::string const& operationIdentifier, + std::vector const& indexSizes, + DataType aType, + std::vector const& aStrides, + DataType bType, + std::vector const& bStrides, + DataType cType, + std::vector const& cStrides, + DataType dType, + std::vector const& dStrides, + double beta); + + /** + * Create a ContractionProblem based on an operation identifier such as + * `Contraction_l_AlikC_Bjlk_Cijk_Dijk` and individual index sizes. + * + * @param operationIdentifier String that represents this exact + * operation in terms of transposes, data + * types, and operations. + * @param indexSizes Size of each index, in the order of appearance in + * the D tensor. + * + * @param aType Data type of A + * @param aStrides Strides of A + * @param aOffset Data offset of buffer A + * + * @param bType Data type of B + * @param bStrides Strides of B + * @param bOffset Data offset of buffer B + * + * @param cType Data type of C + * @param cStrides Strides of C + * @param cOffset Data offset of buffer C + * + * @param dType Data type of D + * @param dStrides Strides of D + * @param dOffset Data offset of buffer D + * + * @param beta Representative value of beta. Is only used to possibly + * select a more efficient kernel if we know that + * `beta == 0` or `beta == 1`. + */ + static ContractionProblem FromIndexSizes(std::string const& operationIdentifier, + std::vector const& indexSizes, + DataType aType, + std::vector const& aStrides, + size_t aOffset, + DataType bType, + std::vector const& bStrides, + size_t bOffset, + DataType cType, + std::vector const& cStrides, + size_t cOffset, + DataType dType, + std::vector const& dStrides, + size_t dOffset, + double beta); + + ContractionProblem(TensorDescriptor const& a, + TensorOps const& aOps, + TensorDescriptor const& b, + TensorOps const& bOps, + TensorDescriptor const& c, + TensorOps const& cOps, + TensorDescriptor const& d, + TensorOps const& dOps, + FreeIndices const& freeIndices, + BatchIndices const& batchIndices, + BoundIndices const& boundIndices, + double beta, + size_t workspaceSize = 0); + + //! Returns size given original index assignment (in range + //! 0..NumIndicesC+boundSizes) + size_t size(size_t idx) const; + + size_t freeSizeA(size_t idx) const; + size_t freeSizeB(size_t idx) const; + + size_t batchSize(size_t idx) const; + size_t boundSize(size_t idx) const; + + size_t toAPos(size_t idx) const; + size_t toBPos(size_t idx) const; + + // Translate specified index into a position of that index in the d tensor. + // Since d tensor order is always index order this is 1:1 translation if the + // index is in-bounds: + size_t toDPos(size_t idx) const + { + if(idx < d().dimensions()) + return idx; + else + throw std::runtime_error("requested index not in D"); + } + + size_t toBoundsPos(size_t idx) const + { + if(idx < d().dimensions()) + throw std::runtime_error("invalid bounds index (is free or batch)"); + else if(idx > d().dimensions() + boundIndices().size()) + throw std::runtime_error("invalid bounds index (out-of-bounds)"); + else + return idx - d().dimensions(); + } + + std::vector const& problemSizes() const + { + return m_problemSizes; + } + + std::vector const& problemStrides() const + { + return m_problemStrides; + } + + std::vector const& convProblemSizes() const + { + return m_convProblemSizes; + } + void setConvProblemSizes(std::vector& convProblemSizes) + { + m_convProblemSizes.assign(convProblemSizes.begin(), convProblemSizes.end()); + } + + void setCEqualsD(bool cEqualsD) + { + m_cEqualsD = cEqualsD; + } + + bool cEqualsD() const + { + return m_cEqualsD; + } + + void setAlphaType(DataType type) + { + m_alphaType = type; + } + + DataType alphaType() const + { + return m_alphaType; + } + + void setAlphaRestriction(ScalarValue alpha) + { + m_alphaRestriction = alpha; + } + + ScalarValue alphaRestriction() const + { + return m_alphaRestriction; + } + + void setBetaType(DataType type) + { + m_betaType = type; + } + + DataType betaType() const + { + return m_betaType; + } + + void setUseBias(bool useBias) + { + m_useBias = useBias; + } + + bool useBias() const + { + return m_useBias; + } + + void setBetaRestriction(ScalarValue beta) + { + m_betaRestriction = beta; + } + + ScalarValue betaRestriction() const + { + return m_betaRestriction; + } + + void setStridedBatched(bool value) + { + m_stridedBatched = value; + } + + bool stridedBatched() const + { + return m_stridedBatched; + } + + void setHighPrecisionAccumulate(bool value) + { + m_highPrecisionAccumulate = value; + } + + bool highPrecisionAccumulate() const + { + return m_highPrecisionAccumulate; + } + + void setArithmeticUnit(ArithmeticUnit value) + { + m_arithmeticUnit = value; + } + ArithmeticUnit arithmeticUnit() const + { + return m_arithmeticUnit; + } + + void setKernelLanguage(KernelLanguage value) + { + m_kernelLanguage = value; + } + KernelLanguage kernelLanguage() const + { + return m_kernelLanguage; + } + + void setPerformanceMetric(PerformanceMetric value) + { + m_performanceMetric = value; + } + + PerformanceMetric performanceMetric() const + { + return m_performanceMetric; + } + + void setDeterministicMode(bool value) + { + m_deterministicMode = value; + } + bool deterministicMode() const + { + return m_deterministicMode; + } + + void setFp16AltImpl(bool value) + { + m_fp16AltImpl = value; + } + + bool fp16AltImpl() const + { + return m_fp16AltImpl; + } + + void setActivationType(ActivationType activationtype) + { + m_activationType = activationtype; + } + + ActivationType activationType() const + { + return m_activationType; + } + + void setActivationHPA(bool value) + { + m_activationHPA = value; + } + + bool activationHPA() const + { + return m_activationHPA; + } + + void setActivationEnumArg(ActivationType activationEnumArg) + { + m_activationEnumArg = activationEnumArg; + } + + ActivationType activationEnumArg() const + { + return m_activationEnumArg; + } + + /// Largest of the free and bound indices. Does not include batch size. + size_t maxProblemSize() const + { + return m_maxProblemSize; + } + + /// Allocated elements excluding batch dimensions + /// Used in assembly kernels to determine buffer limits, if batch dimes not + /// packed + size_t allocatedElementsNonBatchA() const + { + return m_allocatedElementsNonBatchA; + } + size_t allocatedElementsNonBatchB() const + { + return m_allocatedElementsNonBatchB; + } + + size_t flopsPerMac() const; + size_t flopCount() const; + + TensorDescriptor const& a() const + { + return m_a; + } + TensorDescriptor const& b() const + { + return m_b; + } + TensorDescriptor const& c() const + { + return m_c; + } + TensorDescriptor const& d() const + { + return m_d; + } + + TensorOps const& aOps() const + { + return m_aOps; + } + TensorOps const& bOps() const + { + return m_bOps; + } + TensorOps const& cOps() const + { + return m_cOps; + } + TensorOps const& dOps() const + { + return m_dOps; + } + + FreeIndices const& freeIndicesA() const + { + return m_freeIndicesA; + } + FreeIndices const& freeIndicesB() const + { + return m_freeIndicesB; + } + FreeIndices const& freeIndices() const + { + return m_freeIndices; + } + BatchIndices const& batchIndices() const + { + return m_batchIndices; + } + BoundIndices const& boundIndices() const + { + return m_boundIndices; + } + + ZeroPads const& aZeroPad() const + { + return m_aZeroPads; + } + ZeroPads const& bZeroPad() const + { + return m_bZeroPads; + } + + void addAZeroPad(const ZeroPad& zp); + void addBZeroPad(const ZeroPad& zp); + + bool transposeC01() const + { + return m_transposeC01; + }; + + double beta() const + { + return m_beta; + } + + std::string const& aNames() const + { + return m_aNames; + } + std::string const& bNames() const + { + return m_bNames; + } + std::string const& cNames() const + { + return m_cNames; + } + std::string const& dNames() const + { + return m_dNames; + } + std::string const& sumNames() const + { + return m_sumNames; + } + + bool transA() const + { + return m_aNames == "lik"; + } + bool transB() const + { + return m_bNames == "jlk"; + } + + std::string operationName() const; + std::string const& operationIdentifier() const + { + return m_operationIdentifier; + } + std::string operationDescription() const + { + return getOperationDescription(); + } + + void setWorkspaceSize(size_t size) + { + m_workspaceSize = size; + } + + size_t workspaceSize() const + { + return m_workspaceSize; + } + + void checkPersistentKernelEligibility(ContractionSolution const& solution, + Hardware const& hardware); + + bool getPersistentKernelEligibility() const + { + return m_eligibleForPK; + } + + private: + TensorDescriptor m_a; + TensorDescriptor m_b; + TensorDescriptor m_c; + TensorDescriptor m_d; + TensorOps m_aOps; + TensorOps m_bOps; + TensorOps m_cOps; + TensorOps m_dOps; + + std::string m_aNames; + std::string m_bNames; + std::string m_cNames; + std::string m_dNames; + std::string m_sumNames; + std::string m_operationIdentifier; + + bool m_transA; + bool m_transB; + bool m_cEqualsD = false; + bool m_stridedBatched = true; + bool m_highPrecisionAccumulate = false; + bool m_deterministicMode = false; + bool m_eligibleForPK = true; + bool m_fp16AltImpl = false; + bool m_useBias = false; + ActivationType m_activationType = ActivationType::None; + ActivationType m_activationEnumArg = ActivationType::None; + bool m_activationHPA = false; + ArithmeticUnit m_arithmeticUnit = ArithmeticUnit::Any; + KernelLanguage m_kernelLanguage = KernelLanguage::Any; + PerformanceMetric m_performanceMetric = PerformanceMetric::DeviceEfficiency; + + DataType m_alphaType = DataType::None; // if not assigned, will follow d-type + DataType m_betaType = DataType::None; // for bwd-compatible + + ScalarValue m_alphaRestriction = ScalarValue::Any; // restrictions on the alpha value used + ScalarValue m_betaRestriction = ScalarValue::Any; // restrictions on the beta value used + + FreeIndices m_freeIndicesA; //< in same order as IndexAssignmentsA + FreeIndices m_freeIndicesB; //< in same order as IndexAssignmentsB + FreeIndices m_freeIndices; + BatchIndices m_batchIndices; + BoundIndices m_boundIndices; + + ZeroPads m_aZeroPads; + ZeroPads m_bZeroPads; + + std::vector m_freeSizesA; + std::vector m_freeSizesB; + std::vector m_batchSizes; + std::vector m_boundSizes; + + std::vector m_problemSizes; + std::vector m_problemStrides; + std::vector m_convProblemSizes; + + bool m_transposeC01; + double m_beta; + + size_t m_maxProblemSize = 1; + + size_t m_allocatedElementsNonBatchA; + size_t m_allocatedElementsNonBatchB; + + size_t m_workspaceSize; + + void normalize(); + void consistencyCheck() const; + + void getIndexNames(std::string& aNames, + std::string& bNames, + std::string& cNames, + std::string& dNames, + std::string& sumNames) const; + + std::string getOperationIdentifier() const; + std::string getOperationDescription() const; + }; + + struct TENSILE_API ContractionInputs : public ProblemInputs + { + ContractionInputs(); + virtual ~ContractionInputs(); + + constexpr static uint32_t TypeId(DataType aType, + DataType bType, + DataType cType, + DataType dType, + DataType alphaType, + DataType betaType) + { + static_assert(BitFieldGenerator::ElementWidth((uint32_t)DataType::Count) * 6 + <= BitFieldGenerator::maxBitFieldWidth, + "Max bitfield width exceeded"); + + return BitFieldGenerator::GenerateBitField( + BitFieldGenerator::ElementWidth((uint32_t)DataType::Count), + (uint32_t)aType, + (uint32_t)bType, + (uint32_t)cType, + (uint32_t)dType, + (uint32_t)alphaType, + (uint32_t)betaType); + } + }; + + /** + * Contains actual pointer and argument values for a particular set of + * inputs. + */ + template + struct TENSILE_API TypedContractionInputs : public ContractionInputs + { + using AType = A; + using BType = B; + using CType = C; + using DType = D; + using AlphaType = Alpha; + using BetaType = Beta; + + TypedContractionInputs(); + + TypedContractionInputs(A const* _a, + B const* _b, + C const* _c, + D* _d, + Alpha _alpha, + Beta _beta, + A const* _bias = nullptr, + void* _ws = nullptr) + : TypedContractionInputs( + _a, _b, _c, _d, nullptr, nullptr, nullptr, nullptr, _alpha, _beta){}; + + TypedContractionInputs(A const* _a, + B const* _b, + C const* _c, + D* _d, + A const* const* _batchA, + B const* const* _batchB, + C const* const* _batchC, + D* const* _batchD, + Alpha _alpha, + Beta _beta, + A const* _bias = nullptr, + void* _ws = nullptr); + + ~TypedContractionInputs(); + + A const* a = nullptr; + B const* b = nullptr; + C const* c = nullptr; + D* d = nullptr; + + A const* const* batchA = nullptr; + B const* const* batchB = nullptr; + C const* const* batchC = nullptr; + D* const* batchD = nullptr; + + void* ws = nullptr; + + Alpha alpha = static_cast(0); + Beta beta = static_cast(0); + + A const* bias = nullptr; + + std::vector activationArgs; + + constexpr static uint32_t TypeId() + { + return ContractionInputs::TypeId(TypeInfo::Enum, + TypeInfo::Enum, + TypeInfo::Enum, + TypeInfo::Enum, + TypeInfo::Enum, + TypeInfo::Enum); + } + }; + + // Commonly used contraction input type groupings + // Naming: _[Ti_To_Tc]_: + // S=float, D=double, C=complex, Z=complex, + // H=Half, B=BF16, I8x4=Int8x4, I32=int32_t + using ContractionInputs_S_S_S = TypedContractionInputs; + using ContractionInputs_D_D_D = TypedContractionInputs; + using ContractionInputs_C_C_C = TypedContractionInputs>; + using ContractionInputs_Z_Z_Z = TypedContractionInputs>; +#ifdef TENSILE_USE_HALF + using ContractionInputs_H_H_H = TypedContractionInputs; + using ContractionInputs_H_H_S = TypedContractionInputs; + using ContractionInputs_H_S_S = TypedContractionInputs; +#endif // TENSILE_USE_HALF + using ContractionInputs_I8x4_I32_I32 = TypedContractionInputs; + using ContractionInputs_I8_I8_I32 + = TypedContractionInputs; + using ContractionInputs_I8_I32_I32 = TypedContractionInputs; + using ContractionInputs_I32_I32_I32 = TypedContractionInputs; +#ifdef TENSILE_USE_BF16 + using ContractionInputs_B_B_S + = TypedContractionInputs; + using ContractionInputs_B_S_S = TypedContractionInputs; +#endif // TENSILE_USE_BF16 + + TENSILE_API std::ostream& operator<<(std::ostream& stream, + ContractionProblem const& contraction); + + TENSILE_API std::ostream& operator<<(std::ostream& stream, + ContractionProblem::FreeIndex const& free); + TENSILE_API std::ostream& operator<<(std::ostream& stream, + ContractionProblem::BatchIndex const& batch); + TENSILE_API std::ostream& operator<<(std::ostream& stream, + ContractionProblem::BoundIndex const& bound); + + TENSILE_API std::istream& operator>>(std::istream& stream, ContractionProblem::FreeIndex& free); + TENSILE_API std::istream& operator>>(std::istream& stream, + ContractionProblem::BatchIndex& batch); + TENSILE_API std::istream& operator>>(std::istream& stream, + ContractionProblem::BoundIndex& bound); + + /** + * @} + */ +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblemPredicates.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblemPredicates.hpp new file mode 100644 index 0000000000..ba3fce12e0 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblemPredicates.hpp @@ -0,0 +1,1517 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace Tensile +{ + namespace Predicates + { + /** + * \addtogroup Predicates + * @{ + */ + /** + * @brief ContractionProblem predicates + */ + namespace Contraction + { + struct Free0SizeMultiple : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + Free0SizeMultiple() = default; + Free0SizeMultiple(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "Free0SizeMultiple"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return (!problem.transposeC01() ? problem.freeSizeA(index) + : problem.freeSizeB(index)) + % value + == 0; + } + }; + + struct Free1SizeMultiple : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + Free1SizeMultiple() = default; + Free1SizeMultiple(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "Free1SizeMultiple"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return (!problem.transposeC01() ? problem.freeSizeB(index) + : problem.freeSizeA(index)) + % value + == 0; + } + }; + + struct BatchSizeMultiple : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + BatchSizeMultiple() = default; + BatchSizeMultiple(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "BatchSizeMultiple"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.batchSize(index) % value == 0; + } + }; + + struct BatchSizeEqual : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + BatchSizeEqual() = default; + BatchSizeEqual(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "BatchSizeEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.batchSize(index) == value; + } + }; + + struct BoundSizeMultiple : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + int64_t index; + size_t value; + + BoundSizeMultiple() = default; + BoundSizeMultiple(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "BoundSizeMultiple"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + if(index < 0) + return problem.boundSize(problem.boundIndices().size() + index) % value + == 0; + else + return problem.boundSize(index) % value == 0; + } + }; + + struct ProblemSizeEqual : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + ProblemSizeEqual() = default; + ProblemSizeEqual(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "ProblemSizeEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.problemSizes()[index] == value; + } + }; + + struct MaxProblemSizeGreaterThan + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + size_t value; + + MaxProblemSizeGreaterThan() = default; + MaxProblemSizeGreaterThan(size_t value) + : value(value) + { + } + + static std::string Type() + { + return "MaxProblemSizeGreaterThan"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.maxProblemSize() > value; + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.maxProblemSize() << " > " << value + << ") == " << rv; + + return rv; + } + }; + + // If the tensor contains no free dimensions, then the first batch dimension + // serves as the leading free size + struct LeadingFree0SizesGreaterOrEqual + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + size_t value; + + LeadingFree0SizesGreaterOrEqual() = default; + LeadingFree0SizesGreaterOrEqual(size_t value) + : value(value) + { + } + + static std::string Type() + { + return "LeadingFree0SizesGreaterOrEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + assert(problem.batchIndices().size() <= 1); + // TODO: this is not quite right since it assumes batchSize(0) is lowest + // order in index assignments + // If tensor contains multiple batch dims this may not be true. + // Really should modify Contractions.py to select SizeN >= value, based on + // desired index requirement + return (problem.freeIndicesA().size() ? problem.freeSizeA(0) >= value + : problem.batchSize(0) >= value); + } + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" + << (problem.freeIndicesA().size() ? "freeA0:" : "batchA0:") + << (problem.freeIndicesA().size() ? problem.freeSizeA(0) + : problem.batchSize(0)) + << " >= " << value << ") == " << rv; + + return rv; + } + }; + + struct LeadingFree1SizesGreaterOrEqual + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + size_t value; + + LeadingFree1SizesGreaterOrEqual() = default; + LeadingFree1SizesGreaterOrEqual(size_t value) + : value(value) + { + } + + static std::string Type() + { + return "LeadingFree1SizesGreaterOrEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + assert(problem.batchIndices().size() <= 1); + // TODO: this is not quite right since it assumes batchSize(0) is lowest + // order in index assignments + // If tensor contains multiple batch dims this may not be true. + // Really should modify Contractions.py to select SizeN >= value, based on + // desired index requirement + return (problem.freeIndicesB().size() ? problem.freeSizeB(0) >= value + : problem.batchSize(0) >= value); + } + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" + << (problem.freeIndicesB().size() ? "freeB0:" : "batchB0:") + << (problem.freeIndicesB().size() ? problem.freeSizeB(0) + : problem.batchSize(0)) + << " >= " << value << ") == " << rv; + + return rv; + } + }; + + struct SizeEqual : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + SizeEqual() = default; + SizeEqual(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "SizeEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.size(index) == value; + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.size(index) << " == " << value + << ") == " << rv; + + return rv; + } + }; + + struct SizeGreaterThan : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + SizeGreaterThan() = default; + SizeGreaterThan(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "SizeGreaterThan"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return (problem.size(index) > value); + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.size(index) << " > " << value + << ") == " << rv; + + return rv; + } + }; + + struct SizeLessThan : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + SizeLessThan() = default; + SizeLessThan(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "SizeLessThan"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return (problem.size(index) < value); + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.size(index) << " < " << value + << ") == " << rv; + + return rv; + } + }; + + struct SizeMultiple : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + SizeMultiple() = default; + SizeMultiple(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "SizeMultiple"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return ((problem.size(index) % value) == 0); + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.size(index) << " % " << value + << " == 0) == " << rv; + + return rv; + } + }; + + struct StrideAEqual : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + StrideAEqual() = default; + StrideAEqual(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "StrideAEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.a().strides()[index] == value; + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.a().strides()[index] << " == " << value + << ") == " << rv; + + return rv; + } + }; + + struct StrideBEqual : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + StrideBEqual() = default; + StrideBEqual(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "StrideBEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.b().strides()[index] == value; + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.b().strides()[index] << " == " << value + << ") == " << rv; + + return rv; + } + }; + + struct StrideCEqual : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + StrideCEqual() = default; + StrideCEqual(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "StrideCEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.c().strides()[index] == value; + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.c().strides()[index] << " == " << value + << ") == " << rv; + + return rv; + } + }; + + struct StrideDEqual : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + StrideDEqual() = default; + StrideDEqual(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "StrideDEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.d().strides()[index] == value; + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.d().strides()[index] << " == " << value + << ") == " << rv; + + return rv; + } + }; + + struct LDCEqualsLDD : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = false + }; + static std::string Type() + { + return "LDCEqualsLDD"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.c().strides()[1] == problem.d().strides()[1]; + } + }; + + struct CEqualsD : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = false + }; + static std::string Type() + { + return "CEqualsD"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.cEqualsD(); + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << this->type() << "(problem.cEqualsD() == True) == " << rv; + + return rv; + } + }; + + struct BetaZero : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = false + }; + BetaZero() = default; + + static std::string Type() + { + return "BetaZero"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.beta() == 0.0; + } + }; + + struct BetaOne : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = false + }; + BetaOne() = default; + + static std::string Type() + { + return "BetaOne"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.beta() == 1.0; + } + }; + + struct HighPrecisionAccumulateEqual + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + bool value; + + HighPrecisionAccumulateEqual() = default; + HighPrecisionAccumulateEqual(bool value) + : value(value) + { + } + + static std::string Type() + { + return "HighPrecisionAccumulate"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.highPrecisionAccumulate() == value; + } + }; + + struct KernelLanguageCompatible + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + KernelLanguage value; + + KernelLanguageCompatible() = default; + KernelLanguageCompatible(KernelLanguage value) + : value(value) + { + } + + static std::string Type() + { + return "KernelLanguageCompatible"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.kernelLanguage() == value + || problem.kernelLanguage() == KernelLanguage::Any; + } + }; + + struct DeterministicModeEqual + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + bool value; + + DeterministicModeEqual() = default; + DeterministicModeEqual(bool value) + : value(value) + { + } + + static std::string Type() + { + return "DeterministicMode"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.deterministicMode() == value; + } + }; + + struct ArithmeticUnitCompatible + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + + ArithmeticUnit value; + + ArithmeticUnitCompatible() = default; + ArithmeticUnitCompatible(ArithmeticUnit value) + : value(value) + { + } + + static std::string Type() + { + return "ArithmeticUnitCompatible"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.arithmeticUnit() == value + || problem.arithmeticUnit() == ArithmeticUnit::Any; + } + }; + + struct AlphaValue : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + + ScalarValue value; + + AlphaValue() = default; + AlphaValue(ScalarValue value) + : value(value) + { + } + + static std::string Type() + { + return "AlphaValue"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.alphaRestriction() == value || value == ScalarValue::Any; + } + }; + + struct BetaValue : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + + ScalarValue value; + + BetaValue() = default; + BetaValue(ScalarValue value) + : value(value) + { + } + + static std::string Type() + { + return "BetaValue"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.betaRestriction() == value || value == ScalarValue::Any; + } + }; + + struct TypesEqual : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + TypesEqual() = default; + + std::array value; + + static std::string Type() + { + return "TypesEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.a().dataType() == value[0] && problem.b().dataType() == value[1] + && problem.c().dataType() == value[2] + && problem.d().dataType() == value[3]; + } + + virtual std::string toString() const override + { + return concatenate(this->type(), + "(a:", + value[0], + ", b:", + value[1], + ", c:", + value[2], + ", d:", + value[3], + ")"); + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << this->type() << "(a:" << problem.a().dataType() << " == " << value[0] + << "&& b:" << problem.b().dataType() << " == " << value[1] + << "&& c:" << problem.c().dataType() << " == " << value[2] + << "&& d:" << problem.d().dataType() << " == " << value[3] + << "): " << rv; + + return rv; + } + }; + + struct OperationIdentifierEqual + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + OperationIdentifierEqual() = default; + + std::string value; + + static std::string Type() + { + return "OperationIdentifierEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.operationIdentifier() == value; + } + }; + + struct BufferLoadOffsetLimitCheck + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + BufferLoadCheckPacket value; + + BufferLoadOffsetLimitCheck() = default; + BufferLoadOffsetLimitCheck(BufferLoadCheckPacket value) + : value(value) + { + } + + static std::string Type() + { + return "BufferLoadOffsetLimitCheck"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + const uint64_t TWO_POW_32 = 4294967296; + return (problem.a().strides()[1] * value.depthUorMT0 + value.shiftPtrElemA) + * problem.a().elementBytes() + < TWO_POW_32 + && (problem.b().strides()[1] * value.depthUorMT1 + value.shiftPtrElemB) + * problem.b().elementBytes() + < TWO_POW_32; + } + + virtual std::string toString() const override + { + return concatenate(this->type(), + "(DU/MT0:", + value.depthUorMT0, + ", DU/MT1:", + value.depthUorMT1, + ", ShiftPtrPadElementA:", + value.shiftPtrElemA, + ", ShiftPtrPadElementB:", + value.shiftPtrElemB, + ")"); + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" + << " (" << problem.a().strides()[1] << " * " << value.depthUorMT0 + << " + " << value.shiftPtrElemA << ") * " << problem.a().elementBytes() + << " < 4294967296 && " + << " (" << problem.b().strides()[1] << " * " << value.depthUorMT1 + << " + " << value.shiftPtrElemB << ") * " << problem.b().elementBytes() + << " < 4294967296" + << ") == " << rv; + + return rv; + } + }; + + struct BufferLoadOffsetLimitCheck_Beta + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + size_t value; + + BufferLoadOffsetLimitCheck_Beta() = default; + BufferLoadOffsetLimitCheck_Beta(size_t value) + : value(value) + { + } + + static std::string Type() + { + return "BufferLoadOffsetLimitCheck_Beta"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + if(problem.c().empty() || problem.beta() == 0) + { + return true; + } + else + { + const uint64_t TWO_POW_32 = 4294967296; + return problem.c().strides()[1] * problem.c().elementBytes() * value + < TWO_POW_32; + } + } + + virtual std::string toString() const override + { + return concatenate(this->type(), "(MT1:", value, ")"); + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.c().strides()[1] << " * " + << problem.c().elementBytes() << " * " << value << " < 4294967296" + << ") == " << rv; + + return rv; + } + }; + + struct BufferStoreOffsetLimitCheck + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + size_t value; + + BufferStoreOffsetLimitCheck() = default; + BufferStoreOffsetLimitCheck(size_t value) + : value(value) + { + } + + static std::string Type() + { + return "BufferStoreOffsetLimitCheck"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + const uint64_t TWO_POW_32 = 4294967296; + return problem.d().strides()[1] * problem.d().elementBytes() * value + < TWO_POW_32; + } + + virtual std::string toString() const override + { + return concatenate(this->type(), "(MT1:", value, ")"); + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.d().strides()[1] << " * " + << problem.d().elementBytes() << " * " << value << " < 4294967296" + << ") == " << rv; + + return rv; + } + }; + + struct WorkspaceCheck : public Predicate_CRTP + { + enum + { + HasIndex = true, + HasValue = true + }; + size_t index; + size_t value; + + WorkspaceCheck() = default; + WorkspaceCheck(size_t index, size_t value) + : index(index) + , value(value) + { + } + + static std::string Type() + { + return "WorkspaceCheck"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.d().totalLogicalElements() * value <= problem.workspaceSize(); + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.d().totalLogicalElements() << " * " << value + << " <= " << problem.workspaceSize() << ") == " << rv; + + return rv; + } + }; + + struct PersistentKernelCheck + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = false + }; + PersistentKernelCheck() = default; + + static std::string Type() + { + return "PersistentKernelCheck"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.getPersistentKernelEligibility(); + } + }; + + struct GlobalSplitUCheckMinK + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + + size_t value; + + GlobalSplitUCheckMinK() = default; + GlobalSplitUCheckMinK(size_t value) + : value(value) + { + } + + static std::string Type() + { + return "GlobalSplitUCheckMinK"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.boundSize(0) >= value; + } + + virtual std::string toString() const override + { + return concatenate(this->type(), "(value:", value, ")"); + } + + virtual bool debugEval(ContractionProblem const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.boundSize(0) << " >= " << value + << ") == " << rv; + + return rv; + } + }; + + struct CDStridesEqual : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = false + }; + + CDStridesEqual() = default; + + static std::string Type() + { + return "CDStridesEqual"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.c().strides() == problem.d().strides(); + } + }; + + struct StridedBatchedEqual + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + bool value; + + StridedBatchedEqual() = default; + StridedBatchedEqual(bool value) + : value(value) + { + } + + static std::string Type() + { + return "StridedBatched"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.stridedBatched() == value; + } + }; + + struct CUEfficiency : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = false + }; + + CUEfficiency() = default; + + static std::string Type() + { + return "CUEfficiency"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + if(problem.performanceMetric() == PerformanceMetric::CUEfficiency) + { + return true; + } + else if(problem.performanceMetric() == PerformanceMetric::Auto) + { + // True if total flops below a constant threshold + // Current threshold chosen naively as the flops for a + // 1500x1500 square matrix multiply + return problem.flopCount() < size_t(1500ll * 1500ll * 1500ll * 2); + } + else + { + return false; + } + } + }; + + struct Fp16AltImpl : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = false + }; + + Fp16AltImpl() = default; + + static std::string Type() + { + return "Fp16AltImpl"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.fp16AltImpl(); + } + }; + + struct EqualityMatching : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = false + }; + + EqualityMatching() = default; + + static std::string Type() + { + return "EqualityMatching"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return true; + } + }; + + // Activation + struct ActivationEqual : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + ActivationType value; + + ActivationEqual() = default; + ActivationEqual(ActivationType value) + : value(value) + { + } + + static std::string Type() + { + return "Activation"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.activationType() == value; + } + }; + + struct ActivationEnumWhiteList + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + ActivationEnumWhiteList() = default; + + std::string value; + + static std::string Type() + { + return "ActivationEnumWhiteList"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + if(problem.activationType() == ActivationType::All) + { + auto problemEnum = ToString(problem.activationEnumArg()); + if(value.find(problemEnum) != std::string::npos) + { + return true; + } + return false; + } + return true; + } + + virtual std::string toString() const override + { + return std::string("The supported activations are: " + value); + } + }; + + struct ActivationHPAEqual + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + bool value; + + ActivationHPAEqual() = default; + ActivationHPAEqual(bool value) + : value(value) + { + } + + static std::string Type() + { + return "ActivationHPA"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.activationHPA() == value; + } + }; + + struct UseBiasEqual + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + bool value; + + UseBiasEqual() = default; + UseBiasEqual(bool value) + : value(value) + { + } + + static std::string Type() + { + return "UseBias"; + } + + virtual bool operator()(ContractionProblem const& problem) const override + { + return problem.useBias() == value; + } + }; + } // namespace Contraction + + /** + * @} + */ + } // namespace Predicates +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblemProperties.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblemProperties.hpp new file mode 100644 index 0000000000..01b0ba8301 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblemProperties.hpp @@ -0,0 +1,222 @@ +/** + * Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include + +#include + +namespace Tensile +{ + /** + * \addtogroup PropertyClasses + * @{ + */ + namespace Contraction + { + struct FreeSizeA : public Property_CRTP + { + enum + { + HasIndex = true, + HasValue = false + }; + size_t index; + + static std::string Type() + { + return "FreeSizeA"; + } + + virtual size_t operator()(ContractionProblem const& problem) const + { + return problem.freeSizeA(index); + } + }; + + struct FreeSizeB : public Property_CRTP + { + enum + { + HasIndex = true, + HasValue = false + }; + size_t index; + + static std::string Type() + { + return "FreeSizeB"; + } + + virtual size_t operator()(ContractionProblem const& problem) const + { + return problem.freeSizeB(index); + } + }; + + struct BatchSize : public Property_CRTP + { + enum + { + HasIndex = true, + HasValue = false + }; + size_t index; + + static std::string Type() + { + return "BatchSize"; + } + + virtual size_t operator()(ContractionProblem const& problem) const + { + return problem.batchSize(index); + } + }; + + struct BoundSize : public Property_CRTP + { + enum + { + HasIndex = true, + HasValue = false + }; + size_t index; + + static std::string Type() + { + return "BoundSize"; + } + + virtual size_t operator()(ContractionProblem const& problem) const + { + return problem.boundSize(index); + } + }; + + struct AStride : public Property_CRTP + { + enum + { + HasIndex = true, + HasValue = false + }; + size_t index; + + static std::string Type() + { + return "AStride"; + } + + virtual size_t operator()(ContractionProblem const& problem) const + { + return problem.a().strides()[index]; + } + }; + + struct BStride : public Property_CRTP + { + enum + { + HasIndex = true, + HasValue = false + }; + size_t index; + + static std::string Type() + { + return "BStride"; + } + + virtual size_t operator()(ContractionProblem const& problem) const + { + return problem.b().strides()[index]; + } + }; + + struct CStride : public Property_CRTP + { + enum + { + HasIndex = true, + HasValue = false + }; + size_t index; + + static std::string Type() + { + return "CStride"; + } + + virtual size_t operator()(ContractionProblem const& problem) const + { + return problem.c().strides()[index]; + } + }; + + struct DStride : public Property_CRTP + { + enum + { + HasIndex = true, + HasValue = false + }; + size_t index; + + static std::string Type() + { + return "DStride"; + } + + virtual size_t operator()(ContractionProblem const& problem) const + { + return problem.d().strides()[index]; + } + }; + + struct OperationIdentifier + : public Property_CRTP + { + enum + { + HasIndex = false, + HasValue = false + }; + + static std::string Type() + { + return "OperationIdentifier"; + } + + virtual std::string operator()(ContractionProblem const& problem) const + { + return problem.operationIdentifier(); + } + }; + } // namespace Contraction + + /** + * @} + */ +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem_Detail.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem_Detail.hpp new file mode 100644 index 0000000000..6523552181 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem_Detail.hpp @@ -0,0 +1,153 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include +#include + +namespace Tensile +{ + template <> + struct Comparison + { + enum + { + implemented = true + }; + + static int compare(ContractionProblem::FreeIndex const& lhs, + ContractionProblem::FreeIndex const& rhs) + { + return LexicographicCompare(lhs.d, rhs.d, lhs.c, rhs.c, lhs.i, rhs.i, lhs.isA, rhs.isA); + } + }; + + template <> + struct Comparison + { + enum + { + implemented = true + }; + + static int compare(ContractionProblem::BatchIndex const& lhs, + ContractionProblem::BatchIndex const& rhs) + { + return LexicographicCompare(lhs.d, rhs.d, lhs.c, rhs.c, lhs.a, rhs.a, lhs.b, rhs.b); + } + }; + + template <> + struct Comparison + { + enum + { + implemented = true + }; + + static int compare(ContractionProblem::BoundIndex const& lhs, + ContractionProblem::BoundIndex const& rhs) + { + return LexicographicCompare(lhs.a, rhs.a, lhs.b, rhs.b); + } + }; + + template <> + struct Comparison + { + enum + { + implemented = true + }; + + static int compare(ContractionProblem const& lhs, ContractionProblem const& rhs) + { + return LexicographicCompare(lhs.operationIdentifier(), + rhs.operationIdentifier(), + lhs.highPrecisionAccumulate(), + rhs.highPrecisionAccumulate(), + lhs.kernelLanguage(), + rhs.kernelLanguage(), + lhs.deterministicMode(), + rhs.deterministicMode(), + lhs.arithmeticUnit(), + rhs.arithmeticUnit(), + lhs.a(), + rhs.a(), + lhs.b(), + rhs.b(), + lhs.c(), + rhs.c(), + lhs.d(), + rhs.d(), + lhs.workspaceSize(), + rhs.workspaceSize(), + lhs.stridedBatched(), + rhs.stridedBatched(), + lhs.performanceMetric(), + rhs.performanceMetric(), + lhs.fp16AltImpl(), + rhs.fp16AltImpl(), + lhs.activationType(), + rhs.activationType(), + lhs.activationHPA(), + rhs.activationHPA(), + lhs.useBias(), + rhs.useBias()); + } + }; +} // namespace Tensile + +namespace std +{ + template <> + struct hash + { + inline size_t operator()(Tensile::ContractionProblem const& problem) const + { + return Tensile::hash_combine(problem.operationIdentifier(), + problem.a(), + problem.b(), + problem.c(), + problem.d(), + problem.highPrecisionAccumulate(), + problem.kernelLanguage(), + problem.deterministicMode(), + problem.arithmeticUnit(), + problem.workspaceSize(), + problem.stridedBatched(), + problem.performanceMetric(), + problem.fp16AltImpl(), + problem.activationType(), + problem.activationHPA(), + problem.useBias()); + } + }; + +} // namespace std diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem_fwd.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem_fwd.hpp new file mode 100644 index 0000000000..6ef9d47d52 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem_fwd.hpp @@ -0,0 +1,40 @@ +/** + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +namespace Tensile +{ + + class ContractionProblem; + struct ContractionInputs; + + template + struct TypedContractionInputs; +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp new file mode 100644 index 0000000000..f35c62ef6f --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp @@ -0,0 +1,336 @@ +/** + * Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace Tensile +{ + struct PerfModel + { + double clock = std::numeric_limits::quiet_NaN(); + double memClock = std::numeric_limits::quiet_NaN(); + double peakGFlops = std::numeric_limits::quiet_NaN(); + double memBandwidthMBps = std::numeric_limits::quiet_NaN(); + double l2ReadBwMul = std::numeric_limits::quiet_NaN(); + double gFlops = std::numeric_limits::quiet_NaN(); + double readEff = 0.0; + double l2ReadHitRate = 0.0; + double l2WriteHitRate = 0.0; + int CUs = 0; + }; + + extern PerfModel perf; + + struct BufferLoadCheckPacket + { + size_t shiftPtrElemA; + size_t shiftPtrElemB; + size_t depthUorMT0; + size_t depthUorMT1; + }; + + /** + * Represents a single kernel or set of kernels that can perform a single + * tensor contraction. + * + * Can generate `KernelInvocation` objects to solve a particular problem + * given a set of `ContractionInputs`. + */ + class ContractionSolution : public Solution + { + public: + using Problem = ContractionProblem; + using Inputs = ContractionInputs; + + static std::string Type() + { + return "Contraction"; + } + virtual std::string type() const + { + return Type(); + } + + virtual std::string KernelName() const + { + return kernelName; + } + virtual std::string name() const + { + return kernelName; + } + virtual std::string description() const + { + return kernelName; + } + + bool isSourceKernel() const; + + //! Estimates based on problem size, solution tile, and machine hardware + //! charz: + struct StaticPerformanceModel + { + size_t memReadBytesA = 0.0; //! Estimated memory reads A + size_t memReadBytesB = 0.0; //! Estimated memory reads B + size_t memReadBytesC = 0.0; //! Estimated memory reads C + size_t memWriteBytesD = 0.0; //! Estimated memory writes D + size_t memReadBytes = 0.0; + size_t memGlobalReads = 0; + size_t memGlobalWrites = 0; + }; + + struct Granularities + { + double numTiles0 = 0.0; //! number of tiles in 0 dimension + double numTiles1 = 0.0; //! number of tiles in 1 dimension + double totalTiles = 0.0; + double tilesPerCu = 0.0; + + //! Granularity is measured 0..1 with 1.0 meaning no granularity loss + double tile0Granularity = 0.0; // loss due to tile0 + double tile1Granularity = 0.0; + double cuGranularity = 0.0; + double waveGranularity = 0.0; + double totalGranularity = 0.0; + double totalTileAwareGranularity = 0.0; + double natCuGranularity = 0.0; + double natTilesPerCu = 0.0; + double suTilesPerCu = 0.0; + double suCuGranularity = 0.0; + double waves = 0.0; + double suWavesPerSimdx2 = 0.0; + double suWaveGranularity = 0.0; + + int CUs = 0; + + double MT0; + double MT1; + double GSU; + double LSU; + }; + + struct ProjectedPerformance + { + Granularities granularities; + + double speedGFlops = 0.0; //! final gflops projection + int CUs = 0; + + StaticPerformanceModel staticModel; + }; + + struct TAMetricProblemScore + { + Granularities granularites; + + int CUs = 0; + + double summationPerformance = 0.0; + + double M; + double N; + double K; + }; + + /** + * Calculate required workspace size. + */ + size_t requiredWorkspaceSize(Problem const& problem) const; + + Granularities computeGranularities( + Hardware const& hardware, double M, double N, double K, double NumBatches) const; + + StaticPerformanceModel staticPerformanceModel(double M, + double N, + double K, + double NumBatches, + double MT0, + double MT1, + double NumCUs, + double totalGranularity, + int globalSplitU) const; + + TAMetricProblemScore computeProblemScore( + Hardware const& hardware, double M, double N, double K, double NumBatches) const; + + double computeTileAwareMetric(TAMetricProblemScore pp, + TAMetricProblemScore ppReference) const; + + double computeTAMScore(Problem const& problem, + Hardware const& hardware, + double model_M, + double model_N, + double model_K, + double model_NumBatches) const; + + /** + * Calculate the projected performance based on granularity loss. + */ + ProjectedPerformance projectedPerformance(Problem const& problem, + Hardware const& hardware) const; + + /** + * Generate a set of kernel calls to solve a particular problem. + */ + virtual std::vector + solve(Problem const& problem, Inputs const& inputs, Hardware const& hardware) const; + + template + std::vector solveTyped(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const; + + template + KernelInvocation generateSingleCall(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const; + + template + KernelInvocation generateBetaOnlyCall(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const; + + template + std::string betaOnlyKernelName(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const; + + template + KernelInvocation generateOutputConversionCall(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const; + + template + std::string outputConversionKernelName(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const; + + template + KernelInvocation generateActivationOnlyCall(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const; + + template + std::string activationOnlyKernelName(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const; + + struct SizeMapping + { + dim3 workGroupSize; + dim3 threadTile; + dim3 macroTile; + + size_t staggerU = 0; + size_t depthU = 0; + size_t globalSplitU = 0; + size_t staggerStrideShift = 0; + int workGroupMapping = 0; + + size_t packBatchDims = 0; + int packSummationDims = 0; + int magicDivAlg = 1; + int persistentKernel = 0; + bool persistentKernelAlongBatch = false; + + bool sourceKernel = false; + + int globalAccumulation = 0; + size_t workspaceSizePerElemC = 0; + + bool activationFused = true; + }; + + struct ProblemType + { + std::string operationIdentifier; + DataType aType = DataType::Float; + DataType bType = DataType::Float; + DataType cType = DataType::Float; + DataType dType = DataType::Float; + bool highPrecisionAccumulate = false; + bool useBeta = true; + bool useBias = false; + bool useInitialStridesAB = false; + bool useInitialStridesCD = false; + bool stridedBatched = true; + bool fp16AltImpl = false; + ActivationType activationType = ActivationType::None; + bool activationHPA = false; + }; + + struct LinearModel + { + double slope = 1.0; + double intercept = 0.0; + double max = 1000.0; + }; + + int index = 0; + std::string kernelName; + bool debugKernel = false; + bool kernelArgsLog = false; + + std::shared_ptr> problemPredicate + = std::make_shared>(); + std::shared_ptr> hardwarePredicate + = std::make_shared>(); + + SizeMapping sizeMapping; + + ProblemType problemType; + + /// Debugging purposes. Shouldn't contain any vital information that isn't + /// somewhere else. + std::map info; + std::map ideals; + LinearModel linearModel; + + int32_t staggerUIter(Problem const& problem, + Inputs const& inputs, + Hardware const& hardware) const; + + uint32_t magicNumberAlg1(uint32_t x, uint32_t* magicShift) const; + uint32_t magicNumberAlg2(uint32_t x, uint32_t* magicShift) const; + uint32_t magicNumber(int magicDivAlg, uint32_t x, uint32_t* magicShift) const; + uint32_t smallMagicNumber(uint32_t x) const; + }; + + std::ostream& operator<<(std::ostream& stream, + ContractionSolution::StaticPerformanceModel const& spm); + std::ostream& operator<<(std::ostream& stream, + ContractionSolution::ProjectedPerformance const& spm); + std::ostream& operator<<(std::ostream& stream, BufferLoadCheckPacket const& st); +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution_fwd.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution_fwd.hpp new file mode 100644 index 0000000000..f3bfdb0de7 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution_fwd.hpp @@ -0,0 +1,28 @@ +/** + * Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +namespace Tensile +{ + class ContractionSolution; +} diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Contractions.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Contractions.hpp new file mode 100644 index 0000000000..18b8faef85 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Contractions.hpp @@ -0,0 +1,33 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes.hpp new file mode 100644 index 0000000000..7f258a1aae --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes.hpp @@ -0,0 +1,209 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \defgroup DataTypes Data Type Info + * + * @brief Definitions and metadata on supported data types. + */ + + /** + * \ingroup DataTypes + * @{ + */ + + /** + * Data Type + */ + enum class DataType : int + { + Float, + Double, + ComplexFloat, + ComplexDouble, + Half, + Int8x4, + Int32, + BFloat16, + Int8, + Count, + None = Count + }; + + std::string ToString(DataType d); + std::string TypeAbbrev(DataType d); + std::ostream& operator<<(std::ostream& stream, DataType const& t); + std::istream& operator>>(std::istream& stream, DataType& t); + + /** + * \ingroup DataTypes + * \brief Runtime accessible data type metadata + */ + struct DataTypeInfo + { + static DataTypeInfo const& Get(int index); + static DataTypeInfo const& Get(DataType t); + static DataTypeInfo const& Get(std::string const& str); + + DataType dataType; + std::string name; + std::string abbrev; + + size_t elementSize; + size_t packing; + size_t segmentSize; + + bool isComplex; + bool isIntegral; + + private: + static void registerAllTypeInfo(); + static void registerAllTypeInfoOnce(); + + template + static void registerTypeInfo(); + + static void addInfoObject(DataTypeInfo const& info); + + static std::map data; + static std::map typeNames; + }; + + /** + * \ingroup DataTypes + * \brief Compile-time accessible data type metadata. + */ + template + struct TypeInfo + { + }; + + template + struct BaseTypeInfo + { + constexpr static DataType Enum = T_Enum; + + /// Bytes of one element. May contain multiple segments. + constexpr static size_t ElementSize = sizeof(T); + /// Segments per element. + constexpr static size_t Packing = T_Packing; + /// Bytes per segment. + constexpr static size_t SegmentSize = ElementSize / Packing; + + constexpr static bool IsComplex = T_IsComplex; + constexpr static bool IsIntegral = T_IsIntegral; + + static inline std::string Name() + { + return ToString(Enum); + } + static inline std::string Abbrev() + { + return TypeAbbrev(Enum); + } + }; + + template + constexpr DataType BaseTypeInfo::Enum; + template + constexpr size_t BaseTypeInfo::ElementSize; + template + constexpr size_t BaseTypeInfo::Packing; + template + constexpr size_t BaseTypeInfo::SegmentSize; + + template + constexpr bool BaseTypeInfo::IsComplex; + template + constexpr bool BaseTypeInfo::IsIntegral; + + template <> + struct TypeInfo : public BaseTypeInfo + { + }; + template <> + struct TypeInfo : public BaseTypeInfo + { + }; + template <> + struct TypeInfo> + : public BaseTypeInfo, DataType::ComplexFloat, 1, true, false> + { + }; + template <> + struct TypeInfo> + : public BaseTypeInfo, DataType::ComplexDouble, 1, true, false> + { + }; + + template <> + struct TypeInfo : public BaseTypeInfo + { + }; + + template <> + struct TypeInfo : public BaseTypeInfo + { + }; + + template <> + struct TypeInfo : public BaseTypeInfo + { + }; + template <> + struct TypeInfo : public BaseTypeInfo + { + }; + + // Enum DataType::Int8 maps to int8_t, struct Tensile::Int8 is only used for LogTensor now + template <> + struct TypeInfo : public BaseTypeInfo + { + }; + + /** + * @} + */ +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_BFloat16.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_BFloat16.hpp new file mode 100644 index 0000000000..feb2f8e8ae --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_BFloat16.hpp @@ -0,0 +1,253 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include + +#define TENSILE_USE_BF16 + +#ifndef __BYTE_ORDER__ +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ +#endif + +#define BFLOAT16_Q_NAN_VALUE 0xFFC1 + +namespace Tensile +{ + /** + * \ingroup DataTypes + * @{ + */ + + struct BFloat16 + { + BFloat16() + : data(BFLOAT16_ZERO_VALUE) + { + } + + BFloat16(BFloat16 const& other) = default; + + template ::value) + && std::is_convertible::value>::type> + explicit BFloat16(T const& value) + : data(float_to_bfloat16(static_cast(value)).data) + { + } + + explicit operator float() const + { + return bfloat16_to_float(*this); + } + + explicit operator double() const + { + return static_cast(float(*this)); + } + + explicit operator int() const + { + return static_cast(float(*this)); + } + + explicit operator uint32_t() const + { + return static_cast(float(*this)); + } + + uint16_t data; + + private: + static const int16_t BFLOAT16_ZERO_VALUE = 0x00; + + // zero extend lower 16 bits of bfloat16 to convert to IEEE float + static float bfloat16_to_float(const BFloat16 v) + { + union + { + float fp32 = 0; + uint16_t q[2]; + }; + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + q[0] = v.data; +#else + q[1] = v.data; +#endif + return fp32; + } + + // truncate lower 16 bits of IEEE float to convert to bfloat16 + static BFloat16 float_to_bfloat16(const float v) + { + BFloat16 bf16; + if(std::isnan(v)) + { + bf16.data = BFLOAT16_Q_NAN_VALUE; + return bf16; + } + union + { + float fp32; + uint16_t p[2]; + }; + fp32 = v; + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + bf16.data = p[0]; +#else + bf16.data = p[1]; +#endif + return bf16; + } + }; + + inline std::ostream& operator<<(std::ostream& os, const BFloat16& bf16) + { + os << static_cast(bf16); + return os; + } + + inline BFloat16 operator+(BFloat16 a, BFloat16 b) + { + return static_cast(static_cast(a) + static_cast(b)); + } + inline BFloat16 operator+(int a, BFloat16 b) + { + return static_cast(static_cast(a) + static_cast(b)); + } + inline BFloat16 operator+(BFloat16 a, int b) + { + return static_cast(static_cast(a) + static_cast(b)); + } + inline BFloat16 operator-(BFloat16 a, BFloat16 b) + { + return static_cast(static_cast(a) - static_cast(b)); + } + inline BFloat16 operator*(BFloat16 a, BFloat16 b) + { + return static_cast(static_cast(a) * static_cast(b)); + } + inline BFloat16 operator/(BFloat16 a, BFloat16 b) + { + return static_cast(static_cast(a) / static_cast(b)); + } + + inline bool operator<(BFloat16 a, BFloat16 b) + { + return static_cast(a) < static_cast(b); + } + inline bool operator<=(BFloat16 a, BFloat16 b) + { + return static_cast(a) <= static_cast(b); + } + inline bool operator==(BFloat16 a, BFloat16 b) + { + return static_cast(a) == static_cast(b); + } + inline bool operator!=(BFloat16 a, BFloat16 b) + { + return static_cast(a) != static_cast(b); + } + inline bool operator>(BFloat16 a, BFloat16 b) + { + return static_cast(a) > static_cast(b); + } + inline bool operator>=(BFloat16 a, BFloat16 b) + { + return static_cast(a) >= static_cast(b); + } + + inline BFloat16& operator+=(BFloat16& a, BFloat16 b) + { + a = a + b; + return a; + } + inline BFloat16& operator-=(BFloat16& a, BFloat16 b) + { + a = a - b; + return a; + } + inline BFloat16& operator*=(BFloat16& a, BFloat16 b) + { + a = a * b; + return a; + } + inline BFloat16& operator/=(BFloat16& a, BFloat16 b) + { + a = a / b; + return a; + } + + inline BFloat16 operator++(BFloat16& a) + { + a += BFloat16(1); + return a; + } + inline BFloat16 operator++(BFloat16& a, int) + { + BFloat16 original_value = a; + ++a; + return original_value; + } + + /** + * @} + */ +} // namespace Tensile + +namespace std +{ + inline bool isinf(const Tensile::BFloat16& a) + { + return std::isinf(static_cast(a)); + } + inline bool isnan(const Tensile::BFloat16& a) + { + return std::isnan(static_cast(a)); + } + inline bool iszero(const Tensile::BFloat16& a) + { + return (a.data & 0x7FFF) == 0; + } + + inline Tensile::BFloat16 abs(const Tensile::BFloat16& a) + { + return static_cast(std::abs(static_cast(a))); + } + inline Tensile::BFloat16 sin(const Tensile::BFloat16& a) + { + return static_cast(std::sin(static_cast(a))); + } + inline Tensile::BFloat16 cos(const Tensile::BFloat16& a) + { + return static_cast(std::cos(static_cast(a))); + } +} // namespace std diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_Half.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_Half.hpp new file mode 100644 index 0000000000..738b0a18c7 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_Half.hpp @@ -0,0 +1,59 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#ifdef TENSILE_USE_HIP +#include +#endif + +#include + +namespace Tensile +{ +#if defined(TENSILE_USE_HIP) || defined(TENSILE_USE_FLOAT16_BUILTIN) + /** + * \ingroup DataTypes + */ + using Half = _Float16; +#define TENSILE_USE_HALF +#else + /** + * \ingroup DataTypes + */ + struct Half : public DistinctType + { + }; +#endif +} // namespace Tensile + +namespace std +{ + inline ostream& operator<<(ostream& stream, const Tensile::Half val) + { + return stream << static_cast(val); + } +} // namespace std diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_Int8.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_Int8.hpp new file mode 100644 index 0000000000..0619c39c38 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_Int8.hpp @@ -0,0 +1,51 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#ifdef TENSILE_USE_HIP +#include +#endif + +#include + +namespace Tensile +{ + /** + * \ingroup DataTypes + */ + struct Int8 : public DistinctType + { + }; +} // namespace Tensile + +namespace std +{ + inline ostream& operator<<(ostream& stream, const Tensile::Int8 val) + { + return stream << static_cast(val); + } +} // namespace std diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_Int8x4.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_Int8x4.hpp new file mode 100644 index 0000000000..41e989f42e --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/DataTypes_Int8x4.hpp @@ -0,0 +1,77 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +namespace Tensile +{ + /** + * \ingroup DataTypes + */ + struct Int8x4 + { + Int8x4() + : a(0) + , b(0) + , c(0) + , d(0) + { + } + + Int8x4(int8_t xa, int8_t xb, int8_t xc, int8_t xd) + : a(xa) + , b(xb) + , c(xc) + , d(xd) + { + } + + Int8x4(uint32_t v) + : a(v & 0xff) + , b((v << 8) & 0xff) + , c((v << 16) & 0xff) + , d((v << 24) & 0xff) + { + } + + int8_t a, b, c, d; + + int32_t operator*(Int8x4 const& other) const + { + return static_cast(a) * static_cast(other.a) + + static_cast(b) * static_cast(other.b) + + static_cast(c) * static_cast(other.c) + + static_cast(d) * static_cast(other.d); + } + + inline bool operator==(Int8x4 const& rhs) const + { + return a == rhs.a && b == rhs.b && c == rhs.c && d == rhs.d; + } + }; + + static_assert(sizeof(Int8x4) == 4, "Int8x4 must be 4 bytes."); +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Debug.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Debug.hpp new file mode 100644 index 0000000000..2ffe98a3c8 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Debug.hpp @@ -0,0 +1,88 @@ +/** + * Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * cop- ies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include + +#include + +namespace Tensile +{ + /** + * @brief Common place for defining flags which enable debug behaviour. + */ + class Debug : public LazySingleton + { + public: + bool printPropertyEvaluation() const; + bool printPredicateEvaluation() const; + bool printDeviceSelection() const; + bool printCodeObjectInfo() const; + + bool printKernelArguments() const; + + bool printDataInit() const; + + // print tensor dims, strides, memory sizes + bool printTensorInfo() const; + + // 3 levels of debugging for the convolution reference debug + bool printConvolutionReference1() const; + bool printConvolutionReference2() const; + bool printConvolutionReference3() const; + + // if tensors are printed, use hexadecimal output format + bool printTensorModeHex() const; + + bool printLibraryVersion() const; + + bool printLookupEfficiency() const; + + bool printWinningKernelName() const; + + bool printSolutionSelectionTime() const; + + bool naivePropertySearch() const; + + bool skipKernelLaunch() const; + + bool enableDebugSelection() const; + + std::string getMetric() const; + + int getSolutionIndex() const; + + private: + friend LazySingleton; + + int m_value; + int m_value2; + bool m_naivePropertySearch = false; + bool m_debugSelection = false; + int m_solution_index = -1; + std::string m_metric = ""; + + Debug(); + }; +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Distance.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Distance.hpp new file mode 100644 index 0000000000..0d717ada3e --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Distance.hpp @@ -0,0 +1,391 @@ +/** + * Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include + +namespace Tensile +{ + namespace Matching + { + /** + * \ingroup PropertyMatching + * \defgroup DistanceFunctions Distance Functions + */ + + /** + * \addtogroup DistanceFunctions + * @{ + */ + + /** + * @brief Abstract Distance function base class + */ + + /** + * Distance functions must implement the following methods: + * + * double operator()(Key const& p1, Key const& p2) + * + * Returns the distance between p1 and p2. + * + * bool improvementPossible(Key const& p1, Key const& p2, size_t idx, + * double bestDistance) const + * + * May return `false` if it's impossible for `p1` and `p2` to be closer than + * `bestDistance` based on `p1[idx]` and `p2[idx]` alone. This can be used to + * exit early when searching a table sorted by element `idx`. + * + * If that doesn't apply to the given distance metric, returning `true` will + * give the correct result. + */ + template + class Distance + { + public: + virtual std::string type() const = 0; + virtual ~Distance() = default; + }; + + // Not really a distance, but defined as one for template specialization purposes + template + struct Equality : public Distance + { + enum + { + HasIndex = false, + HasValue = false + }; + static std::string Type() + { + return "Equality"; + } + virtual std::string type() const override + { + return Type(); + } + + inline bool operator()(Key const& p1, Key const& p2) const + { + double distance = 0.0; + + for(int i = 0; i < p1.size(); i++) + { + double di = p1[i] - p2[i]; + distance += di * di; + } + return distance; + } + }; + + template + struct RatioDistance : public Distance + { + enum + { + HasIndex = false, + HasValue = false + }; + static std::string Type() + { + return "Ratio"; + } + virtual std::string type() const override + { + return Type(); + } + + inline double operator()(Key const& p1, Key const& p2) const + { + double distance = 1.0; + for(int i = 0; i < p1.size(); i++) + { + distance += std::abs(std::log(double(p1[i]) / double(p2[i]))); + } + + return distance; + } + + inline bool improvementPossible(Key const& p1, + Key const& p2, + size_t idx, + double bestDistance) const + { + return true; + } + }; + + template + struct ManhattanDistance : public Distance + { + enum + { + HasIndex = false, + HasValue = false + }; + static std::string Type() + { + return "Manhattan"; + } + virtual std::string type() const override + { + return Type(); + } + + inline double operator()(Key const& p1, Key const& p2) const + { + double distance = 0; + for(int i = 0; i < p1.size(); i++) + { + distance += std::abs(p1[i] - p2[i]); + } + return distance; + } + + inline bool improvementPossible(Key const& p1, + Key const& p2, + size_t idx, + double bestDistance) const + { + double d0 = std::abs(p1[idx] - p2[idx]); + return (d0 < bestDistance) || (p1 == p2); + } + }; + + template + struct EuclideanDistance : public Distance + { + enum + { + HasIndex = false, + HasValue = false + }; + + static std::string Type() + { + return "Euclidean"; + } + virtual std::string type() const override + { + return Type(); + } + + inline double operator()(Key const& p1, Key const& p2) const + { + double distance = 0.0; + + for(int i = 0; i < p1.size(); i++) + { + double di = p1[i] - p2[i]; + distance += di * di; + } + return distance; + } + + inline bool improvementPossible(Key const& p1, + Key const& p2, + size_t idx, + double bestDistance) const + { + double d0 = p1[idx] - p2[idx]; + return ((d0 * d0) < bestDistance) || (p1 == p2); + } + }; + + template + struct JSDivergence : public Distance + { + enum + { + HasIndex = false, + HasValue = false + }; + + static std::string Type() + { + return "JSD"; + } + virtual std::string type() const override + { + return Type(); + } + + double kl_divergence(double* p, double* q) const + { + double acc = 0.0; + int dims = 3; + + for(int i = 0; i < dims; i++) + acc += p[i] * std::log(p[i] / q[i]); + + return acc; + } + + void normalize(Key const& p, Key const& q, double* norm_p, double* norm_q) const + { + int const dims = 3; + double sum_p = 0.0; + double sum_q = 0.0; + + for(int i = 0; i < dims; i++) + { + sum_p += p[i]; + sum_q += q[i]; + } + + for(int i = 0; i < dims; i++) + { + norm_p[i] = p[i] / sum_p; + norm_q[i] = q[i] / sum_q; + } + } + + inline double operator()(Key const& p1, Key const& p2) const + { + double distance = 0.0; + + double norm_p[3]; + double norm_q[3]; + double m[3]; + + this->normalize(p1, p2, norm_p, norm_q); + + for(int i = 0; i < p1.size(); i++) + m[i] = 0.5 * (norm_p[i] + norm_q[i]); + + distance + = 0.5 * this->kl_divergence(norm_p, m) + 0.5 * this->kl_divergence(norm_q, m); + + return distance; + } + + inline bool improvementPossible(Key const& p1, + Key const& p2, + size_t idx, + double bestDistance) const + { + double norm_p[3]; + double norm_q[3]; + double m; + + this->normalize(p1, p2, norm_p, norm_q); + + m = 0.5 * (norm_p[idx] + norm_q[idx]); + + double d0 = 0.5 * (norm_p[idx] * std::log(norm_p[idx] / m)) + + 0.5 * (norm_q[idx] * std::log(norm_q[idx] / m)); + + return (d0 < bestDistance) || (p1 == p2); + } + }; + + template + struct RandomDistance : public Distance + { + enum + { + HasIndex = false, + HasValue = false + }; + + static std::string Type() + { + return "Random"; + } + virtual std::string type() const override + { + return Type(); + } + + inline double operator()(Key const& p1, Key const& p2) const + { + return double(rand()); + } + + inline bool improvementPossible(Key const& p1, + Key const& p2, + size_t idx, + double bestDistance) const + { + return true; + } + }; + + template + struct GridBasedDistance : public Distance + { + enum + { + HasIndex = false, + HasValue = false + }; + + static std::string Type() + { + return "GridBased"; + } + virtual std::string type() const override + { + return Type(); + } + + inline double operator()(Key const& p1, Key const& p2) const + { + double distance = 0.0; + + double M = p2[0]; + double N = p2[1]; + double K = p2[2]; + + // This is hard coding workaround solution // + // If incoming_size falls inside grid boundary (32768 in this case), searching toward larger M,N. + // If incoming_N > 32768, searching toward larger M and nearest boundary. + // IF incoming_M > 32768, searching toward larger N and nearest boundary. + double stepM = (p1[0] <= 32768)? std::ceil(p1[0] / M): 1; + double stepN = (p1[1] <= 32768)? std::ceil(p1[1] / N): 1; + if (p1[0] <= 32768 && p1[1] <= 32768) { + distance = std::round(100 * stepM * stepN / std::pow((p1[0] * p1[1]) / (stepM * M * stepN * N),2) ); + } else { + distance = std::round(10000 * stepM * stepN * std::pow(std::pow(p1[0] - p2[0],2) + std::pow(p1[1] - p2[1],2),0.5)); + } + // and nearest K + distance += (std::abs(K - p1[2]) / (K + p1[2] * 8)); + + return distance; + } + + inline bool improvementPossible(Key const& p1, + Key const& p2, + size_t idx, + double bestDistance) const + { + return true; + } + }; + + /** + * @} + */ + + } // namespace Matching +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/DistinctType.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/DistinctType.hpp new file mode 100644 index 0000000000..51c143f66d --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/DistinctType.hpp @@ -0,0 +1,107 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include + +#include + +namespace Tensile +{ + /** + * \ingroup DataTypes + * @{ + */ + + /** + * Allows the creation of a new type which is not automatically convertible + * to its underlying data type. + */ + template + struct DistinctType + { + using Value = T; + + DistinctType() = default; + DistinctType(DistinctType const& other) = default; + + DistinctType(T const& v) + : value(v) + { + } + + DistinctType& operator=(DistinctType const& other) = default; + DistinctType& operator =(T const& other) + { + value = other; + return *this; + } + + operator const T&() const + { + return value; + } + + T value; + }; + + template + struct Comparison> + { + enum + { + implemented = true + }; + + static int compare(DistinctType const& lhs, + DistinctType const& rhs) + { + return LexicographicCompare(lhs.value, rhs.value); + } + }; + + template + struct Comparison, T> + { + enum + { + implemented = true + }; + + static int compare(DistinctType const& lhs, T const& rhs) + { + return LexicographicCompare(lhs.value, rhs); + } + }; + + /** + * @} + */ +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/EmbeddedData.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/EmbeddedData.hpp new file mode 100644 index 0000000000..1c37feb4ab --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/EmbeddedData.hpp @@ -0,0 +1,141 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \defgroup Embedding Data Embedding + * + * @brief Mechanism which allows binary data to be stored in an executable + * or shared library as an array of bytes. + */ + + template + struct EmbedData; + + /** + * \ingroup Embedding + * + * @brief EmbeddedData is the mechanism for retrieving data which has been + * registered with EmbedData. + */ + template + class TENSILE_API EmbeddedData : public LazySingleton> + { + public: + using Base = LazySingleton>; + + using Item = std::vector; + using Items = std::vector; + using Map = std::unordered_map; + + static Items const& Get() + { + return Get(""); + } + + static Items const& Get(std::string const& key) + { + auto const& items = Base::Instance().items; + auto iter = items.find(key); + if(iter == items.end()) + return Base::Instance().empty; + + return iter->second; + } + + protected: + friend Base; + friend struct EmbedData; + + static Items& GetMutable() + { + return GetMutable(""); + } + + static Items& GetMutable(std::string const& key) + { + if(Debug::Instance().printDataInit()) + std::cout << "Embedding an object of type " << typeid(Object).name() << " with key " + << key << std::endl; + return Base::Instance().items[key]; + } + + EmbeddedData() = default; + + Map items; + const Items empty = Items{}; + }; + + /** + * \ingroup Embedding + * + * @brief Object which registers embedded data when it's instantiated. + */ + template + struct TENSILE_API EmbedData + { + EmbedData(std::initializer_list data) + { + EmbeddedData::GetMutable().emplace_back(data); + } + + EmbedData(std::vector const& data) + { + EmbeddedData::GetMutable().push_back(data); + } + + EmbedData(std::string const& key, std::initializer_list data) + { + EmbeddedData::GetMutable(key).emplace_back(data); + } + + EmbedData(std::string const& key, std::vector const& data) + { + EmbeddedData::GetMutable(key).push_back(data); + } + }; + +} // namespace Tensile + +#define TENSILE_CONCATENATE_SYMBOLS(a, b) TENSILE_CONCATENATE_SYMBOLS1(a, b) +#define TENSILE_CONCATENATE_SYMBOLS1(a, b) TENSILE_CONCATENATE_SYMBOLS2(a, b) +#define TENSILE_CONCATENATE_SYMBOLS2(a, b) a##b + +#define TENSILE_EMBED_SYMBOL_NAME TENSILE_CONCATENATE_SYMBOLS(TensileEmbeddedData, __LINE__) diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/EmbeddedLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/EmbeddedLibrary.hpp new file mode 100644 index 0000000000..060f0514d0 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/EmbeddedLibrary.hpp @@ -0,0 +1,102 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#ifdef TENSILE_DEFAULT_SERIALIZATION + +#include +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \ingroup Embedding + * + * @brief Interface for retrieving a SolutionLibrary object which as been + * stored in the executable via EmbedData/EmbeddedData. + */ + template + class TENSILE_API EmbeddedLibrary : public LazySingleton> + { + public: + using Singleton = LazySingleton>; + + /** + * Constructs and returns a new SolutionLibrary instance from the static data. + */ + static std::shared_ptr> NewLibrary(); + + /** + * Constructs and returns a new SolutionLibrary instance from the static data + * for the specified key. + */ + static std::shared_ptr> + NewLibrary(std::string const& key); + + /** + * Constructs (if necessary) and returns the shared SolutionLibrary for this + * problem type. + */ + static std::shared_ptr> Get() + { + return Get(""); + } + + /** + * Constructs (if necessary) and returns the shared SolutionLibrary for this + * problem type and key. + */ + static std::shared_ptr> Get(std::string const& key) + { + return Singleton::Instance().Library(key); + } + + std::shared_ptr> Library(std::string const& key) + { + std::lock_guard lock(m_mutex); + + auto& ptr = m_libraries[key]; + + if(!ptr) + ptr = NewLibrary(key); + + return ptr; + } + + private: + friend Singleton; + EmbeddedLibrary() = default; + + std::mutex m_mutex; + std::unordered_map>> + m_libraries; + }; + +} // namespace Tensile + +#endif diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ExactLogicLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ExactLogicLibrary.hpp new file mode 100644 index 0000000000..e4502868ac --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ExactLogicLibrary.hpp @@ -0,0 +1,220 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include + +namespace Tensile +{ + /** + * \addtogroup SolutionLibrary + * @{ + */ + template + using LibraryEntry = std::shared_ptr>; + template + using LibraryRow = std::pair>; + + /** + * Represents a set of sub-libraries, each with associated predicates. It + * should be placed in order of best to worst solutions. We assume the best + * solution is the first one where we match the predicates. + * + * Examples: Picking solutions written for a particular GPU, solutions that + * assume that a particular size is a multiple of something, etc. + */ + template + struct ExactLogicLibrary : public SolutionLibrary + { + using Row = LibraryRow; + std::vector rows; + + ExactLogicLibrary() = default; + ExactLogicLibrary(std::initializer_list init) + : rows(init) + { + } + + ExactLogicLibrary(std::vector const& init) + : rows(init) + { + } + + virtual std::shared_ptr findBestSolution(MyProblem const& problem, + Hardware const& hardware, + double* fitness + = nullptr) const override + { + std::shared_ptr rv; + + for(auto const& row : rows) + { + if(row.first(problem, hardware)) + { + rv = row.second->findBestSolution(problem, hardware, fitness); + if(rv) + return rv; + } + } + + return rv; + } + + virtual SolutionSet findAllSolutions(MyProblem const& problem, + Hardware const& hardware) const override + { + SolutionSet rv; + + for(auto const& row : rows) + { + if(row.first(problem, hardware)) + { + auto rowSolutions = row.second->findAllSolutions(problem, hardware); + rv.insert(rowSolutions.begin(), rowSolutions.end()); + } + } + + return rv; + } + + virtual std::string description() const override + { + return concatenate(this->type(), " library (", rows.size(), " rows)"); + } + }; + + struct HardwarePredicate + { + std::shared_ptr> value; + + HardwarePredicate() = default; + HardwarePredicate(std::shared_ptr> init) + : value(init) + { + } + + template + bool operator()(Any const& problem, Hardware const& hardware) const + { + bool debug = Debug::Instance().printDeviceSelection(); + + if(debug) + { + value->debugEval(hardware, std::cout); + std::cout << std::endl; + } + + return (*value)(hardware); + } + }; + + template + struct HardwareSelectionLibrary + : public ExactLogicLibrary + { + using Base = ExactLogicLibrary; + + HardwareSelectionLibrary() = default; + HardwareSelectionLibrary(std::initializer_list init) + : Base(init) + { + } + + HardwareSelectionLibrary(std::vector const& init) + : Base(init) + { + } + + static std::string Type() + { + return "Hardware"; + } + virtual std::string type() const override + { + return Type(); + } + }; + + template + struct ProblemPredicate + { + std::shared_ptr> value; + + ProblemPredicate() = default; + ProblemPredicate(std::shared_ptr> init) + : value(init) + { + } + + bool operator()(MyProblem const& problem, Hardware const& hardware) const + { + bool debug = Debug::Instance().printPredicateEvaluation(); + + if(debug) + { + value->debugEval(problem, std::cout); + std::cout << std::endl; + } + + return (*value)(problem); + } + }; + + template + struct ProblemSelectionLibrary + : public ExactLogicLibrary> + { + using Base = ExactLogicLibrary>; + + ProblemSelectionLibrary() = default; + ProblemSelectionLibrary(std::initializer_list init) + : Base(init) + { + } + + ProblemSelectionLibrary(std::vector const& init) + : Base(init) + { + } + + static std::string Type() + { + return "Problem"; + } + virtual std::string type() const + { + return Type(); + } + }; + + /** + * @} + */ + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/GranularitySelectionLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/GranularitySelectionLibrary.hpp new file mode 100644 index 0000000000..d46db894a3 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/GranularitySelectionLibrary.hpp @@ -0,0 +1,203 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include +#include +#include + +#include + +namespace Tensile +{ + /** + * \ingroup SolutionLibrary + */ + struct ExactSelectionTableEntry + { + std::vector key; + int value; + }; + + /** + * \ingroup SolutionLibrary + * + * Compares the tile sizes of each kernel, the dimensions of the problem, + * and the number of compute units on the target GPU to select a kernel + * that fits the best on the GPU with the lowest amount of waste + * ("granularity loss"). + */ + template + struct GranularitySelectionLibrary : public SolutionLibrary + { + std::map> solutions; + std::map, int> exactMap; + + static std::string Type() + { + return "GranularitySelection"; + } + virtual std::string type() const override + { + return Type(); + } + virtual std::string description() const override + { + std::string rv = this->type(); + + return rv; + } + + virtual std::shared_ptr findBestSolution(MyProblem const& problem, + Hardware const& hardware, + double* fitness + = nullptr) const override + { + const bool debug = Debug::Instance().printPropertyEvaluation(); + + std::vector key; + size_t M = problem.freeSizeA(0); + key.push_back(M); + size_t N = problem.freeSizeB(0); + key.push_back(N); + size_t NumBatches = problem.batchSize(0); + key.push_back(NumBatches); + size_t K = problem.boundSize(0); + key.push_back(K); + + auto exactMatch = exactMap.find(key); + if(exactMatch != this->exactMap.end()) + { + int index = exactMatch->second; + + auto rv = solutions.at(index); + + if(debug) + { + std::cout << "Exact match: " << rv->description(); + rv->problemPredicate->debugEval(problem, std::cout); + std::cout << std::endl; + rv->hardwarePredicate->debugEval(hardware, std::cout); + std::cout << std::endl; + } + + if((*rv->problemPredicate)(problem) && (*rv->hardwarePredicate)(hardware)) + { + return rv; + } + else if(debug) + { + std::cout << "Predicate failure" << std::endl; + } + } + + double bestPerformance = 0.0; + std::shared_ptr bestSolution; + + for(auto const& row : solutions) + { + auto myPerformance + = row.second->projectedPerformance(problem, hardware).speedGFlops; + + if(debug) + { + std::cout << row.second->description() << ": " << myPerformance; + } + + if(myPerformance > bestPerformance) + { + if((*row.second->problemPredicate)(problem) + && (*row.second->hardwarePredicate)(hardware)) + { + bestPerformance = myPerformance; + bestSolution = row.second; + + if(debug) + std::cout << " <-- Best so far"; + } + else if(debug) + { + std::cout << " <-- Best, but predicate failure"; + } + + if(debug) + { + row.second->problemPredicate->debugEval(problem, std::cout); + std::cout << std::endl; + row.second->hardwarePredicate->debugEval(hardware, std::cout); + std::cout << std::endl; + } + } + } + + return bestSolution; + } + + virtual SolutionSet findAllSolutions(MyProblem const& problem, + Hardware const& hardware) const override + { + bool debug = Debug::Instance().printPropertyEvaluation(); + + SolutionSet rv; + + for(auto const& row : solutions) + { + if(debug) + { + std::cout << row.second->description() << ": "; + } + + if((*row.second->problemPredicate)(problem) + && (*row.second->hardwarePredicate)(hardware)) + { + rv.insert(row.second); + + if(debug) + std::cout << " Works"; + } + else if(debug) + { + if(debug) + std::cout << " Predicate failed"; + } + + if(debug) + { + row.second->problemPredicate->debugEval(problem, std::cout); + std::cout << std::endl; + row.second->hardwarePredicate->debugEval(hardware, std::cout); + std::cout << std::endl; + } + } + + return rv; + } + }; +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/KernelArguments.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/KernelArguments.hpp new file mode 100644 index 0000000000..3d4403d18e --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/KernelArguments.hpp @@ -0,0 +1,250 @@ +/** + * Copyright 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +namespace Tensile +{ + + class TENSILE_API KernelArguments + { + public: + KernelArguments(bool log = true); + virtual ~KernelArguments(); + + void reserve(size_t bytes, size_t count); + + template + void append(std::string const& name, T value); + + template + void appendUnbound(std::string const& name); + + template + void bind(std::string const& name, T value); + + bool isFullyBound() const; + + void const* data() const; + size_t size() const; + + friend std::ostream& operator<<(std::ostream& stream, const KernelArguments& t); + friend class const_iterator; + + using ArgPair = std::pair; + class const_iterator : public std::iterator + { + public: + const_iterator(KernelArguments const& args); + const_iterator(KernelArguments const& args, std::string const& name); + const_iterator(const const_iterator& other) = default; + const_iterator& operator++(); + const_iterator operator++(int); + bool operator==(const const_iterator& rhs) const; + bool operator!=(const const_iterator& rhs) const; + ArgPair const& operator*() const; + ArgPair const* operator->() const; + void reset(); + template + operator T() const; + + private: + void assignCurrentArg(); + + std::vector::const_iterator m_currentArg; + KernelArguments const& m_args; + ArgPair m_value; + }; + + const_iterator begin() const; + const_iterator end() const; + + private: + enum + { + ArgOffset, + ArgSize, + ArgBound, + ArgString, + NumArgFields + }; + using Arg = std::tuple; + static_assert(std::tuple_size::value == NumArgFields, + "Enum for fields of Arg tuple doesn't match size of tuple."); + + void alignTo(size_t alignment); + + template + void append(std::string const& name, T value, bool bound); + + template + std::string stringForValue(T value, bool bound); + + void appendRecord(std::string const& name, Arg info); + + template + void writeValue(size_t offset, T value); + + std::vector m_data; + + std::vector m_names; + std::unordered_map m_argRecords; + + bool m_log; + }; + + TENSILE_API KernelArguments::const_iterator begin(KernelArguments const&); + TENSILE_API KernelArguments::const_iterator end(KernelArguments const&); + + template + inline void KernelArguments::append(std::string const& name, T value) + { + append(name, value, true); + } + + template + inline void KernelArguments::appendUnbound(std::string const& name) + { + append(name, static_cast(0), false); + } + + template + inline void KernelArguments::bind(std::string const& name, T value) + { + if(!m_log) + { + throw std::runtime_error("Binding is not supported without logging."); + } + + auto it = m_argRecords.find(name); + if(it == m_argRecords.end()) + { + throw std::runtime_error("Attempt to bind unknown argument " + name); + } + + auto& record = it->second; + + if(std::get(record)) + { + throw std::runtime_error("Attempt to bind already bound argument " + name); + } + + if(sizeof(T) != std::get(record)) + { + throw std::runtime_error("Size mismatch in binding argument " + name); + } + + size_t offset = std::get(record); + + if(offset % alignof(T) != 0) + { + throw std::runtime_error("Alignment error in argument " + name + ": type mismatch?"); + } + + writeValue(offset, value); + + std::get(record) = stringForValue(value, true); + std::get(record) = true; + } + + template + inline std::string KernelArguments::stringForValue(T value, bool bound) + { + if(!m_log) + return ""; + + if(!bound) + return ""; + + std::ostringstream msg; + msg << value; + return msg.str(); + } + + template + inline void KernelArguments::append(std::string const& name, T value, bool bound) + { + alignTo(alignof(T)); + + size_t offset = m_data.size(); + size_t size = sizeof(T); + + if(m_log) + { + std::string valueString = stringForValue(value, bound); + appendRecord(name, Arg(offset, size, bound, valueString)); + } + + m_data.insert(m_data.end(), sizeof(value), 0); + writeValue(offset, value); + } + + template + inline void KernelArguments::writeValue(size_t offset, T value) + { + if(offset + sizeof(T) > m_data.size()) + { + throw std::runtime_error("Value exceeds allocated bounds."); + } + + std::memcpy(&m_data[offset], &value, sizeof(T)); + } + + inline void KernelArguments::alignTo(size_t alignment) + { + size_t extraElements = m_data.size() % alignment; + size_t padding = (alignment - extraElements) % alignment; + + m_data.insert(m_data.end(), padding, 0); + } + + inline void KernelArguments::appendRecord(std::string const& name, KernelArguments::Arg record) + { + auto it = m_argRecords.find(name); + if(it != m_argRecords.end()) + { + throw std::runtime_error("Duplicate argument name: " + name); + } + + m_argRecords[name] = record; + m_names.push_back(name); + } + + template + KernelArguments::const_iterator::operator T() const + { + if(sizeof(T) != m_value.second) + { + throw std::bad_cast(); + } + return *reinterpret_cast(const_cast(m_value.first)); + } +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/KernelLanguageTypes.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/KernelLanguageTypes.hpp new file mode 100644 index 0000000000..95682ee4f4 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/KernelLanguageTypes.hpp @@ -0,0 +1,148 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \defgroup KernelLanguages Kernel language type Info + * + * @brief Definitions and metadata on supported kernel language types. + */ + + /** + * \ingroup KernelLanguages + * @{ + */ + + /** + * Kernel Language + */ + enum class KernelLanguage : int + { + Any, + Assembly, + Source, + Count + }; + + std::string ToString(KernelLanguage d); + std::string TypeAbbrev(KernelLanguage d); + std::ostream& operator<<(std::ostream& stream, KernelLanguage const& t); + std::istream& operator>>(std::istream& stream, KernelLanguage& t); + + /** + * \ingroup KernelLanguages + * \brief Runtime accessible kernel language type metadata + */ + struct KernelLanguageTypeInfo + { + static KernelLanguageTypeInfo const& Get(int index); + static KernelLanguageTypeInfo const& Get(KernelLanguage t); + static KernelLanguageTypeInfo const& Get(std::string const& str); + + KernelLanguage m_kernelLanguage; + std::string name; + std::string abbrev; + + private: + static void registerAllTypeInfo(); + static void registerAllTypeInfoOnce(); + + template + static void registerTypeInfo(); + + static void addInfoObject(KernelLanguageTypeInfo const& info); + + static std::map data; + static std::map typeNames; + }; + + /** + * \ingroup KernelLanguages + * \brief Compile-time accessible kernel language type metadata. + */ + template + struct KernelLanguageInfo; + + template + struct BaseKernelLanguageInfo + { + constexpr static KernelLanguage Enum = T_Enum; + + static inline std::string Name() + { + return ToString(Enum); + } + static inline std::string Abbrev() + { + return TypeAbbrev(Enum); + } + }; + + template + constexpr KernelLanguage BaseKernelLanguageInfo::Enum; + + template <> + struct KernelLanguageInfo + : public BaseKernelLanguageInfo + { + }; + template <> + struct KernelLanguageInfo + : public BaseKernelLanguageInfo + { + }; + template <> + struct KernelLanguageInfo + : public BaseKernelLanguageInfo + { + }; + + /** + * @} + */ +} // namespace Tensile + +namespace std +{ + template <> + struct hash + { + inline size_t operator()(Tensile::KernelLanguage const& val) const + { + return hash()(static_cast(val)); + } + }; +} // namespace std diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Macros.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Macros.hpp new file mode 100644 index 0000000000..3268342c41 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Macros.hpp @@ -0,0 +1,41 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#ifdef TENSILE_STATIC_ONLY +#define TENSILE_API +#else +/** + * Marks a symbol as being visible from outside of a shared library which + * Tensile is a part of. + */ +#ifdef _WIN32 +#define TENSILE_API __declspec(dllexport) +#else +#define TENSILE_API __attribute__((visibility("default"))) +#endif +#endif diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/MapLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/MapLibrary.hpp new file mode 100644 index 0000000000..9e838919b6 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/MapLibrary.hpp @@ -0,0 +1,134 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include + +namespace Tensile +{ + /** + * \ingroup SolutionLibrary + */ + template + using LibraryMap = std::unordered_map>; + + /** + * \ingroup SolutionLibrary + * + * Represents a set of solutions where each problem can map to only one + * sub-library, as opposed to an `ExactLogicLibrary` where fall-through can + * occur. + * + * Example: Problem type: Number of dimensions, transposes, etc. + */ + template + struct ProblemMapLibrary : public SolutionLibrary + { + static std::string Type() + { + return "ProblemMap"; + } + virtual std::string type() const override + { + return Type(); + } + virtual std::string description() const override + { + if(property == nullptr) + { + return concatenate(type(), " (property: nullptr, ", map.size(), " rows)"); + } + else + { + return concatenate( + type(), " (property: ", property->toString(), ", ", map.size(), " rows)"); + } + } + + LibraryEntry lookup(MyProblem const& problem, + Hardware const& hardware) const + { + auto key = (*property)(problem); + auto iter = map.find(key); + + bool debug = Debug::Instance().printPropertyEvaluation(); + if(debug) + { + std::cout << type() << " Searching for " << key; + + if(iter == map.end()) + { + std::cout << " (not found). Available keys:" << std::endl; + for(auto const& pair : map) + { + std::cout << " " << pair.first << std::endl; + } + } + else + { + std::cout << " found " << iter->second->description(); + } + + std::cout << std::endl; + } + + if(iter == map.end()) + return nullptr; + + return iter->second; + } + + virtual std::shared_ptr findBestSolution(MyProblem const& problem, + Hardware const& hardware, + double* fitness + = nullptr) const override + { + auto library = lookup(problem, hardware); + + if(library == nullptr) + return std::shared_ptr(); + + return library->findBestSolution(problem, hardware, fitness); + } + + virtual SolutionSet findAllSolutions(MyProblem const& problem, + Hardware const& hardware) const override + { + auto library = lookup(problem, hardware); + + if(library == nullptr) + return SolutionSet(); + + return library->findAllSolutions(problem, hardware); + } + + std::shared_ptr> property; + LibraryMap map; + }; +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/MasterSolutionLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/MasterSolutionLibrary.hpp new file mode 100644 index 0000000000..c6bca125ab --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/MasterSolutionLibrary.hpp @@ -0,0 +1,140 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace Tensile +{ + + /** + * \ingroup SolutionLibrary + */ + template + using SolutionMap = std::map>; + + /** + * \ingroup SolutionLibrary + * + * Root level library object. Contains all individual solutions in a map + * for serialization purposes. + */ + template + struct MasterSolutionLibrary : public SolutionLibrary + { + static std::string Type() + { + return "Master"; + } + std::string type() const override + { + return Type(); + } + std::string description() const override + { + if(library == nullptr) + return concatenate( + type(), " (", solutions.size(), " solutions, next level: nullptr)"); + else + return concatenate(type(), + " (", + solutions.size(), + " solutions, next level: ", + library->type(), + ")"); + } + + std::shared_ptr> library; + SolutionMap solutions; + std::string version; + + MasterSolutionLibrary() = default; + + virtual std::shared_ptr findBestSolution(MyProblem const& problem, + Hardware const& hardware, + double* fitness + = nullptr) const override + { + if(Debug::Instance().printSolutionSelectionTime()) + { + auto start = std::chrono::steady_clock::now(); + auto result = findBestSolution_runner(problem, hardware, fitness); + auto end = std::chrono::steady_clock::now(); + + double time = std::chrono::duration(end - start).count(); + std::cout << "Solution selection time: " << time << " us" << std::endl; + + return result; + } + else + { + return findBestSolution_runner(problem, hardware, fitness); + } + } + + std::shared_ptr findBestSolution_runner(MyProblem const& problem, + Hardware const& hardware, + double* fitness = nullptr) const + { + const int solution_index = Debug::Instance().getSolutionIndex(); + + if(solution_index >= 0) + { + std::cout << "Tensile will use solution index: " << solution_index << std::endl; + std::cout + << "Warning: Tensile will only work for a particular transpose and data type." + << std::endl; + std::cout << "Set TENSILE_SOLUTION_INDEX to a negative number to restore the " + "default behavior." + << std::endl; + + auto selected_solution = solutions.at(solution_index); + + if((*selected_solution->problemPredicate)(problem) + && (*selected_solution->hardwarePredicate)(hardware)) + return selected_solution; + else + return nullptr; + } + else + return library->findBestSolution(problem, hardware, fitness); + } + + virtual SolutionSet findAllSolutions(MyProblem const& problem, + Hardware const& hardware) const override + { + return library->findAllSolutions(problem, hardware); + } + }; + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/MatchingLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/MatchingLibrary.hpp new file mode 100644 index 0000000000..bd08ab8848 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/MatchingLibrary.hpp @@ -0,0 +1,120 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include + +namespace Tensile +{ + /** + * \ingroup SolutionLibrary + * + * Uses a distance function to select kernels based on benchmarks. + * Benchmarks are performed to determine the optimal kernel at a number of + * specific sizes. At runtime, we find the benchmarked size that is closest + * to the size asked for. + */ + template + struct ProblemMatchingLibrary : public SolutionLibrary + { + using Element = std::shared_ptr>; + using Table = Matching::MatchingTable>; + std::shared_ptr table; + + static std::string Type() + { + return "Matching"; + } + virtual std::string type() const override + { + return Type(); + } + virtual std::string description() const override + { + if(table == nullptr) + return concatenate(type(), ", table: nullptr"); + else + return concatenate(type(), ": ", table->description()); + } + + virtual std::shared_ptr findBestSolution(MyProblem const& problem, + Hardware const& hardware, + double* fitness + = nullptr) const override + { + bool useDebugSelection = Debug::Instance().enableDebugSelection(); + + typename Table::Transform transform + = [&](Element library) -> std::shared_ptr { + return library->findBestSolution(problem, hardware); + }; + + if(useDebugSelection) + { + std::shared_ptr evaluationSolution + = table->findBestEvaluationSolution(problem, hardware, transform); + return evaluationSolution; + } + else + { + double localFitness = std::numeric_limits::max(); + fitness = (fitness) ? fitness : &localFitness; + std::shared_ptr solution; + std::tie(solution, *fitness) = table->findBestMatch(problem, transform); + return solution; + } + } + + virtual SolutionSet findAllSolutions(MyProblem const& problem, + Hardware const& hardware) const override + { + bool debug = Debug::Instance().printPropertyEvaluation(); + + SolutionSet rv; + + auto matches = table->matchesInOrder(problem); + + for(auto const& row : matches) + { + if(debug) + std::cout << row->description() << std::endl; + + auto rowSolutions = row->findAllSolutions(problem, hardware); + rv.insert(rowSolutions.begin(), rowSolutions.end()); + + if(debug) + std::cout << std::endl; + } + + return rv; + } + }; + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/PerformanceMetricTypes.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/PerformanceMetricTypes.hpp new file mode 100644 index 0000000000..95b8a38fe2 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/PerformanceMetricTypes.hpp @@ -0,0 +1,148 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \defgroup PerformanceMetrics Performance metric type Info + * + * @brief Definitions and metadata on supported performance metric types. + */ + + /** + * \ingroup PerformanceMetrics + * @{ + */ + + /** + * Performance Metric + */ + enum class PerformanceMetric : int + { + Auto, + CUEfficiency, + DeviceEfficiency, + Count + }; + + std::string ToString(PerformanceMetric d); + std::string TypeAbbrev(PerformanceMetric d); + std::ostream& operator<<(std::ostream& stream, PerformanceMetric const& t); + std::istream& operator>>(std::istream& stream, PerformanceMetric& t); + + /** + * \ingroup PerformanceMetrics + * \brief Runtime accessible performance metric type metadata + */ + struct PerformanceMetricTypeInfo + { + static PerformanceMetricTypeInfo const& Get(int index); + static PerformanceMetricTypeInfo const& Get(PerformanceMetric t); + static PerformanceMetricTypeInfo const& Get(std::string const& str); + + PerformanceMetric m_performanceMetric; + std::string name; + std::string abbrev; + + private: + static void registerAllTypeInfo(); + static void registerAllTypeInfoOnce(); + + template + static void registerTypeInfo(); + + static void addInfoObject(PerformanceMetricTypeInfo const& info); + + static std::map data; + static std::map typeNames; + }; + + /** + * \ingroup PerformanceMetrics + * \brief Compile-time accessible performance metric type metadata. + */ + template + struct PerformanceMetricInfo; + + template + struct BasePerformanceMetricInfo + { + constexpr static PerformanceMetric Enum = T_Enum; + + static inline std::string Name() + { + return ToString(Enum); + } + static inline std::string Abbrev() + { + return TypeAbbrev(Enum); + } + }; + + template + constexpr PerformanceMetric BasePerformanceMetricInfo::Enum; + + template <> + struct PerformanceMetricInfo + : public BasePerformanceMetricInfo + { + }; + template <> + struct PerformanceMetricInfo + : public BasePerformanceMetricInfo + { + }; + template <> + struct PerformanceMetricInfo + : public BasePerformanceMetricInfo + { + }; + + /** + * @} + */ +} // namespace Tensile + +namespace std +{ + template <> + struct hash + { + inline size_t operator()(Tensile::PerformanceMetric const& val) const + { + return hash()(static_cast(val)); + } + }; +} // namespace std diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Predicates.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Predicates.hpp new file mode 100644 index 0000000000..6f0a83d750 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Predicates.hpp @@ -0,0 +1,336 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include + +#include +#include + +namespace Tensile +{ + /** + * \addtogroup Properties + * + * @brief A Predicate is a Property whose value is of type `bool`. + * + * This allows it to represent a condition of an object, such as `object.size > + * 4`. + */ + namespace Predicates + { + /** + * @brief A Property whose value is of type `bool`. + * + * Predicates should generally implement better `debugEval()` + * functionality than other Properties since it's often helpful to know + * the value of both sides of a comparison. + */ + template + using Predicate = Property; + + /** + * \copydoc Tensile::Property_CRTP + */ + template + using Predicate_CRTP = Property_CRTP; + + /** + * \ingroup Properties + * \defgroup Predicates Predicate Classes + * + * @brief Individual Predicate classes. + */ + + /** + * \addtogroup Predicates + * @{ + */ + + template + class True : public Predicate_CRTP, Object> + { + public: + enum + { + HasIndex = false, + HasValue = false + }; + static std::string Type() + { + return "TruePred"; + } + + virtual bool operator()(Object const&) const + { + return true; + } + }; + + template + class False : public Predicate_CRTP, Object> + { + public: + enum + { + HasIndex = false, + HasValue = false + }; + static std::string Type() + { + return "FalsePred"; + } + + virtual bool operator()(Object const&) const + { + return false; + } + }; + + template + class And : public Predicate_CRTP, Object> + { + public: + enum + { + HasIndex = false, + HasValue = true + }; + std::vector>> value; + + And() = default; + And(std::initializer_list>> init) + : value(init) + { + } + And(std::vector>> const& init) + : value(init) + { + } + + static std::string Type() + { + return "And"; + } + + virtual bool operator()(Object const& obj) const + { + return std::all_of( + value.begin(), value.end(), [&obj](std::shared_ptr> pred) { + return (*pred)(obj); + }); + } + + virtual bool debugEval(Object const& obj, std::ostream& stream) const + { + bool rv = (*this)(obj); + + stream << this->type() << "("; + + bool first = true; + for(auto const& term : value) + { + if((*term)(obj)) + continue; + + if(!first) + stream << ", "; + first = false; + + term->debugEval(obj, stream); + } + + stream << "): " << rv << std::endl; + return rv; + } + }; + + template + class Or : public Predicate_CRTP, Object> + { + public: + enum + { + HasIndex = false, + HasValue = true + }; + std::vector>> value; + + Or() = default; + Or(std::initializer_list>> init) + : value(init) + { + } + Or(std::vector>> const& init) + : value(init) + { + } + + static std::string Type() + { + return "Or"; + } + + virtual bool operator()(Object const& obj) const + { + return std::any_of( + value.begin(), value.end(), [&obj](std::shared_ptr> pred) { + return (*pred)(obj); + }); + } + + virtual bool debugEval(Object const& obj, std::ostream& stream) const + { + bool rv = (*this)(obj); + + stream << this->type() << "("; + + bool first = true; + for(auto const& term : value) + { + if(!first) + stream << ", "; + first = false; + + term->debugEval(obj, stream); + } + + stream << "): " << rv << std::endl; + return rv; + } + }; + + template + class Not : public Predicate_CRTP, Object> + { + public: + enum + { + HasIndex = false, + HasValue = true + }; + std::shared_ptr> value; + Not() = default; + Not(std::shared_ptr> init) + : value(init) + { + } + + static std::string Type() + { + return "Not"; + } + + virtual bool operator()(Object const& obj) const + { + return !(*value)(obj); + } + + virtual bool debugEval(Object const& obj, std::ostream& stream) const + { + bool rv = (*this)(obj); + + stream << this->type() << "("; + value->debugEval(obj, stream); + stream << "): " << rv << std::endl; + + return rv; + } + }; + + /** + * @brief Predicate which returns `true` if the object is of type Subclass. + * + * @param Object + * @param Subclass Must be a subclass of Object. + */ + template + class IsSubclass : public Predicate_CRTP, Object> + { + public: + enum + { + HasIndex = false, + HasValue = true + }; + static_assert(std::is_base_of::value, + "Subclass must be derived from Object."); + + std::shared_ptr> value; + + IsSubclass() = default; + IsSubclass(std::shared_ptr> init) + : value(init) + { + } + + static std::string Type() + { + return Subclass::Type(); + } + + virtual bool operator()(Object const& obj) const override + { + auto const* sc = dynamic_cast(&obj); + if(!sc) + return false; + + return (*value)(*sc); + } + + virtual bool debugEval(Object const& obj, std::ostream& stream) const override + { + bool rv = (*this)(obj); + + stream << this->type() << "("; + + auto const* sc = dynamic_cast(&obj); + + if(sc) + { + stream << "matches: "; + value->debugEval(*sc, stream); + } + else + { + stream << "no match. actual type: " << typeid(obj).hash_code() << ", expected " + << typeid(Subclass).hash_code(); + } + + stream << "): " << rv; + + return rv; + } + }; + + /** + * @} + */ + } // namespace Predicates +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Properties.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Properties.hpp new file mode 100644 index 0000000000..7792db1a87 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Properties.hpp @@ -0,0 +1,193 @@ +/** + * Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include + +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \defgroup Properties Properties and Predicates + * + * @brief Abstract expression evaluation + * + * Property: \copydoc Tensile::Property + * + * Predicate: \copydoc Tensile::Predicates::Predicate + */ + + /** + * \addtogroup Properties + * @{ + */ + + /** + * @brief Simplifies implementation of `ToString()` for Property subclasses + * which may have `index` and/or `value` members. + */ + template + struct PropertyHelper + { + }; + + template + struct PropertyHelper + { + static std::string ToString(Class const& obj) + { + return obj.type(); + } + }; + + template + struct PropertyHelper + { + static std::string ToString(Class const& obj) + { + return concatenate(obj.type(), "(", obj.value, ")"); + } + }; + + template + struct PropertyHelper + { + static std::string ToString(Class const& obj) + { + return concatenate(obj.type(), "(", obj.index, ")"); + } + }; + + template + struct PropertyHelper + { + static std::string ToString(Class const& obj) + { + return concatenate(obj.type(), "(index=", obj.index, ", value=", obj.value, ")"); + } + }; + + /** + * Abstract object which retrieves a value from another object. + */ + template + class Property + { + public: + /** + * Name which uniquely identifies each subclass. + */ + virtual std::string type() const = 0; + virtual ~Property() = default; + + /** + * Retrieve the value from the specified object. + */ + virtual Value operator()(Object const& object) const = 0; + + virtual std::string toString() const = 0; + + /** + * Retrieve the value from the specified object, while printing + * relevant debug information to the specified stream. + */ + virtual Value debugEval(Object const& object, std::ostream& stream) const + { + Value rv = (*this)(object); + stream << *this << ": " << rv; + return rv; + } + }; + + /** + * @brief CRTP helper class which simplifies implementation of Property + * subclasses. + * + * Implements the `type()` and `toString()` methods automatically. + * + * The subclass must: + * - Implement a `static std::string Type()` function which returns a + * unique name for the class. + * - Have `HasIndex` and `HasValue` value definitions which match the + * reality of if it has `index` and/or `value` members. + * + * \see https://en.wikipedia.org/wiki/Curiously_recurring_template_pattern + */ + template + class Property_CRTP : public Property + { + public: + virtual std::string type() const final + { + return Class::Type(); + } + + virtual std::string toString() const + { + Class const& this_typed = dynamic_cast(*this); + return PropertyHelper::ToString(this_typed); + } + }; + + template + inline std::ostream& operator<<(std::ostream& stream, Property const& prop) + { + return stream << prop.toString(); + } + + template + inline std::ostream& + operator<<(std::ostream& stream, + std::vector>> const& props) + { + stream << "("; + + bool first = true; + for(auto const& v : props) + { + if(!first) + stream << ", "; + first = false; + + stream << *v; + } + + stream << ")"; + + return stream; + } + + /** + * @} + */ + + /** + * \ingroup Properties + * \defgroup PropertyClasses Property Classes + * + * @brief Individual Property classes + */ +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/PropertyMatching.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/PropertyMatching.hpp new file mode 100644 index 0000000000..ad67886828 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/PropertyMatching.hpp @@ -0,0 +1,671 @@ +/** + * Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \defgroup PropertyMatching Property Matching + * + * @brief Distance-based matching of Property values to a table. + * + * Generic algorithm for comparing an object to a table of predefined + * values based on Property objects and a Distance function. Used for + * MatchingLibrary. + */ + + /** + * \ingroup PropertyMatching + */ + namespace Matching + { + template + struct MatchingTableEntry + { + Key key; + Value value; + double speed; + }; + + template + struct MatchingTable + { + using Properties = std::vector>>; + using Transform = std::function; + + MatchingTable() = default; + MatchingTable(Properties const& properties) + : properties(properties) + { + } + + virtual ~MatchingTable() = default; + + virtual std::tuple findBestMatch(Object const& object, + Transform transform) const = 0; + + virtual ReturnValue findBestEvaluationSolution(Object const& object, + Hardware const& hardware, + Transform transform) const = 0; + + virtual std::vector matchesInOrder(Object const& object) const = 0; + + virtual std::string description() const = 0; + + virtual std::string distanceType() const = 0; + + Properties properties; + }; + + /** + * This exists to provide an abstraction around the different syntax of creating + * a vector of a size given at runtime vs. creating an array with a fixed size. + */ + template + struct KeyFactory + { + }; + + template + struct KeyFactory> + { + static std::vector MakeKey(size_t size) + { + return std::vector(size); + } + }; + + template + struct KeyFactory> + { + static std::array MakeKey(size_t size) + { + return std::array(); + } + }; + + /** + * Shared code between the generic DistanceMatchingTable and the specialization + * for the special Equality distance + */ + template + class DistanceMatchingCommon : public MatchingTable + { + public: + using Base = MatchingTable; + using Entry = MatchingTableEntry; + using Transform = typename Base::Transform; + using Properties = typename Base::Properties; + + DistanceMatchingCommon(ReturnValue nullValue = ReturnValue()) + : nullValue(nullValue) + { + } + + DistanceMatchingCommon(Properties const& properties, + ReturnValue nullValue = ReturnValue()) + : Base(properties) + , nullValue(nullValue) + { + } + + DistanceMatchingCommon(Distance const& distance, + Properties const& properties, + ReturnValue nullValue = ReturnValue()) + : Base(properties) + , nullValue(nullValue) + , distance(distance) + { + } + + virtual std::tuple findBestKeyMatch(Key const& key, + Transform transform) const = 0; + + virtual std::tuple + findBestMatch(Object const& object, Transform transform) const override + { + return findBestKeyMatch(keyForProblem(object), transform); + } + + virtual ReturnValue findBestEvaluationSolution(Object const& object, + Hardware const& hardware, + Transform transform) const override + { + double bestDistance = std::numeric_limits::max(); + + auto iter = this->table.begin(); + if(iter == this->table.end()) + return this->nullValue; + + ReturnValue theMatch = transform(iter->value); + + ReturnValue bestMatch = theMatch; + if(theMatch != nullptr) + { + size_t model_M = iter->key[0]; + size_t model_N = iter->key[1]; + size_t model_K = 1; + size_t model_NumBatches = 1; + + if(iter->key.size() > 3) + { + model_K = iter->key[3]; + model_NumBatches = iter->key[2]; + } + else + { + model_K = iter->key[2]; + } + bestDistance = theMatch->computeTAMScore(object, + hardware, + (double)model_M, + (double)model_N, + (double)model_K, + (double)model_NumBatches); + } + + iter++; + + while(iter != this->table.end()) + { + auto nextMatch = transform(iter->value); + + if(nextMatch != nullptr) + { + size_t model_M = iter->key[0]; + size_t model_N = iter->key[1]; + size_t model_K = 1; + size_t model_NumBatches = 1; + + if(iter->key.size() > 3) + { + model_K = iter->key[3]; + model_NumBatches = iter->key[2]; + } + else + { + model_K = iter->key[2]; + } + double nextDistance = theMatch->computeTAMScore(object, + hardware, + (double)model_M, + (double)model_N, + (double)model_K, + (double)model_NumBatches); + + if(nextDistance < bestDistance) + { + bestMatch = nextMatch; + bestDistance = nextDistance; + } + } + + ++iter; + } + + return bestMatch; + } + + virtual std::vector matchesInOrder(Object const& object) const override + { + return keyMatchesInOrder(keyForProblem(object)); + } + + std::vector keyMatchesInOrder(Key const& key) const + { + std::vector> indices(this->table.size()); + + for(size_t i = 0; i < this->table.size(); i++) + indices[i] = std::make_pair(distance(key, this->table[i].key), i); + + std::sort(indices.begin(), indices.end()); + + std::vector result; + result.reserve(this->table.size()); + + for(auto const& entry : indices) + result.push_back(this->table[entry.second].value); + + return result; + } + + Key keyForProblem(Object const& object) const + { + bool debug = Debug::Instance().printPropertyEvaluation(); + + Key myKey = KeyFactory::MakeKey(this->properties.size()); + + for(int i = 0; i < this->properties.size(); i++) + myKey[i] = (*this->properties[i])(object); + + if(debug) + { + std::cout << "Object key: "; + streamJoin(std::cout, myKey, ", "); + std::cout << std::endl; + } + + return myKey; + } + + virtual std::string description() const override + { + std::string rv = concatenate( + "Table: Properties: ", this->properties, ", ", table.size(), " rows, "); + + rv += concatenate("Distance: ", Distance::Type()); + + return rv; + } + + virtual std::string distanceType() const override + { + return Distance::Type(); + } + + protected: + std::vector table; + Distance distance; + + ReturnValue nullValue; + }; + + /** + * Generic version of DistanceMatchingTable + */ + template + class DistanceMatchingTable + : public DistanceMatchingCommon + { + public: + using Base = MatchingTable; + using Entry = MatchingTableEntry; + using Transform = typename Base::Transform; + using Properties = typename Base::Properties; + using Common = DistanceMatchingCommon; + using Common::distance; + using Common::nullValue; + using Common::table; + + DistanceMatchingTable(ReturnValue nullValue = ReturnValue()) + : Common(nullValue) + { + } + + DistanceMatchingTable(Properties const& properties, + ReturnValue nullValue = ReturnValue()) + : Common(properties, nullValue) + { + } + + DistanceMatchingTable(Distance const& distance, + Properties const& properties, + ReturnValue nullValue = ReturnValue()) + : Common(distance, properties, nullValue) + { + } + + std::tuple findBestKeyMatch(Key const& key, + Transform transform) const + { + const bool debug = Debug::Instance().printPropertyEvaluation(); + const bool naive = Debug::Instance().naivePropertySearch(); + + if(naive) + { + if(debug) + return findBestKeyMatch_NaiveSearch(key, transform); + else + return findBestKeyMatch_NaiveSearch(key, transform); + } + else + { + if(debug) + return findBestKeyMatch_BinSearch(key, transform); + else + return findBestKeyMatch_BinSearch(key, transform); + } + } + + template + std::tuple findBestKeyMatch_BinSearch(Key const& key, + Transform transform) const + { + if(this->table.empty()) + return std::make_tuple(this->nullValue, std::numeric_limits::max()); + + auto comp = [](Entry const& e, Key const& key) { return e.key < key; }; + + auto origIter = std::lower_bound(table.begin(), table.end(), key, comp); + + if(T_Debug) + { + std::cout << "Key: "; + streamJoin(std::cout, key, ", "); + std::cout << std::endl; + + std::cout << "Starting point: "; + streamJoin(std::cout, origIter->key, ", "); + std::cout << std::endl; + + std::cout << "Rightward search..." << std::endl; + } + + double bestDistance = std::numeric_limits::max(); + auto bestMatch = this->nullValue; + double bestSpeed = 0.0; + + ptrdiff_t count = 0; + + for(auto iter = origIter; iter != table.end(); iter++) + { + if(bestMatch && !distance.improvementPossible(key, iter->key, 0, bestDistance)) + { + if(T_Debug) + { + streamJoin(std::cout, iter->key, ", "); + std::cout << ": Stopping rightward search early." << std::endl; + } + + break; + } + + count++; + + auto myDistance = distance(key, iter->key); + bool thisMatch = false; + + if(myDistance < bestDistance + || (myDistance == bestDistance && iter->speed > bestSpeed)) + { + auto myMatch = transform(iter->value); + + if(myMatch) + { + bestDistance = myDistance; + bestMatch = myMatch; + bestSpeed = iter->speed; + thisMatch = true; + } + } + + if(T_Debug) + { + if(myDistance <= bestDistance) + std::cout << std::endl; + + std::cout << "speed: " << iter->speed << " | "; + streamJoin(std::cout, iter->key, ", "); + std::cout << ": " << myDistance; + + if(myDistance < bestDistance) + std::cout << " < "; + else if(myDistance > bestDistance) + std::cout << " > "; + else + std::cout << " == "; + + std::cout << bestDistance; + + if(myDistance < bestDistance) + { + if(thisMatch) + std::cout << " <-- Best so far"; + else + std::cout << " <-- Best distance, but no matching solution"; + } + + std::cout << std::endl; + } + } + + auto iter = table.rbegin(); + + if(origIter != table.end()) + iter = std::make_reverse_iterator(origIter); + + if(T_Debug) + std::cout << "Leftward search..." << std::endl; + + for(; iter != table.rend(); iter++) + { + if(bestMatch && !distance.improvementPossible(key, iter->key, 0, bestDistance)) + { + if(T_Debug) + { + streamJoin(std::cout, iter->key, ", "); + std::cout << ": Stopping leftward search early." << std::endl; + } + + break; + } + + count++; + + auto myDistance = distance(key, iter->key); + bool thisMatch = false; + + if(myDistance < bestDistance + || (myDistance == bestDistance && iter->speed > bestSpeed)) + { + auto myMatch = transform(iter->value); + + if(myMatch) + { + bestDistance = myDistance; + bestMatch = myMatch; + bestSpeed = iter->speed; + thisMatch = true; + } + } + + if(T_Debug) + { + if(myDistance <= bestDistance) + std::cout << std::endl; + + streamJoin(std::cout, iter->key, ", "); + std::cout << ": " << myDistance; + + if(myDistance < bestDistance) + std::cout << " < "; + else if(myDistance > bestDistance) + std::cout << " > "; + else + std::cout << " == "; + + std::cout << bestDistance; + + if(myDistance < bestDistance) + { + if(thisMatch) + std::cout << " <-- Best so far"; + else + std::cout << " <-- Best distance, but no matching solution"; + } + + std::cout << std::endl; + } + } + + if((T_Debug || Debug::Instance().printLookupEfficiency()) && table.size() > 0) + { + double considered = count; + considered /= table.size(); + considered *= 100; + std::cout << "Considered " << considered << "% of entries." << std::endl; + } + + if(T_Debug && bestMatch) + std::cout << "Solution index selected: " << bestMatch->index << std::endl; + + return std::make_tuple(bestMatch, bestDistance); + } + + template + std::tuple findBestKeyMatch_NaiveSearch(Key const& key, + Transform transform) const + { + double bestDistance = std::numeric_limits::max(); + + auto iter = this->table.begin(); + if(iter == this->table.end()) + return std::make_tuple(this->nullValue, bestDistance); + + ReturnValue bestMatch = transform(iter->value); + + if(bestMatch) + bestDistance = distance(key, iter->key); + + if(T_Debug) + { + std::cout << "Key: "; + streamJoin(std::cout, key, ", "); + std::cout << std::endl; + + streamJoin(std::cout, iter->key, ", "); + + std::cout << ": " << bestDistance << " <-- First" << std::endl; + } + + iter++; + + while(iter != this->table.end()) + { + auto myDistance = distance(key, iter->key); + bool thisMatch = false; + + if(myDistance < bestDistance) + { + auto myMatch = transform(iter->value); + + if(myMatch) + { + bestDistance = myDistance; + bestMatch = myMatch; + thisMatch = true; + } + } + + if(T_Debug) + { + streamJoin(std::cout, iter->key, ", "); + std::cout << ": " << myDistance; + if(myDistance < bestDistance) + { + std::cout << " <-- Best so far"; + + if(thisMatch) + std::cout << " (has a matching solution)"; + else + std::cout << " (no match)"; + } + + std::cout << std::endl; + } + + iter++; + } + + if(T_Debug && bestMatch) + std::cout << "Solution index selected: " << bestMatch->index << std::endl; + + return std::make_tuple(bestMatch, bestDistance); + } + }; + + /** + * Specialization of DistanceMatchingTable for Equality Distance. This special case will + * only select key in the table if it exactly matches the provided key + */ + template + class DistanceMatchingTable> + : public DistanceMatchingCommon> + { + public: + using Base = MatchingTable; + using Entry = MatchingTableEntry; + using Transform = typename Base::Transform; + using Properties = typename Base::Properties; + using Equality = Matching::Equality; + using Common + = DistanceMatchingCommon>; + using Common::distance; + using Common::nullValue; + using Common::table; + + DistanceMatchingTable(ReturnValue nullValue = ReturnValue()) + : Common(nullValue) + { + } + + DistanceMatchingTable(Properties const& properties, + ReturnValue nullValue = ReturnValue()) + : Common(properties, nullValue) + { + } + + DistanceMatchingTable(Equality const& distance, + Properties const& properties, + ReturnValue nullValue = ReturnValue()) + : Common(distance, properties, nullValue) + { + } + + std::tuple findBestKeyMatch(Key const& key, + Transform transform) const + { + auto comp = [](Entry const& e, Key const& key) { return e.key < key; }; + auto iter = std::lower_bound(table.begin(), table.end(), key, comp); + + return (iter->key == key) + ? std::make_tuple(transform(iter->value), 0.0) + : std::make_tuple(this->nullValue, std::numeric_limits::max()); + } + }; + } // namespace Matching +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ScalarValueTypes.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ScalarValueTypes.hpp new file mode 100644 index 0000000000..44bea088b8 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ScalarValueTypes.hpp @@ -0,0 +1,160 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \defgroup ScalarValue Scalar value type Info + * + * @brief Definitions and metadata on scalar value enumerations. + */ + + /** + * \ingroup ScalarValue + * @{ + */ + + /** + * Scalar value + */ + enum class ScalarValue : int + { + Any, + One, + NegativeOne, + Count + }; + + std::string ToString(ScalarValue d); + std::string TypeAbbrev(ScalarValue d); + std::ostream& operator<<(std::ostream& stream, ScalarValue const& t); + std::istream& operator>>(std::istream& stream, ScalarValue& t); + + /** + * \ingroup ScalarValue + * \brief Runtime accessible scalar value type metadata + */ + struct ScalarValueTypeInfo + { + static ScalarValueTypeInfo const& Get(int index); + static ScalarValueTypeInfo const& Get(ScalarValue t); + static ScalarValueTypeInfo const& Get(std::string const& str); + + ScalarValue m_value; + std::string name; + + private: + static void registerAllTypeInfo(); + static void registerAllTypeInfoOnce(); + + template + static void registerTypeInfo(); + + static void addInfoObject(ScalarValueTypeInfo const& info); + + static std::map data; + static std::map typeNames; + }; + + /** + * \ingroup ScalarValue + * \brief Compile-time accessible scalar type metadata. + */ + template + struct ScalarValueInfo; + + template + struct BaseScalarValueInfo + { + constexpr static ScalarValue Enum = T_Enum; + + static inline std::string Name() + { + return ToString(Enum); + } + static inline std::string Abbrev() + { + return TypeAbbrev(Enum); + } + }; + + template + constexpr ScalarValue BaseScalarValueInfo::Enum; + + template <> + struct ScalarValueInfo : public BaseScalarValueInfo + { + }; + template <> + struct ScalarValueInfo : public BaseScalarValueInfo + { + }; + template <> + struct ScalarValueInfo + : public BaseScalarValueInfo + { + }; + + /** +* \ingroup ScalarValue +* \brief Gets ScalarValue enum from value. Returns ScalarValue::Any if there is no match. +*/ + template + ScalarValue toScalarValueEnum(T value) + { + if(value == T(1)) + return ScalarValue::One; + else if(value == T(-1)) + return ScalarValue::NegativeOne; + else + return ScalarValue::Any; + } + + /** + * @} + */ +} // namespace Tensile + +namespace std +{ + template <> + struct hash + { + inline size_t operator()(Tensile::ScalarValue const& val) const + { + return hash()(static_cast(val)); + } + }; +} // namespace std diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization.hpp new file mode 100644 index 0000000000..5ff4197519 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization.hpp @@ -0,0 +1,42 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Base.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Base.hpp new file mode 100644 index 0000000000..b25d670a47 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Base.hpp @@ -0,0 +1,430 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace Tensile +{ + namespace Serialization + { + struct EmptyContext + { + }; + + template + struct IOTraits + { + }; + + template + struct MappingTraits + { + static const bool flow = false; + }; + + template + struct CustomMappingTraits + { + static const bool flow = false; + }; + + template + struct SequenceTraits + { + using Value = int; + static const bool flow = false; + }; + + template + struct EnumTraits + { + }; + + template + struct EmptyMappingTraits + { + using iot = IOTraits; + static_assert(Object::HasValue == false, + "Object has a value. Use the value base class."); + static void mapping(IO& io, Object& obj) {} + + const static bool flow = true; + }; + + template + struct ValueMappingTraits + { + using iot = IOTraits; + static_assert(Object::HasValue == true, + "Object has no value. Use the empty base class."); + static void mapping(IO& io, Object& obj) + { + iot::mapRequired(io, "value", obj.value); + } + + const static bool flow = true; + }; + + template + struct IndexMappingTraits + { + using iot = IOTraits; + static_assert(Object::HasIndex == true, + "Object doesn't have index/value. Use the empty base class."); + static void mapping(IO& io, Object& obj) + { + iot::mapRequired(io, "index", obj.index); + } + + const static bool flow = true; + }; + + template + struct IndexValueMappingTraits + { + using iot = IOTraits; + static_assert(Object::HasIndex == true && Object::HasValue == true, + "Object doesn't have index/value. Use the empty base class."); + static void mapping(IO& io, Object& obj) + { + iot::mapRequired(io, "index", obj.index); + iot::mapRequired(io, "value", obj.value); + } + + const static bool flow = true; + }; + + template + struct AutoMappingTraits + { + }; + + template + struct AutoMappingTraits : public EmptyMappingTraits + { + }; + + template + struct AutoMappingTraits : public ValueMappingTraits + { + }; + + template + struct AutoMappingTraits : public IndexMappingTraits + { + }; + + template + struct AutoMappingTraits + : public IndexValueMappingTraits + { + }; + + template + struct SubclassMappingTraits + { + }; + + template + struct PointerMappingTraits; + + template + struct PointerMappingTraits + { + using iot = IOTraits; + template + static bool mapping(IO& io, std::shared_ptr& p, Context& ctx) + { + std::shared_ptr sc; + + if(iot::outputting(io)) + { + sc = std::dynamic_pointer_cast(p); + } + else + { + sc = std::make_shared(); + p = sc; + } + + MappingTraits::mapping(io, *sc, ctx); + + return true; + } + }; + + template + struct PointerMappingTraits + { + using iot = IOTraits; + template + static bool mapping(IO& io, std::shared_ptr& p) + { + std::shared_ptr sc; + + if(iot::outputting(io)) + { + sc = std::dynamic_pointer_cast(p); + } + else + { + sc = std::make_shared(); + p = sc; + } + + MappingTraits::mapping(io, *sc); + + return true; + } + }; + + template + struct BaseClassMappingTraits + { + using iot = IOTraits; + + static void mapping(IO& io, std::shared_ptr& value) + { + std::string type; + + if(iot::outputting(io)) + type = value->type(); + + iot::mapRequired(io, "type", type); + + if(!SubclassMappingTraits::mapping(io, type, value)) + iot::setError(io, "Unknown subclass type " + type); + } + + const static bool flow = Flow; + }; + + template + struct DefaultSubclassMappingTraits; + + template + struct DefaultSubclassMappingTraits + { + using iot = IOTraits; + using SubclassFn = bool(IO&, typename std::shared_ptr&, Context&); + using SubclassMap = std::unordered_map>; + + template + static typename SubclassMap::value_type Pair() + { + auto f = PointerMappingTraits::template mapping; + return typename SubclassMap::value_type(Subclass::Type(), f); + } + + static bool + mapping(IO& io, std::string const& type, std::shared_ptr& p, Context& ctx) + { + auto iter = CRTP_Traits::subclasses.find(type); + if(iter != CRTP_Traits::subclasses.end()) + return iter->second(io, p, ctx); + return false; + } + }; + + template + struct DefaultSubclassMappingTraits + { + using iot = IOTraits; + using SubclassFn = bool(IO&, typename std::shared_ptr&); + using SubclassMap = std::unordered_map>; + + template + static typename SubclassMap::value_type Pair() + { + auto f = PointerMappingTraits::template mapping; + return typename SubclassMap::value_type(Subclass::Type(), f); + } + + static bool mapping(IO& io, std::string const& type, std::shared_ptr& p) + { + auto iter = CRTP_Traits::subclasses.find(type); + if(iter != CRTP_Traits::subclasses.end()) + return iter->second(io, p); + return false; + } + }; + + template + struct SequenceTraits, IO> + { + using Value = Data; + + static size_t size(IO& io, vector2& v) + { + return 2; + } + static Data& element(IO& io, vector2& v, size_t index) + { + if(index == 0) + return v.x; + return v.y; + } + + const static bool flow = true; + }; + + template + struct SequenceTraits, IO> + { + using Value = Data; + + static size_t size(IO& io, vector3& v) + { + return 3; + } + static Data& element(IO& io, vector3& v, size_t index) + { + if(index == 0) + return v.x; + if(index == 1) + return v.y; + return v.z; + } + + const static bool flow = true; + }; + + template + struct SequenceTraits, IO> + { + using Value = Data; + + static size_t size(IO& io, vector4& v) + { + return 4; + } + static Data& element(IO& io, vector4& v, size_t index) + { + if(index == 0) + return v.x; + if(index == 1) + return v.y; + if(index == 2) + return v.z; + return v.w; + } + + const static bool flow = true; + }; + + template + struct EnumTraits + { + using iot = IOTraits; + + static void enumeration(IO& io, DataType& value) + { + for(int i = 0; i < static_cast(DataType::Count); i++) + { + auto const& info = DataTypeInfo::Get(i); + iot::enumCase(io, value, info.name.c_str(), info.dataType); + } + } + }; + + template + struct EnumTraits + { + using iot = IOTraits; + + static void enumeration(IO& io, ActivationType& value) + { + for(int i = 0; i < static_cast(ActivationType::Count); i++) + { + auto const& info = static_cast(i); + iot::enumCase(io, value, Tensile::ToString(info).c_str(), info); + } + } + }; + + template + struct EnumTraits + { + using iot = IOTraits; + + static void enumeration(IO& io, KernelLanguage& value) + { + for(int i = 0; i < static_cast(KernelLanguage::Count); i++) + { + auto const& info = KernelLanguageTypeInfo::Get(i); + iot::enumCase(io, value, info.name.c_str(), info.m_kernelLanguage); + } + } + }; + + template + struct EnumTraits + { + using iot = IOTraits; + + static void enumeration(IO& io, ArithmeticUnit& value) + { + for(int i = 0; i < static_cast(ArithmeticUnit::Count); i++) + { + auto const& info = ArithmeticUnitTypeInfo::Get(i); + iot::enumCase(io, value, info.name.c_str(), info.m_arithmeticUnit); + } + } + }; + + template + struct EnumTraits + { + using iot = IOTraits; + + static void enumeration(IO& io, ScalarValue& value) + { + for(int i = 0; i < static_cast(ScalarValue::Count); i++) + { + auto const& info = ScalarValueTypeInfo::Get(i); + iot::enumCase(io, value, info.name.c_str(), info.m_value); + } + } + }; + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Containers.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Containers.hpp new file mode 100644 index 0000000000..34cce31c99 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Containers.hpp @@ -0,0 +1,221 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace Tensile +{ + namespace Serialization + { + template + struct KeyConversion + { + static std::string toString(Key const& value) + { + return concatenate(value); + } + + static Key fromString(std::string const& value) + { + std::istringstream stream(value); + Key rv; + stream >> rv; + + return rv; + } + }; + + template <> + struct KeyConversion + { + static std::string const& toString(std::string const& value) + { + return value; + } + + static std::string const& fromString(std::string const& value) + { + return value; + } + }; + + template + struct DefaultCustomMappingTraits + { + using iot = IOTraits; + using key_type = typename Map::key_type; + using mapped_type = typename Map::mapped_type; + + static void inputOne(IO& io, std::string const& keyStr, Map& value) + { + iot::mapRequired( + io, keyStr.c_str(), value[KeyConversion::fromString(keyStr)]); + } + + static void output(IO& io, Map& value) + { + if(Sort) + { + std::vector keys; + keys.reserve(value.size()); + for(auto const& pair : value) + keys.push_back(pair.first); + std::sort(keys.begin(), keys.end()); + + for(auto const& key : keys) + { + auto keyStr = KeyConversion::toString(key); + iot::mapRequired(io, keyStr.c_str(), value.find(key)->second); + } + } + else + { + for(auto& pair : value) + { + auto keyStr = KeyConversion::toString(pair.first); + iot::mapRequired(io, keyStr.c_str(), pair.second); + } + } + } + + static const bool flow = Flow; + }; + + template + struct CustomMappingTraits, IO> + : public DefaultCustomMappingTraits, IO, false, true> + { + }; + + template + struct CustomMappingTraits, IO> + : public DefaultCustomMappingTraits, IO, false, true> + { + }; + + template + struct CustomMappingTraits, IO> + : public DefaultCustomMappingTraits, IO, false, true> + { + }; + + template + struct DefaultSequenceTraits + { + using Value = typename Seq::value_type; + + static size_t size(IO& io, Seq& s) + { + return s.size(); + } + static Value& element(IO& io, Seq& s, size_t index) + { + if(index >= s.size()) + { + size_t n = index - s.size() + 1; + s.insert(s.end(), n, Value()); + } + + return s[index]; + } + + const static bool flow = Flow; + }; + +#define TENSILE_SERIALIZE_VECTOR(flow, ...) \ + template \ + struct SequenceTraits, IO> \ + : public DefaultSequenceTraits, IO, flow> \ + { \ + } + + TENSILE_SERIALIZE_VECTOR(true, ExactSelectionTableEntry); + + TENSILE_SERIALIZE_VECTOR(true, + Tensile::ExactLogicLibrary::Row); + + TENSILE_SERIALIZE_VECTOR( + true, + Tensile::ExactLogicLibrary>::Row); + + TENSILE_SERIALIZE_VECTOR( + true, std::shared_ptr>); + TENSILE_SERIALIZE_VECTOR( + true, std::shared_ptr>); + TENSILE_SERIALIZE_VECTOR(true, + std::shared_ptr>); + TENSILE_SERIALIZE_VECTOR(true, + std::shared_ptr>); + + TENSILE_SERIALIZE_VECTOR(false, std::shared_ptr); + + template + struct SequenceTraits>, IO> + : public DefaultSequenceTraits< + std::vector>, + IO, + false> + { + }; + + template + struct SequenceTraits, IO> + { + using Value = T; + static size_t size(IO& io, std::array& v) + { + return N; + } + static T& element(IO& io, std::array& v, size_t index) + { + if(index >= N) + { + IOTraits::setError(io, + concatenate("invalid array index ", index)); + } + + return v[index]; + } + + const static bool flow = (N < 10); + }; + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionPredicates.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionPredicates.hpp new file mode 100644 index 0000000000..6c83fb6af4 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionPredicates.hpp @@ -0,0 +1,378 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include +#include + +namespace Tensile +{ + namespace Serialization + { + template + struct SubclassMappingTraits, IO> + : public DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + Predicates::Predicate, + IO> + { + using Self = SubclassMappingTraits, IO>; + using Base = DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + Predicates::Predicate, + IO>; + using SubclassMap = typename Base::SubclassMap; + const static SubclassMap subclasses; + + using Generic = PredicateMappingTraits; + + static SubclassMap GetSubclasses() + { + SubclassMap rv({ + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + }); + + auto gmap = Generic::GetSubclasses(); + rv.insert(gmap.begin(), gmap.end()); + + return rv; + } + }; + + template + using ContractionProblemPredicateSMT + = SubclassMappingTraits, IO>; + + template + const typename ContractionProblemPredicateSMT::SubclassMap + ContractionProblemPredicateSMT::subclasses + = ContractionProblemPredicateSMT::GetSubclasses(); + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionSolution.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionSolution.hpp new file mode 100644 index 0000000000..375c1103b1 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionSolution.hpp @@ -0,0 +1,160 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * Copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include +#include + +namespace Tensile +{ + namespace Serialization + { + template + struct MappingTraits, IO> + { + static void mapping(IO& io, std::shared_ptr& p) + { + PointerMappingTraits::mapping(io, p); + } + + const static bool flow = false; + }; + + template + struct MappingTraits + { + using iot = IOTraits; + static void mapping(IO& io, ContractionSolution& s) + { + iot::mapRequired(io, "name", s.kernelName); + iot::mapRequired(io, "index", s.index); + + iot::mapRequired(io, "hardwarePredicate", s.hardwarePredicate); + iot::mapRequired(io, "problemPredicate", s.problemPredicate); + + iot::mapRequired(io, "debugKernel", s.debugKernel); + iot::mapRequired(io, "info", s.info); + iot::mapOptional(io, "ideals", s.ideals); + iot::mapOptional(io, "linearModel", s.linearModel); + + iot::mapRequired(io, "sizeMapping", s.sizeMapping); + iot::mapRequired(io, "problemType", s.problemType); + } + + const static bool flow = false; + }; + + template + struct MappingTraits + { + using iot = IOTraits; + static void mapping(IO& io, ContractionSolution::SizeMapping& s) + { + iot::mapRequired(io, "workGroup", s.workGroupSize); + iot::mapRequired(io, "threadTile", s.threadTile); + iot::mapRequired(io, "macroTile", s.macroTile); + + iot::mapRequired(io, "staggerU", s.staggerU); + iot::mapRequired(io, "depthU", s.depthU); + iot::mapRequired(io, "globalSplitU", s.globalSplitU); + iot::mapRequired(io, "staggerStrideShift", s.staggerStrideShift); + iot::mapRequired(io, "workGroupMapping", s.workGroupMapping); + + iot::mapOptional(io, "packBatchDims", s.packBatchDims); + iot::mapOptional(io, "packSummationDims", s.packSummationDims); + iot::mapOptional(io, "magicDivAlg", s.magicDivAlg); + iot::mapOptional(io, "persistentKernel", s.persistentKernel); + iot::mapOptional(io, "persistentKernelAlongBatch", s.persistentKernelAlongBatch); + iot::mapRequired(io, "sourceKernel", s.sourceKernel); + + iot::mapRequired(io, "globalAccumulation", s.globalAccumulation); + iot::mapRequired(io, "workspaceSizePerElemC", s.workspaceSizePerElemC); + + iot::mapOptional(io, "activationFused", s.activationFused); + } + + const static bool flow = false; + }; + + template + struct MappingTraits + { + using iot = IOTraits; + static void mapping(IO& io, ContractionSolution::ProblemType& s) + { + iot::mapRequired(io, "operationIdentifier", s.operationIdentifier); + + iot::mapRequired(io, "aType", s.aType); + iot::mapRequired(io, "bType", s.bType); + iot::mapRequired(io, "cType", s.cType); + iot::mapRequired(io, "dType", s.dType); + iot::mapRequired(io, "useBeta", s.useBeta); + iot::mapOptional(io, "useBias", s.useBias); + iot::mapRequired(io, "highPrecisionAccumulate", s.highPrecisionAccumulate); + iot::mapOptional(io, "useInitialStridesAB", s.useInitialStridesAB); + iot::mapOptional(io, "useInitialStridesCD", s.useInitialStridesCD); + iot::mapOptional(io, "stridedBatched", s.stridedBatched); + iot::mapOptional(io, "fp16AltImpl", s.fp16AltImpl); + iot::mapOptional(io, "activationType", s.activationType); + iot::mapOptional(io, "activationHPA", s.activationHPA); + } + + const static bool flow = false; + }; + + template + struct MappingTraits + { + using iot = IOTraits; + static void mapping(IO& io, ContractionSolution::LinearModel& s) + { + iot::mapOptional(io, "slope", s.slope); + iot::mapOptional(io, "intercept", s.intercept); + iot::mapOptional(io, "max", s.max); + } + + const static bool flow = false; + }; + + template + struct MappingTraits + { + using iot = IOTraits; + static void mapping(IO& io, BufferLoadCheckPacket& s) + { + iot::mapRequired(io, "ShiftPtrElemA", s.shiftPtrElemA); + iot::mapRequired(io, "ShiftPtrElemB", s.shiftPtrElemB); + iot::mapRequired(io, "DUorMT0", s.depthUorMT0); + iot::mapRequired(io, "DUorMT1", s.depthUorMT1); + } + + const static bool flow = false; + }; + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ExactLogicLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ExactLogicLibrary.hpp new file mode 100644 index 0000000000..ce61ce9226 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ExactLogicLibrary.hpp @@ -0,0 +1,81 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include + +namespace Tensile +{ + namespace Serialization + { + template + struct MappingTraits, IO> + { + using Library = HardwareSelectionLibrary; + using iot = IOTraits; + + static void mapping(IO& io, Library& lib) + { + iot::mapRequired(io, "rows", lib.rows); + } + + const static bool flow = false; + }; + + template + struct MappingTraits, IO> + { + using Library = ProblemSelectionLibrary; + using iot = IOTraits; + + static void mapping(IO& io, Library& lib) + { + iot::mapRequired(io, "rows", lib.rows); + } + + const static bool flow = false; + }; + + template + struct MappingTraits, IO> + { + using Row = typename ExactLogicLibrary::Row; + using iot = IOTraits; + + static void mapping(IO& io, Row& row) + { + iot::mapRequired(io, "predicate", row.first.value); + iot::mapRequired(io, "library", row.second); + } + + const static bool flow = false; + }; + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/GranularitySelectionLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/GranularitySelectionLibrary.hpp new file mode 100644 index 0000000000..8fc47811d2 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/GranularitySelectionLibrary.hpp @@ -0,0 +1,124 @@ + +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +namespace Tensile +{ + namespace Serialization + { + template + struct MappingTraits + { + using Entry = ExactSelectionTableEntry; //; + using iot = IOTraits; + + static void mapping(IO& io, Entry& entry) + { + iot::mapRequired(io, "key", entry.key); + iot::mapRequired(io, "value", entry.value); + } + + const static bool flow = true; + }; + + template + struct MappingTraits, IO> + { + using Library = GranularitySelectionLibrary; + + using iot = IOTraits; + + static void mapping(IO& io, Library& lib) + { + SolutionMap* ctx + = static_cast*>(iot::getContext(io)); + if(ctx == nullptr) + { + iot::setError(io, + "GranularitySelectionLibrary requires that context be " + "set to a SolutionMap."); + } + + std::vector mappingIndices; + std::vector mapEntries; + if(iot::outputting(io)) + { + mappingIndices.reserve(lib.solutions.size()); + + for(auto const& pair : lib.solutions) + mappingIndices.push_back(pair.first); + + iot::mapRequired(io, "indices", mappingIndices); + + for(auto it = lib.exactMap.begin(); it != lib.exactMap.end(); ++it) + { + ExactSelectionTableEntry newEntry; + newEntry.key = it->first; + newEntry.value = it->second; + mapEntries.push_back(newEntry); + } + iot::mapRequired(io, "exact", mapEntries); + } + else + { + iot::mapRequired(io, "indices", mappingIndices); + if(mappingIndices.empty()) + iot::setError(io, + "GranularitySelectionLibrary requires non empty " + "mapping index set."); + + for(int index : mappingIndices) + { + auto slnIter = ctx->find(index); + if(slnIter == ctx->end()) + { + iot::setError(io, concatenate("Invalid solution index: ", index)); + } + else + { + auto solution = slnIter->second; + lib.solutions.insert(std::make_pair(index, solution)); + } + } + + iot::mapRequired(io, "exact", mapEntries); + + for(ExactSelectionTableEntry entry : mapEntries) + { + std::vector key = entry.key; + int value = entry.value; + lib.exactMap.insert(std::pair, int>(key, value)); + } + } + } + }; + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/HasTraits.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/HasTraits.hpp new file mode 100644 index 0000000000..733a54c45f --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/HasTraits.hpp @@ -0,0 +1,133 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include + +namespace Tensile +{ + namespace Serialization + { + template + struct SameType; + + template + class has_MappingTraits + { + using mapping = void (*)(IO&, T&, Context&); + + template + static uint8_t test(SameType*); + + template + static uint32_t test(...); + + public: + static const bool value = sizeof(test>(nullptr)) == 1; + }; + + template + class has_EmptyMappingTraits + { + using mapping = void (*)(IO&, T&); + + template + static uint8_t test(SameType*); + + template + static uint32_t test(...); + + public: + static const bool value = sizeof(test>(nullptr)) == 1; + }; + + template + class has_EnumTraits + { + using enumeration = void (*)(IO&, T&); + + template + static uint8_t test(SameType*); + + template + static uint32_t test(...); + + public: + static const bool value = sizeof(test>(nullptr)) == 1; + }; + + template + class has_SequenceTraits + { + using size = size_t (*)(IO&, T&); + + template + static uint8_t test(SameType*); + + template + static uint32_t test(...); + + public: + static const bool value = sizeof(test>(nullptr)) == 1; + }; + + template + class has_CustomMappingTraits + { + using inputOne = void (*)(IO&, std::string const&, T&); + using output = void (*)(IO&, T&); + + template + static uint8_t test(SameType*, SameType*); + + template + static uint32_t test(...); + + public: + static const bool value + = sizeof(test>(nullptr, nullptr)) == 1; + }; + + template + struct has_SerializationTraits + { + static const bool value0 = has_EmptyMappingTraits::value; + static const bool value1 = has_EnumTraits::value; + static const bool value2 = has_SequenceTraits::value; + static const bool value3 = has_CustomMappingTraits::value; + + static const int count = value0 + value1 + value2 + value3; + + static_assert(count == 0 || count == 1, "Ambiguous serialization!"); + + static const bool value = value0 || value1 || value2 || value3; + }; + + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/MapLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/MapLibrary.hpp new file mode 100644 index 0000000000..28c6420749 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/MapLibrary.hpp @@ -0,0 +1,64 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include +#include + +namespace Tensile +{ + namespace Serialization + { + template + struct MappingTraits, IO> + { + using Library = ProblemMapLibrary; + using iot = IOTraits; + + static void mapping(IO& io, Library& lib) + { + iot::mapRequired(io, "property", lib.property); + iot::mapRequired(io, "map", lib.map); + } + + const static bool flow = false; + }; + + template + struct CustomMappingTraits, IO> + : public DefaultCustomMappingTraits< + Tensile::LibraryMap, + IO, + true, + false> + { + }; + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/MatchingLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/MatchingLibrary.hpp new file mode 100644 index 0000000000..fe2a42607d --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/MatchingLibrary.hpp @@ -0,0 +1,295 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include + +#include +#include + +namespace Tensile +{ + namespace Serialization + { + template + struct MappingTraits< + Matching::DistanceMatchingTable, + IO> + { + using Table + = Matching::DistanceMatchingTable; + using iot = IOTraits; + + static void mapping(IO& io, Table& table) + { + iot::mapRequired(io, "table", table.table); + + if(!iot::outputting(io)) + { + using Entry = typename Table::Entry; + auto comp = [](Entry const& e1, Entry const& e2) { + return e1.key < e2.key || (e1.key == e2.key && e1.speed > e2.speed); + }; + std::sort(table.table.begin(), table.table.end(), comp); + } + } + + const static bool flow = false; + }; + + template + struct MappingTraits, IO> + { + using Library = ProblemMatchingLibrary; + using Properties = typename Library::Table::Properties; + using Element = typename Library::Element; + + using iot = IOTraits; + + static void mapping(IO& io, Library& lib) + { + Properties properties; + if(iot::outputting(io)) + { + properties = lib.table->properties; + } + + iot::mapRequired(io, "properties", properties); + + bool success = false; + if(properties.size() == 0) + iot::setError(io, "Matching table must have at least one property."); + else if(properties.size() == 1) + success = mappingKey>(io, lib, properties); + else if(properties.size() == 2) + success = mappingKey>(io, lib, properties); + else if(properties.size() == 3) + success = mappingKey>(io, lib, properties); + else if(properties.size() == 4) + success = mappingKey>(io, lib, properties); + else if(properties.size() == 5) + success = mappingKey>(io, lib, properties); + else if(properties.size() == 6) + success = mappingKey>(io, lib, properties); + else if(properties.size() == 7) + success = mappingKey>(io, lib, properties); + else if(properties.size() == 8) + success = mappingKey>(io, lib, properties); + else if(properties.size() == 9) + success = mappingKey>(io, lib, properties); + else if(properties.size() == 10) + success = mappingKey>(io, lib, properties); + + if(!success) + success = mappingKey>(io, lib, properties); + + if(!success) + iot::setError(io, "Can't write out key: wrong type."); + } + + template + static bool mappingKey(IO& io, Library& lib, Properties const& properties) + { + std::string distanceType; + + if(iot::outputting(io)) + distanceType = lib.table->distanceType(); + + iot::mapRequired(io, "distance", distanceType); + + bool success = false; + + std::string tensile_metric = Debug::Instance().getMetric(); + + if(!tensile_metric.empty()) + distanceType = tensile_metric; + + if(distanceType == "Euclidean") + { + success = mappingDistance>( + io, lib, properties); + } + else if(distanceType == "Equality") + { + success = mappingDistance>(io, lib, properties); + } + else if(distanceType == "JSD") + { + success + = mappingDistance>(io, lib, properties); + } + else if(distanceType == "Manhattan") + { + success = mappingDistance>( + io, lib, properties); + } + else if(distanceType == "Ratio") + { + success + = mappingDistance>(io, lib, properties); + } + else if(distanceType == "Random") + { + success + = mappingDistance>(io, lib, properties); + } + else if(distanceType == "GridBased") + { + success + = mappingDistance>(io, lib, properties); + } + else + { + iot::setError(io, concatenate("Unknown distance function", distanceType)); + } + + return success; + } + + template + static bool mappingDistance(IO& io, Library& lib, Properties const& properties) + { + using Table = Matching::DistanceMatchingTable, + Distance>; + + std::shared_ptr
table; + + if(iot::outputting(io)) + { + table = std::dynamic_pointer_cast
(lib.table); + if(!table) + return false; + } + else + { + table = std::make_shared
(); + table->properties = properties; + lib.table = table; + } + + MappingTraits::mapping(io, *table); + + return true; + } + + const static bool flow = false; + }; + + template + struct MappingTraits, IO> + { + using Entry = Matching::MatchingTableEntry; + using iot = IOTraits; + + static void mapping(IO& io, Entry& entry) + { + iot::mapRequired(io, "key", entry.key); + iot::mapRequired(io, "value", entry.value); + iot::mapRequired(io, "speed", entry.speed); + } + + const static bool flow = true; + }; + + template + struct MappingTraits>, IO> + : public BaseClassMappingTraits, IO, true> + { + }; + + template + struct SubclassMappingTraits, IO> + : public DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + Matching::Distance, + IO> + { + using Self = SubclassMappingTraits, IO>; + using Base + = DefaultSubclassMappingTraits, IO>, + Matching::Distance, + IO>; + + using SubclassMap = typename Base::SubclassMap; + const static SubclassMap subclasses; + + static SubclassMap GetSubclasses() + { + return SubclassMap({Base::template Pair>(), + Base::template Pair>(), + Base::template Pair>(), + Base::template Pair>(), + Base::template Pair>()}); + } + }; + + template + const typename SubclassMappingTraits, IO>::SubclassMap + SubclassMappingTraits, IO>::subclasses + = SubclassMappingTraits, IO>::GetSubclasses(); + + template + struct MappingTraits, IO> + : public AutoMappingTraits, IO> + { + }; + + template + struct MappingTraits, IO> + : public AutoMappingTraits, IO> + { + }; + + template + struct MappingTraits, IO> + : public AutoMappingTraits, IO> + { + }; + + template + struct MappingTraits, IO> + : public AutoMappingTraits, IO> + { + }; + + template + struct MappingTraits, IO> + : public AutoMappingTraits, IO> + { + }; + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Predicates.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Predicates.hpp new file mode 100644 index 0000000000..c06752668d --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Predicates.hpp @@ -0,0 +1,224 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include +#include +#include + +namespace Tensile +{ + namespace Serialization + { + + template + struct MappingTraits>, IO> + : public BaseClassMappingTraits, IO, true> + { + }; + + /** + * This is not part of the main traits classes but it eliminates the duplication + * of the "Base" predicate types for different object types. + */ + template + struct PredicateMappingTraits + : public DefaultSubclassMappingTraits, + Predicates::Predicate, + IO> + { + using Self = PredicateMappingTraits; + using Base = DefaultSubclassMappingTraits, IO>; + using SubclassMap = typename Base::SubclassMap; + const static SubclassMap subclasses; + + static typename Base::SubclassMap GetSubclasses() + { + return SubclassMap({Base::template Pair>(), + Base::template Pair>(), + Base::template Pair>(), + Base::template Pair>(), + Base::template Pair>()}); + } + }; + + template + const typename PredicateMappingTraits::SubclassMap + PredicateMappingTraits::subclasses + = PredicateMappingTraits::GetSubclasses(); + + template + struct MappingTraits, IO> + : public AutoMappingTraits, IO> + { + }; + + template + struct MappingTraits, IO> + : public AutoMappingTraits, IO> + { + }; + + template + struct MappingTraits, IO> + : public AutoMappingTraits, IO> + { + }; + + template + struct MappingTraits, IO> + : public AutoMappingTraits, IO> + { + }; + + template + struct MappingTraits, IO> + : public AutoMappingTraits, IO> + { + }; + + template + struct SubclassMappingTraits, IO> + : public DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + Predicates::Predicate, + IO> + { + using Self = SubclassMappingTraits, IO>; + using Base = DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + Predicates::Predicate, + IO>; + using SubclassMap = typename Base::SubclassMap; + const static SubclassMap subclasses; + + using Generic = PredicateMappingTraits; + + static SubclassMap GetSubclasses() + { + SubclassMap rv({Base::template Pair>()}); + + auto gmap = Generic::GetSubclasses(); + rv.insert(gmap.begin(), gmap.end()); + + return rv; + } + }; + + template + using HardwarePredicateSMT = SubclassMappingTraits, IO>; + + template + const typename HardwarePredicateSMT::SubclassMap HardwarePredicateSMT::subclasses + = HardwarePredicateSMT::GetSubclasses(); + + template + struct MappingTraits, IO> + : public AutoMappingTraits, IO> + { + }; + + template + struct SubclassMappingTraits, IO> + : public DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + Predicates::Predicate, + IO> + { + using Self = SubclassMappingTraits, IO>; + using Base = DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + Predicates::Predicate, + IO>; + using SubclassMap = typename Base::SubclassMap; + const static SubclassMap subclasses; + + using Generic = PredicateMappingTraits; + + static SubclassMap GetSubclasses() + { + SubclassMap rv({Base::template Pair(), + Base::template Pair(), + Base::template Pair()}); + + auto gmap = Generic::GetSubclasses(); + rv.insert(gmap.begin(), gmap.end()); + + return rv; + } + }; + + template + using AMDGPUPredicateSMT = SubclassMappingTraits, IO>; + + template + const typename AMDGPUPredicateSMT::SubclassMap AMDGPUPredicateSMT::subclasses + = AMDGPUPredicateSMT::GetSubclasses(); + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct EnumTraits + { + using iot = IOTraits; + + static void enumeration(IO& io, AMDGPU::Processor& value) + { + iot::enumCase(io, value, "gfx803", AMDGPU::Processor::gfx803); + iot::enumCase(io, value, "gfx900", AMDGPU::Processor::gfx900); + iot::enumCase(io, value, "gfx906", AMDGPU::Processor::gfx906); + iot::enumCase(io, value, "gfx908", AMDGPU::Processor::gfx908); + iot::enumCase(io, value, "gfx90a", AMDGPU::Processor::gfx90a); + iot::enumCase(io, value, "gfx1010", AMDGPU::Processor::gfx1010); + iot::enumCase(io, value, "gfx1011", AMDGPU::Processor::gfx1011); + iot::enumCase(io, value, "gfx1012", AMDGPU::Processor::gfx1012); + iot::enumCase(io, value, "gfx1030", AMDGPU::Processor::gfx1030); + iot::enumCase(io, value, "gfx1100", AMDGPU::Processor::gfx1100); + iot::enumCase(io, value, "gfx1101", AMDGPU::Processor::gfx1101); + iot::enumCase(io, value, "gfx1102", AMDGPU::Processor::gfx1102); + } + }; + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Properties.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Properties.hpp new file mode 100644 index 0000000000..c4f9d61160 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/Properties.hpp @@ -0,0 +1,162 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include +#include + +#include + +namespace Tensile +{ + namespace Serialization + { + + template + struct MappingTraits>, IO> + : public BaseClassMappingTraits, IO, true> + { + }; + + template + struct SubclassMappingTraits, IO> + : public DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + Property, + IO> + { + using Self = SubclassMappingTraits, IO>; + using Base = DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + Property, + IO>; + using SubclassMap = typename Base::SubclassMap; + const static SubclassMap subclasses; + + static SubclassMap GetSubclasses() + { + return SubclassMap({Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair(), + Base::template Pair()}); + } + }; + + template + struct SubclassMappingTraits, IO> + : public DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + Property, + IO> + { + using Self = SubclassMappingTraits, IO>; + using Base = DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + Property, + IO>; + using SubclassMap = typename Base::SubclassMap; + const static SubclassMap subclasses; + + static SubclassMap GetSubclasses() + { + return SubclassMap({Base::template Pair()}); + } + }; + + template + const typename SubclassMappingTraits, IO>::SubclassMap + SubclassMappingTraits, IO>::subclasses + = SubclassMappingTraits, IO>::GetSubclasses(); + + template + const typename SubclassMappingTraits, + IO>::SubclassMap + SubclassMappingTraits, IO>::subclasses + = SubclassMappingTraits, IO>::GetSubclasses(); + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/SolutionLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/SolutionLibrary.hpp new file mode 100644 index 0000000000..e2e3ad9956 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/SolutionLibrary.hpp @@ -0,0 +1,191 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace Tensile +{ + namespace Serialization + { + template + struct MappingTraits>, IO> + : public BaseClassMappingTraits, IO, false> + { + }; + + template + struct SubclassMappingTraits, IO> + : public DefaultSubclassMappingTraits< + SubclassMappingTraits, IO>, + SolutionLibrary, + IO> + { + using Self = SubclassMappingTraits, IO>; + using Base + = DefaultSubclassMappingTraits, IO>; + using SubclassMap = typename Base::SubclassMap; + const static SubclassMap subclasses; + + static typename Base::SubclassMap GetSubclasses() + { + return typename Base::SubclassMap( + {Base::template Pair>(), + Base::template Pair>(), + Base::template Pair>(), + Base::template Pair>(), + Base::template Pair>(), + Base::template Pair>()}); + } + }; + + template + using dsmt = SubclassMappingTraits, IO>; + + template + const typename dsmt::SubclassMap + dsmt::subclasses + = dsmt::GetSubclasses(); + + template + struct MappingTraits, IO> + { + using Library = SingleSolutionLibrary; + using iot = IOTraits; + + static void mapping(IO& io, Library& lib) + { + SolutionMap* ctx + = static_cast*>(iot::getContext(io)); + if(ctx == nullptr) + { + iot::setError(io, + "SingleSolutionLibrary requires that context be set to " + "a SolutionMap."); + } + + int index; + + if(iot::outputting(io)) + index = lib.solution->index; + + iot::mapRequired(io, "index", index); + + if(!iot::outputting(io)) + { + auto iter = ctx->find(index); + if(iter == ctx->end()) + { + std::ostringstream msg; + msg << "Invalid solution index: " << index; + iot::setError(io, msg.str()); + } + else + { + lib.solution = iter->second; + } + } + } + + const static bool flow = true; + }; + + template + struct MappingTraits, IO, EmptyContext> + { + using Library = MasterSolutionLibrary; + using iot = IOTraits; + + static void mapping(IO& io, Library& lib) + { + iot::mapOptional(io, "version", lib.version); + + std::vector> solutions; + + if(iot::outputting(io)) + { + solutions.reserve(lib.solutions.size()); + for(auto const& pair : lib.solutions) + solutions.push_back(pair.second); + } + + iot::mapRequired(io, "solutions", solutions); + + if(!iot::outputting(io)) + { + for(auto const& s : solutions) + lib.solutions[s->index] = s; + } + + iot::setContext(io, &lib.solutions); + + std::shared_ptr> innerLibrary; + + if(iot::outputting(io)) + { + auto cache = std::dynamic_pointer_cast>( + lib.library); + if(cache) + { + innerLibrary = cache->library(); + } + else + { + innerLibrary = lib.library; + } + } + + iot::mapRequired(io, "library", innerLibrary); + + if(!iot::outputting(io)) + { + auto cache + = std::make_shared>(innerLibrary); + + lib.library = cache; + } + } + + const static bool flow = false; + }; + } // namespace Serialization +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/SingleSolutionLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/SingleSolutionLibrary.hpp new file mode 100644 index 0000000000..7310f9b336 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/SingleSolutionLibrary.hpp @@ -0,0 +1,123 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +namespace Tensile +{ + /** + * \ingroup SolutionLibrary + * + * Leaf of the tree. Represents a single `Solution` object. Can eliminate + * itself from consideration based on restrictions of that particular + * `Solution`. + */ + template + struct SingleSolutionLibrary : public SolutionLibrary + { + static std::string Type() + { + return "Single"; + } + std::string type() const override + { + return Type(); + } + std::string description() const override + { + std::string rv = type(); + if(solution != nullptr) + { + rv += ": "; + rv += solution->name(); + } + else + { + rv += " (nullptr)"; + } + + return rv; + } + + std::shared_ptr solution; + + SingleSolutionLibrary() = default; + SingleSolutionLibrary(std::shared_ptr s) + : solution(s) + { + } + + virtual std::shared_ptr findBestSolution(MyProblem const& problem, + Hardware const& hardware, + double* fitness + = nullptr) const override + { + bool debug = Debug::Instance().printPredicateEvaluation(); + + if(solution) + { + if(debug) + { + solution->hardwarePredicate->debugEval(hardware, std::cout); + solution->problemPredicate->debugEval(problem, std::cout); + } + + if((*solution->hardwarePredicate)(hardware) + && (*solution->problemPredicate)(problem)) + return solution; + } + else if(debug) + { + std::cout << " (empty library)"; + } + + return std::shared_ptr(); + } + + virtual SolutionSet findAllSolutions(MyProblem const& problem, + Hardware const& hardware) const override + { + + auto result = this->findBestSolution(problem, hardware); + + bool debug = Debug::Instance().printPredicateEvaluation(); + if(debug) + { + if(result) + std::cout << " (match)"; + else + std::cout << " (no match)"; + } + + if(result) + return SolutionSet({result}); + + return SolutionSet(); + } + }; +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Singleton.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Singleton.hpp new file mode 100644 index 0000000000..5dc4b195bc --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Singleton.hpp @@ -0,0 +1,46 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +namespace Tensile +{ + template + class LazySingleton + { + public: + static Class& Instance() + { + static Class instance; + + return instance; + } + + private: + }; +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/SolutionLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/SolutionLibrary.hpp new file mode 100644 index 0000000000..7b8b5dd3c1 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/SolutionLibrary.hpp @@ -0,0 +1,100 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include + +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \defgroup SolutionLibrary Solution Library Classes + * + * \copydoc Tensile::SolutionLibrary + */ + + template + using SolutionSet = std::set>; + + /** + * \ingroup SolutionLibrary + * + * @brief Abstract base class for Library objects that can provide a + * mapping from `Problem` and `Hardware` objects to Solution objects. + * + * A complete SolutionLibrary is a tree of objects which each handles a + * single aspect of selecting a solution for a given problem. Each node in + * the tree will handle an aspect such as: + * - Compatibility with a particular model of GPU + * - Selecting kernels that solve a particular type of problem (transpose, + * data type, etc.) + * - Selecting the fastest kernel based on benchmark results or other logic + * - Ensuring that a problem is compatible with any assumptions made by a + * particular kernel (e.g. size or stride requirements) + * + * A particular complete library might look like: + * - Master library which manages serialization + * - GPU selection + * - Problem type selection + * - Predicated logic for specific sizes + * - Matching library based on benchmarks + * - Individual kernels + * + */ + template + struct TENSILE_API SolutionLibrary + { + virtual ~SolutionLibrary() = default; + + /** + * Returns the single `Solution` object that best solves this + * particular `problem` on this particular piece of `hardware`. + * + * May return `nullptr` if no such object exists. + */ + virtual std::shared_ptr findBestSolution(MyProblem const& problem, + Hardware const& hardware, + double* fitness = nullptr) const = 0; + + /** + * Returns all `Solution` objects that are capable of correctly solving this + * `problem` on this `hardware`. + * + * May return an empty set if no such object exists. + */ + virtual SolutionSet findAllSolutions(MyProblem const& problem, + Hardware const& hardware) const = 0; + + virtual std::string type() const = 0; + virtual std::string description() const = 0; + }; + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/SolutionLibrary_fwd.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/SolutionLibrary_fwd.hpp new file mode 100644 index 0000000000..8f271a8377 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/SolutionLibrary_fwd.hpp @@ -0,0 +1,33 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +namespace Tensile +{ + template + struct SolutionLibrary; +} diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/SolutionMapLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/SolutionMapLibrary.hpp new file mode 100644 index 0000000000..f42f206cea --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/SolutionMapLibrary.hpp @@ -0,0 +1,37 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +namespace Tensile +{ + template + class SolutionMapLibrary : public SolutionLibrary + { + }; +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Tensile.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Tensile.hpp new file mode 100644 index 0000000000..f1bcfcea76 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Tensile.hpp @@ -0,0 +1,191 @@ +/******************************************************************************* + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +/** + * \mainpage + * + * Tensile is a tool for creating a library of tensor contractions, including + * matrix multiplications in a benchmark-driven manner. + * + * The host library contains classes and functions for selecting one or more + * kernels to launch to solve a particular problem, and then launching those + * kernels. Kernels are selected based on the performance on a particular GPU + * (from the benchmark runs) as well as the limitations and applicability of + * the kernels to particular problem sizes. + * + * Design goals for the host library include: + * - A library that requires no generated code on the host + * - Kernel selection configured via a file + * - No recompilation needed to enable a different set of kernels. + * - Kernel selection & determination of kernel launches decoupled from host + * runtime language + * - Eliminate as much of the Hip and OpenCL-specific code as possible + * - Support future work including implicit GEMMs + * - Use modern C++ + * - Eliminate the need for client code to call intricately named functions + * such as `tensile_Cijk_Ailk_Bljk_SB()`. + */ + +/** + * \addtogroup Tensile + * @{ + */ + +/** + * @brief Primary namespace for Tensile host code. + */ +namespace Tensile +{ + /** + * \ingroup Tensile + * \defgroup Problem Problem Definition + * + * @brief Classes for defining problems + */ + + /** + * \ingroup Problem + * Base Problem class. A Problem object generically describes a problem to + * be solved including the type of problem, all sizes and strides, but not + * including actual pointers to data. + */ + class TENSILE_API Problem + { + public: + virtual ~Problem(); + virtual std::string description() const = 0; + }; + + /** + * \ingroup Problem + * Base class for problem inputs. This stores the actual pointers to the + * data. + */ + class TENSILE_API ProblemInputs + { + public: + virtual ~ProblemInputs(); + }; + + /** + * \ingroup Tensile + * \defgroup Launching Kernel Launching + */ + + /** + * \ingroup Launching + * Describes a single kernel invocation including kernel name, launch + * bounds, and arguments. + */ + struct TENSILE_API KernelInvocation + { + public: + std::string kernelName; + + dim3 workGroupSize; + dim3 numWorkGroups; + dim3 numWorkItems; + size_t sharedMemBytes = 0; + + KernelArguments args; + }; + + /** + * \ingroup Tensile + * \defgroup Hardware Hardware Description Classes + */ + + /** + * \ingroup Hardware + * Abstract base class for describing hardware capabilities and properties. + */ + class TENSILE_API Hardware + { + public: + Hardware(); + virtual ~Hardware(); + + virtual size_t id() const = 0; + virtual std::string description() const = 0; + }; + + /** + * \ingroup Solution + * Generally encapsulates a single kernel or set of kernels that can be + * used to solve a particular problem. + */ + class TENSILE_API Solution + { + public: + virtual ~Solution(); + + virtual std::string name() const = 0; + virtual std::string description() const = 0; + }; + + /** + * \ingroup Launching + * Base class for objects capable of launching kernels based on + * KernelArguments objects. + */ + class TENSILE_API SolutionAdapter + { + public: + virtual ~SolutionAdapter(); + + virtual std::string name() const = 0; + }; + +#ifdef TENSILE_DEFAULT_SERIALIZATION + /** + * Interface for deserializing a library file. + */ + template + TENSILE_API std::shared_ptr> + LoadLibraryFile(std::string const& filename); + + template + std::shared_ptr> + LoadLibraryData(std::vector const& data); +#endif +} // namespace Tensile + +/** @} */ diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Tensile_fwd.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Tensile_fwd.hpp new file mode 100644 index 0000000000..e23347ecad --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Tensile_fwd.hpp @@ -0,0 +1,34 @@ +/** + * Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +namespace Tensile +{ + class Problem; + class ProblemInputs; + class KernelArguments; + struct KernelInvocation; + class Hardware; + class Solution; + class SolutionAdapter; +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/TensorDescriptor.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/TensorDescriptor.hpp new file mode 100644 index 0000000000..b0278dbec2 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/TensorDescriptor.hpp @@ -0,0 +1,441 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace Tensile +{ + template + inline size_t CoordCount(SizeIter sizeBegin, SizeIter sizeEnd) + { + size_t rv = 1; + + while(sizeBegin != sizeEnd) + { + rv *= *sizeBegin; + sizeBegin++; + } + + return rv; + } + + template + inline void CoordNumbered( + size_t num, CoordIter coordBegin, CoordIter coordEnd, SizeIter sizeBegin, SizeIter sizeEnd) + { + auto coord = coordBegin; + auto size = sizeBegin; + + while(coord != coordEnd && size != sizeEnd) + { + *coord = num % *size; + num /= *size; + + coord++; + size++; + } + + if(coord != coordEnd || size != sizeEnd) + throw std::runtime_error("Inconsistent size of coordinates."); + } + + template + inline bool IncrementCoord(CoordIter coordBegin, + CoordIter coordEnd, + SizeIter sizeBegin, + SizeIter sizeEnd) + { + auto coord = coordBegin; + auto size = sizeBegin; + + while(coord != coordEnd) + { + (*coord)++; + if(*coord < *size) + return true; + + *coord = 0; + + coord++; + size++; + } + + return false; + } + + /** + * Describes a tensor including dimensions, memory layout, and data type. + * Decoupled from any particular pointer value or memory location. + * + * Provides functions for indexing and otherwise iterating through a tensor. + */ + class TENSILE_API TensorDescriptor + { + public: + static const size_t UseDefaultStride; + + TensorDescriptor() + { + this->calculate(); + } + + template + TensorDescriptor(DataType t, + IterA sizesBegin, + IterA sizesEnd, + IterB stridesBegin, + IterB stridesEnd, + size_t offset = 0) + : m_sizes(sizesBegin, sizesEnd) + , m_strides(stridesBegin, stridesEnd) + , m_dataType(t) + , m_offset(offset) + { + this->calculate(); + } + + template + TensorDescriptor(DataType t, Iter sizesBegin, Iter sizesEnd, size_t offset = 0) + : m_sizes(sizesBegin, sizesEnd) + , m_dataType(t) + , m_offset(offset) + { + this->calculate(); + } + + TensorDescriptor(DataType t, std::initializer_list sizes, size_t offset = 0) + : m_sizes(sizes) + , m_dataType(t) + , m_offset(offset) + + { + this->calculate(); + } + + TensorDescriptor(DataType t, + std::initializer_list sizes, + std::initializer_list strides, + size_t offset = 0) + : m_sizes(sizes) + , m_strides(strides) + , m_dataType(t) + , m_offset(offset) + { + this->calculate(); + } + + inline void calculate() + { + if(m_sizes.empty()) + { + m_strides = m_sizes; + m_totalLogicalElements = 0; + m_totalAllocatedElements = 0; + return; + } + + m_strides.resize(m_sizes.size(), UseDefaultStride); + if(m_strides[0] == UseDefaultStride) + { + m_strides[0] = 1; + } + m_totalLogicalElements = m_sizes[0]; + + for(int i = 1; i < m_sizes.size(); i++) + { + m_totalLogicalElements *= m_sizes[i]; + + if(m_strides[i] == UseDefaultStride) + { + m_strides[i] = m_strides[i - 1] * m_sizes[i - 1]; + } + } + + m_totalAllocatedElements = 1; + for(int i = 0; i < m_sizes.size(); i++) + m_totalAllocatedElements += m_strides[i] * (m_sizes[i] - 1); + + m_totalAllocatedElements += m_offset; + + if(Debug::Instance().printTensorInfo()) + { + std::cout << "TensorDescriptor:calculate " << *this + << "totalLogicalElements=" << m_totalLogicalElements + << " totalAllocatedElem=" << m_totalAllocatedElements << "\n"; + } + } + + const std::vector& sizes() const + { + return m_sizes; + } + const std::vector& strides() const + { + return m_strides; + } + + size_t offset() const + { + return m_offset; + } + + bool empty() const + { + return m_sizes.empty(); + } + + void appendDim(size_t logicalCount); + void appendDim(size_t logicalCount, size_t allocatedCount); + + /** + * Returns the number of elements of padding in the given dimension (0 if + * unpadded). May be negative if stride is less than size + */ + int64_t dimensionPadding(size_t dim) const; + + /** + * Collapses dimensions in the interval [begin, end). + * + * preconditions: + * - end >= begin + * - begin < dimensions() + * - end <= dimensions() + * - dimensions in the interval [begin, end-1) are not padded. + * + * postconditions: + * - dimensions() is diminished by end-begin + * - total elements (allocated and logical) remain the same + * - dimension 'begin' is the product of all the dimensions in the interval + * [begin, end). + */ + void collapseDims(size_t begin, size_t end); + + size_t dimensions() const + { + return m_sizes.size(); + } + size_t totalLogicalElements() const + { + return m_totalLogicalElements; + } + size_t totalAllocatedElements() const + { + return m_totalAllocatedElements; + } + size_t totalAllocatedBytes() const + { + return totalAllocatedElements() * elementBytes(); + } + + size_t elementBytes() const + { + return DataTypeInfo::Get(m_dataType).elementSize; + } + + DataType dataType() const + { + return m_dataType; + } + + template + inline size_t index(Container const& indices) const + { + if(indices.size() != dimensions()) + throw std::runtime_error("Incorrect number of indices."); + + for(int i = 0; i < indices.size(); i++) + if(indices[i] >= m_sizes[i]) + throw std::runtime_error("Index out of bounds."); + + return std::inner_product(indices.begin(), indices.end(), m_strides.begin(), m_offset); + } + + template + inline size_t index(std::initializer_list indices) const + { + if(indices.size() != dimensions()) + throw std::runtime_error("Incorrect number of indices."); + + for(auto i = std::make_pair(indices.begin(), m_sizes.begin()); i.first != indices.end(); + i.first++, i.second++) + if(*i.first >= *i.second) + throw std::runtime_error("Index out of bounds."); + + return std::inner_product(indices.begin(), indices.end(), m_strides.begin(), m_offset); + } + + template ::type>::value>::type> + inline size_t index(Ts... is) const + { + return this->index({is...}); + } + + inline bool incrementCoord(std::vector& coord, size_t firstDimension = 0) const + { + if(coord.size() != dimensions()) + throw std::runtime_error(concatenate( + "Invalid coordinate size ", coord.size(), " for ", dimensions(), "-tensor")); + + if(firstDimension >= dimensions()) + return false; + + return IncrementCoord( + coord.begin() + firstDimension, coord.end(), m_sizes.begin(), m_sizes.end()); + } + + bool operator==(const TensorDescriptor& rhs) const; + bool operator!=(const TensorDescriptor& rhs) const; + + std::string ToString() const; + + friend std::ostream& operator<<(std::ostream& stream, const TensorDescriptor& t); + + private: + std::vector m_sizes; + std::vector m_strides; + size_t m_offset = 0; + + size_t m_totalLogicalElements = 0; + size_t m_totalAllocatedElements = 0; + + DataType m_dataType = DataType::Float; + }; + + std::ostream& operator<<(std::ostream& stream, const TensorDescriptor& t); + + template + void WriteTensor1D(std::ostream& stream, + T* data, + TensorDescriptor const& desc, + bool decorated = true) + { + if(desc.dimensions() != 1) + throw std::runtime_error( + "WriteTensor1D is only compatible with 1-dimensional tensors."); + + if(decorated) + stream << "["; + + if(desc.sizes()[0] > 0) + stream << data[0]; + + for(size_t i = 1; i < desc.sizes()[0]; i++) + stream << " " << data[i]; + + if(decorated) + stream << "]" << std::endl; + } + + /** + * @brief Writes a tensor to an output stream. + * + * \param stream The stream to write to + * \param data Pointer to the tensor data + * \param desc Tensor descriptor + * \param ptrValue Pointer value to print to describe the location of the data. + * \param decorated Print brackets [] to indicate start/end of tensor dims + */ + template + void WriteTensor(std::ostream& stream, + T const* data, + TensorDescriptor const& desc, + T const* ptrValue = nullptr, + bool decorated = true) + { + stream << "Tensor("; + streamJoin(stream, desc.sizes(), ", "); + stream << ", data_ptr: " << data << ")" << std::endl; + + if(desc.dimensions() == 0) + return; + + if(desc.dimensions() == 1) + { + WriteTensor1D(stream, data, desc, decorated); + return; + } + + auto const& sizes = desc.sizes(); + std::vector coord(desc.dimensions(), 0); + const auto stride0 = desc.strides()[0]; + + auto upperDimCount = CoordCount(sizes.begin() + 2, sizes.end()); + + for(size_t idx = 0; idx < upperDimCount; idx++) + { + CoordNumbered(idx, coord.begin() + 2, coord.end(), sizes.begin() + 2, sizes.end()); + + coord[0] = 0; + coord[1] = 0; + + if(decorated) + { + stream << "("; + streamJoin(stream, coord, ", "); + stream << ")" << std::endl << "[" << std::endl; + } + + for(coord[1] = 0; coord[1] < sizes[1]; coord[1]++) + { + coord[0] = 0; + + auto const* localPtr = data + desc.index(coord); + + if(sizes[0] > 0) + stream << localPtr[0]; + + for(coord[0] = 1; coord[0] < sizes[0]; coord[0]++) + { + stream << " " << localPtr[coord[0] * stride0]; + } + + stream << std::endl; + } + + if(decorated) + { + stream << std::endl << "]" << std::endl; + } + } + } + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/TensorDescriptor_Detail.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/TensorDescriptor_Detail.hpp new file mode 100644 index 0000000000..ef01740eb2 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/TensorDescriptor_Detail.hpp @@ -0,0 +1,72 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +namespace Tensile +{ + template <> + struct Comparison + { + enum + { + implemented = true + }; + + static int compare(TensorDescriptor const& lhs, TensorDescriptor const& rhs) + { + return LexicographicCompare(lhs.dataType(), + rhs.dataType(), + lhs.sizes(), + rhs.sizes(), + lhs.strides(), + rhs.strides(), + lhs.offset(), + rhs.offset()); + } + }; + +} // namespace Tensile + +namespace std +{ + + template <> + struct hash + { + inline size_t operator()(Tensile::TensorDescriptor const& tensor) const + { + return Tensile::combine_hashes( + std::hash()((size_t)tensor.dataType()), + Tensile::hash_combine_iter(tensor.sizes().begin(), tensor.sizes().end()), + Tensile::hash_combine_iter(tensor.strides().begin(), tensor.strides().end()), + std::hash()((size_t)tensor.offset())); + } + }; +} // namespace std diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/TensorDescriptor_fwd.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/TensorDescriptor_fwd.hpp new file mode 100644 index 0000000000..9a73685e48 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/TensorDescriptor_fwd.hpp @@ -0,0 +1,35 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2017-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +namespace Tensile +{ + enum class DataType : int; + + class TensorDescriptor; + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/TensorOps.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/TensorOps.hpp new file mode 100644 index 0000000000..89f43966fd --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/TensorOps.hpp @@ -0,0 +1,115 @@ +/** + * Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include + +namespace Tensile +{ + /** + * \addtogroup Problem + * @{ + */ + + using TensorOps = std::vector; + + /** + * Represents a tensor operation that can be performed inline to a kernel. + * For now can represent a complex conjugate but this could be where we + * represent inline activation layers later. + */ + class TENSILE_API TensorOp + { + public: + enum class Type : int + { + None, + ComplexConjugate, + Count + }; + + Type type = Type::None; + + TensorOp(); + TensorOp(Type type); + + static TensorOp ComplexConjugate() + { + return TensorOp(Type::ComplexConjugate); + } + + std::string name() const; + std::string suffix() const; + + // static TensorOp ReLU(); + // static TensorOp LeakyReLU(float alpha); + // ... + + bool operator==(TensorOp const& rhs) const + { + return this->type == rhs.type; + } + bool operator<(TensorOp const& rhs) const + { + return this->type < rhs.type; + } + + bool operator>(TensorOp const& rhs) const + { + return rhs < *this; + } + bool operator!=(TensorOp const& rhs) const + { + return !(*this == rhs); + } + bool operator<=(TensorOp const& rhs) const + { + return !(*this > rhs); + } + bool operator>=(TensorOp const& rhs) const + { + return !(*this < rhs); + } + + static Type GetType(std::string const& name); + + private: + static void InitTypeNames(); + static std::map typeNames; + }; + + std::string ToString(TensorOp::Type t); + std::string Suffix(TensorOp::Type t); + + std::ostream& operator<<(std::ostream& stream, TensorOp const& t); + std::istream& operator>>(std::istream& stream, TensorOp& t); + + std::ostream& operator<<(std::ostream& stream, TensorOp::Type const& t); + std::istream& operator>>(std::istream& stream, TensorOp::Type& t); + + /** + * @} + */ +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/TensorOps_fwd.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/TensorOps_fwd.hpp new file mode 100644 index 0000000000..4b052ccd00 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/TensorOps_fwd.hpp @@ -0,0 +1,28 @@ +/** + * Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +namespace Tensile +{ + class TensorOp; +} diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Utils.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Utils.hpp new file mode 100644 index 0000000000..a47a5a8d7e --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Utils.hpp @@ -0,0 +1,217 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace Tensile +{ + + /** + * \ingroup Tensile + * \addtogroup Utilities + * @{ + */ + + template + T CeilDivide(T num, T den) + { + return (num + (den - 1)) / den; + } + + template + T RoundUpToMultiple(T val, T den) + { + return CeilDivide(val, den) * den; + } + + template ::value>> + T IsPrime(T val) + { + if(val < 2) + return false; + if(val < 4) + return true; + + T end = sqrt(val); + + for(T i = 2; i <= end; i++) + if(val % i == 0) + return false; + return true; + } + + template ::value>> + T NextPrime(T val) + { + if(val < 2) + return 2; + while(!IsPrime(val)) + val++; + return val; + } + + template + void streamJoin(std::ostream& stream, Container const& c, Joiner const& j) + { + bool first = true; + for(auto const& item : c) + { + if(!first) + stream << j; + stream << item; + first = false; + } + } + + template + inline std::ostream& operator<<(std::ostream& stream, std::array const& array) + { + streamJoin(stream, array, ", "); + return stream; + } + + template + inline std::ostream& stream_write(std::ostream& stream, T&& val) + { + return stream << std::forward(val); + } + + template + inline std::ostream& stream_write(std::ostream& stream, T&& val, Ts&&... vals) + { + return stream_write(stream << std::forward(val), std::forward(vals)...); + } + + template + inline std::string concatenate(Ts&&... vals) + { + std::ostringstream msg; + stream_write(msg, std::forward(vals)...); + + return msg.str(); + } + + template + inline std::string concatenate_if(Ts&&... vals) + { + if(!T_Enable) + return ""; + + return concatenate(std::forward(vals)...); + } + + class StreamRead + { + public: + StreamRead(std::string const& value, bool except = true); + ~StreamRead(); + + bool read(std::istream& stream); + + private: + std::string const& m_value; + bool m_except; + bool m_success = false; + }; + + // inline std::istream & operator>>(std::istream & stream, StreamRead & value); + inline std::istream& operator>>(std::istream& stream, StreamRead& value) + { + value.read(stream); + return stream; + } + + struct BitFieldGenerator + { + constexpr static uint32_t maxBitFieldWidth = 32; + + // Get the minimum width of the given maxVal in bits. + constexpr static uint32_t ElementWidth(uint32_t maxVal) + { + return maxVal ? 1 + ElementWidth(maxVal >> 1) : 0; + } + + // Get the bit mask for the element size in bits. + constexpr static uint32_t BitMask(uint32_t elementWidth) + { + if(elementWidth == 1) + return (uint32_t)0x1; + return (BitMask(elementWidth - 1) << 1) | (uint32_t)0x1; + } + + // Generate a 32 bit field containing val0 in the LSB, occupying the first + // elementWidth bits. + constexpr static uint32_t GenerateBitField(uint32_t elementWidth, uint32_t val0) + { + int mask = BitMask(elementWidth); + return mask & val0; + } + + // Generate a 32 bit field containing val0... valN in order starting from LSB, each + // value occupying elementWidth bits of the field. + template + constexpr static uint32_t + GenerateBitField(uint32_t elementWidth, uint32_t val0, ArgsT... valN) + { + int mask = BitMask(elementWidth); + return (GenerateBitField(elementWidth, valN...) << elementWidth) | (mask & val0); + } + }; + + /** + * @} + */ +} // namespace Tensile + +/** + * \addtogroup Utilities + * @{ + */ +#define TENSILE_STR_(x) #x +#define TENSILE_STR(x) TENSILE_STR_(x) +#define TENSILE_LINENO TENSILE_STR(__LINE__) +#define TENSILE_LINEINFO __FILE__ ":" TENSILE_LINENO + +#define TENSILE_ASSERT_EXC(exp) \ + do \ + { \ + if(!(exp)) \ + { \ + throw std::runtime_error("Error in " TENSILE_LINEINFO ": " #exp); \ + } \ + } while(false) + +/** + * @} + */ diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/geom.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/geom.hpp new file mode 100644 index 0000000000..e2d46fccae --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/geom.hpp @@ -0,0 +1,142 @@ +/** + * Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include + +#include + +namespace Tensile +{ + /** + * \ingroup Tensile + * \addtogroup Utilities + * @{ + */ + template + struct TENSILE_API vector2 + { + vector2() = default; + vector2(T _x, T _y) + : x(_x) + , y(_y) + { + } + enum + { + count = 2 + }; + + T x; + T y; + }; + + template + TENSILE_API inline bool operator==(vector2 const& l, vector2 const& r) + { + return (l.x == r.x) && (l.y == r.y); + } + + template + TENSILE_API inline std::ostream& operator<<(std::ostream& stream, vector2 const& v) + { + return stream << "(" << v.x << ", " << v.y << ")"; + } + + template + struct TENSILE_API vector3 + { + vector3() = default; + vector3(T _x, T _y, T _z) + : x(_x) + , y(_y) + , z(_z) + { + } + + enum + { + count = 3 + }; + + T x; + T y; + T z; + }; + + template + inline TENSILE_API bool operator==(vector3 const& l, vector3 const& r) + { + return (l.x == r.x) && (l.y == r.y) && (l.z == r.z); + } + + template + inline TENSILE_API std::ostream& operator<<(std::ostream& stream, vector3 const& v) + { + return stream << "(" << v.x << ", " << v.y << ", " << v.z << ")"; + } + + template + struct TENSILE_API vector4 + { + vector4() = default; + vector4(T _x, T _y, T _z, T _w) + : x(_x) + , y(_y) + , z(_z) + , w(_w) + { + } + + enum + { + count = 4 + }; + + T x; + T y; + T z; + T w; + }; + + template + inline TENSILE_API bool operator==(vector4 const& l, vector4 const& r) + { + return (l.x == r.x) && (l.y == r.y) && (l.z == r.z) && (l.w == r.w); + } + + template + inline TENSILE_API std::ostream& operator<<(std::ostream& stream, vector4 const& v) + { + return stream << "(" << v.x << ", " << v.y << ", " << v.z << ", " << v.w << ")"; + } + + using dim3 = vector3; + using int3 = vector3; + + /** + * @} + */ +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/hip/HipHardware.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/hip/HipHardware.hpp new file mode 100644 index 0000000000..bb82519703 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/hip/HipHardware.hpp @@ -0,0 +1,50 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include + +namespace Tensile +{ + namespace hip + { + struct HipAMDGPU : public AMDGPU + { + HipAMDGPU() = default; + HipAMDGPU(hipDeviceProp_t const& prop); + + hipDeviceProp_t properties; + }; + + std::shared_ptr GetCurrentDevice(); + std::shared_ptr GetDevice(int deviceId); + std::shared_ptr GetDevice(hipDeviceProp_t const& prop); + } // namespace hip +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/hip/HipSolutionAdapter.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/hip/HipSolutionAdapter.hpp new file mode 100644 index 0000000000..74af953c56 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/hip/HipSolutionAdapter.hpp @@ -0,0 +1,99 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include + +namespace Tensile +{ + namespace hip + { + class SolutionAdapter : public Tensile::SolutionAdapter + { + public: + SolutionAdapter(); + SolutionAdapter(bool debug); + SolutionAdapter(bool debug, std::string const& name); + ~SolutionAdapter(); + + virtual std::string name() const + { + return m_name; + } + + hipError_t loadCodeObjectFile(std::string const& path); + + hipError_t loadCodeObject(const void* image); + + hipError_t loadCodeObjectBytes(std::vector const& bytes); + + void loadEmbeddedCodeObjects(); + void loadEmbeddedCodeObjects(std::string const& key); + + hipError_t launchKernel(KernelInvocation const& kernel); + hipError_t launchKernel(KernelInvocation const& kernel, + hipStream_t stream, + hipEvent_t startEvent, + hipEvent_t stopEvent); + + hipError_t launchKernels(std::vector const& kernels); + + hipError_t launchKernels(std::vector const& kernels, + hipStream_t stream, + hipEvent_t startEvent, + hipEvent_t stopEvent); + + hipError_t launchKernels(std::vector const& kernels, + hipStream_t stream, + std::vector const& startEvents, + std::vector const& stopEvents); + + hipError_t initKernel(std::string const& name); + + private: + hipError_t getKernel(hipFunction_t& rv, std::string const& name); + + std::mutex m_access; + + std::vector m_modules; + std::unordered_map m_kernels; + bool m_debug = false; + bool m_debugSkipLaunch = false; + std::string m_name = "HipSolutionAdapter"; + + std::vector m_loadedModuleNames; + + friend std::ostream& operator<<(std::ostream& stream, SolutionAdapter const& adapter); + }; + + std::ostream& operator<<(std::ostream& stream, SolutionAdapter const& adapter); + std::ostream& operator<<(std::ostream& stream, std::shared_ptr const& ptr); + } // namespace hip +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/hip/HipUtils.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/hip/HipUtils.hpp new file mode 100644 index 0000000000..8a5492efa7 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/hip/HipUtils.hpp @@ -0,0 +1,135 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include + +#include +#include + +#define HIP_CHECK_EXC(expr) \ + do \ + { \ + hipError_t e = (expr); \ + if(e) \ + { \ + const char* errName = hipGetErrorName(e); \ + const char* errMsg = hipGetErrorString(e); \ + std::ostringstream msg; \ + msg << "Error " << e << "(" << errName << ") " << __FILE__ << ":" << __LINE__ << ": " \ + << std::endl \ + << #expr << std::endl \ + << errMsg << std::endl; \ + throw std::runtime_error(msg.str()); \ + } \ + } while(0) + +#define HIP_CHECK_EXC_MESSAGE(expr, message) \ + do \ + { \ + hipError_t e = (expr); \ + if(e) \ + { \ + const char* errName = hipGetErrorName(e); \ + const char* errMsg = hipGetErrorString(e); \ + std::ostringstream msg; \ + msg << "Error " << e << "(" << errName << ") " << __FILE__ << ":" << __LINE__ << ": " \ + << std::endl \ + << #expr << std::endl \ + << errMsg << std::endl \ + << (message) << std::endl; \ + throw std::runtime_error(msg.str()); \ + } \ + } while(0) + +#define HIP_CHECK_RETURN(expr) \ + do \ + { \ + hipError_t e = (expr); \ + if(e) \ + return e; \ + } while(0) + +namespace Tensile +{ + namespace hip + { + template + void CopyTensor(T* dst, + T const* src, + TensorDescriptor const& desc, + hipMemcpyKind direction, + hipStream_t stream = 0) + { + if(desc.dimensions() == 0 || desc.totalLogicalElements() == 0) + return; + + auto const& sizes = desc.sizes(); + auto const& strides = desc.strides(); + std::vector coord(desc.dimensions(), 0); + + size_t contiguousDimensions = 0; + size_t expectedStride = 1; + + // Optimize the number of copy operations by coalescing all the + // dimensions that are contiguous in memory. + for(size_t i = 0; i < desc.dimensions(); i++) + { + if(strides[i] > expectedStride) + break; + + contiguousDimensions = i + 1; + + if(i < desc.dimensions() - 1) + expectedStride = strides[i] * sizes[i]; + } + + auto copyCount = CoordCount(sizes.begin() + contiguousDimensions, sizes.end()); + + size_t maxStride + = *std::max_element(strides.begin(), strides.begin() + contiguousDimensions); + size_t copyBytes = maxStride * sizes.at(contiguousDimensions - 1) * sizeof(T); + + for(size_t idx = 0; idx < copyCount; idx++) + { + CoordNumbered(idx, + coord.begin() + contiguousDimensions, + coord.end(), + sizes.begin() + contiguousDimensions, + sizes.end()); + + auto beginOffset = desc.index(coord); + + HIP_CHECK_EXC(hipMemcpyAsync( + dst + beginOffset, src + beginOffset, copyBytes, direction, stream)); + } + } + } // namespace hip +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/llvm/Loading.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/llvm/Loading.hpp new file mode 100644 index 0000000000..f169398d2f --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/llvm/Loading.hpp @@ -0,0 +1,40 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +namespace Tensile +{ + template + std::shared_ptr> + LLVMLoadLibraryFile(std::string const& filename); + + template + std::shared_ptr> + LLVMLoadLibraryData(std::vector const& data); +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/llvm/YAML.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/llvm/YAML.hpp new file mode 100644 index 0000000000..0d00f470f3 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/llvm/YAML.hpp @@ -0,0 +1,310 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include + +#include + +namespace llvm +{ + namespace yaml + { + namespace sn = Tensile::Serialization; + + template + struct Hide + { + T& _value; + + Hide(T& value) + : _value(value) + { + } + + T& operator*() + { + return _value; + } + }; + + template + struct has_SequenceTraits> + { + const static bool value = sn::has_SequenceTraits::value; + }; + + template + struct has_ScalarEnumerationTraits> + { + const static bool value = sn::has_EnumTraits::value; + }; + + template + struct has_MappingTraits, EmptyContext> + { + const static bool value = sn::has_EmptyMappingTraits::value; + }; + + template + struct has_CustomMappingTraits> + { + const static bool value = sn::has_CustomMappingTraits::value; + }; + + template + struct missingTraits + : public std::integral_constant< + bool, + !has_ScalarEnumerationTraits::value && !has_ScalarBitSetTraits::value + && !has_ScalarTraits::value && !has_BlockScalarTraits::value + && !has_MappingTraits::value && !has_SequenceTraits::value + && !has_CustomMappingTraits::value && !has_DocumentListTraits::value + && !sn::has_SerializationTraits::value> + { + }; + + template + typename std::enable_if::value, void>::type + yamlize(IO& io, T& Val, bool b, EmptyContext& ctx) + { + Hide hide(Val); + + yamlize(io, hide, b, ctx); + } + + template + typename std::enable_if::value, Input&>::type + operator>>(Input& input, T& Val) + { + Hide hide(Val); + + return input >> hide; + } + + template + typename std::enable_if::value, Output&>::type + operator<<(Output& output, T& Val) + { + Hide hide(Val); + + return output << hide; + } + } // namespace yaml +} // namespace llvm + +namespace Tensile +{ + namespace Serialization + { + template <> + struct IOTraits + { + using IO = llvm::yaml::IO; + + template + static void mapRequired(IO& io, const char* key, T& obj) + { + io.mapRequired(key, obj); + } + + template + static void mapRequired(IO& io, const char* key, T& obj, Context& ctx) + { + io.mapRequired(key, obj, ctx); + } + + template + static void mapOptional(IO& io, const char* key, T& obj) + { + io.mapOptional(key, obj); + } + + template + static void mapOptional(IO& io, const char* key, T& obj, Context& ctx) + { + io.mapOptional(key, obj, ctx); + } + + static bool outputting(IO& io) + { + return io.outputting(); + } + + static void setError(IO& io, std::string const& msg) + { + // throw std::runtime_error(msg); + return io.setError(msg); + } + + static void setContext(IO& io, void* ctx) + { + io.setContext(ctx); + } + + static void* getContext(IO& io) + { + return io.getContext(); + } + + template + static void enumCase(IO& io, T& member, const char* key, T value) + { + io.enumCase(member, key, value); + } + }; + + } // namespace Serialization +} // namespace Tensile + +namespace llvm +{ + namespace yaml + { + LLVM_YAML_STRONG_TYPEDEF(size_t, FooType); + + using mysize_t + = std::conditional::value, FooType, size_t>::type; + + template <> + struct ScalarTraits + { + static void output(const mysize_t& value, void* ctx, raw_ostream& stream) + { + uint64_t tmp = value; + ScalarTraits::output(tmp, ctx, stream); + } + + static StringRef input(StringRef str, void* ctx, mysize_t& value) + { + uint64_t tmp; + auto rv = ScalarTraits::input(str, ctx, tmp); + value = tmp; + return rv; + } + + static bool mustQuote(StringRef) + { + return false; + } + }; + + template + struct MappingTraits> + { + static void mapping(IO& io, Hide& value) + { + sn::MappingTraits::mapping(io, *value); + } + + static const bool flow = sn::MappingTraits::flow; + }; + + template + struct SequenceTraits> + { + using Impl = sn::SequenceTraits; + using Value = typename Impl::Value; + + static size_t size(IO& io, Hide& t) + { + return Impl::size(io, *t); + } + static Value& element(IO& io, Hide& t, size_t index) + { + return Impl::element(io, *t, index); + } + + static const bool flow = Impl::flow; + }; + + template + struct ScalarEnumerationTraits> + { + static void enumeration(IO& io, Hide& value) + { + sn::EnumTraits::enumeration(io, *value); + } + }; + + template + struct CustomMappingTraits> + { + using Impl = sn::CustomMappingTraits; + + static void inputOne(IO& io, StringRef key, Hide& value) + { + Impl::inputOne(io, key.str(), *value); + } + + static void output(IO& io, Hide& value) + { + Impl::output(io, *value); + } + }; + + template <> + struct MappingTraits> + { + using obj = Tensile::MasterContractionLibrary; + + static void mapping(IO& io, std::shared_ptr& o) + { + sn::PointerMappingTraits::mapping(io, o); + } + }; + + static_assert(sn::has_EmptyMappingTraits< + std::shared_ptr>, + IO>::value, + "asdf2"); + + static_assert(sn::has_SerializationTraits< + std::shared_ptr>, + IO>::value, + "asdf"); + + static_assert( + !has_SequenceTraits>>>::value, + "fdsa"); + static_assert( + has_MappingTraits< + Hide>>, + EmptyContext>::value, + "fdsa"); + + static_assert( + !missingTraits< + Hide>>, + EmptyContext>::value, + "fdsa"); + } // namespace yaml +} // namespace llvm diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/msgpack/Loading.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/msgpack/Loading.hpp new file mode 100644 index 0000000000..99023d208f --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/msgpack/Loading.hpp @@ -0,0 +1,44 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include + +#include + +namespace Tensile +{ + template + std::shared_ptr> + MessagePackLoadLibraryFile(std::string const& filename); + + template + std::shared_ptr> + MessagePackLoadLibraryData(std::vector const& data); +} diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/msgpack/MessagePack.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/msgpack/MessagePack.hpp new file mode 100644 index 0000000000..0e87746bb6 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/msgpack/MessagePack.hpp @@ -0,0 +1,355 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include +#include +#include + +#include + +namespace Tensile +{ + namespace Serialization + { + template + struct is_Convertible + { + static const bool value = false; + }; + + template <> + struct is_Convertible + { + static const bool value = true; + }; + template <> + struct is_Convertible + { + static const bool value = true; + }; + template <> + struct is_Convertible + { + static const bool value = true; + }; + template <> + struct is_Convertible + { + static const bool value = true; + }; + template <> + struct is_Convertible + { + static const bool value = true; + }; + template <> + struct is_Convertible + { + static const bool value = true; + }; + template <> + struct is_Convertible + { + static const bool value = true; + }; + template <> + struct is_Convertible + { + static const bool value = true; + }; + template <> + struct is_Convertible + { + static const bool value = true; + }; + + void objectToMap(msgpack::object& object, + std::unordered_map& map); + + struct MessagePackInput + { + msgpack::object object; + std::vector error; + + std::unordered_map objectMap; + std::unordered_set usedKeys; + + int enumFound = 0; + + void* context = nullptr; + + MessagePackInput(msgpack::object object, void* context = nullptr) + : object(object) + , context(context) + { + } + + MessagePackInput createSubRef(msgpack::object otherObject) + { + return MessagePackInput(otherObject, context); + } + + void addError(std::string const& msg) + { + error.push_back(msg); + } + + template + void mapRequired(const char* key, T& obj) + { + if(objectMap.empty()) + objectToMap(object, objectMap); + + auto iterator = objectMap.find(key); + if(iterator != objectMap.end()) + { + auto& value = iterator->second; + MessagePackInput subRef = createSubRef(value); + subRef.input(obj); + error.insert(error.end(), subRef.error.begin(), subRef.error.end()); + if(Tensile::Debug::Instance().printDataInit()) + usedKeys.insert(key); + } + else + { + std::string msg = "Unknown key "; + msg += key; + msg += " (keys: "; + bool first = true; + for(auto const& pair : objectMap) + { + if(!first) + msg += ", "; + msg += pair.first; + first = false; + } + msg += ")"; + addError(msg); + } + } + + template + void mapOptional(const char* key, T& obj) + { + if(objectMap.empty()) + objectToMap(object, objectMap); + + auto iterator = objectMap.find(key); + if(iterator != objectMap.end()) + { + auto& value = iterator->second; + createSubRef(value).input(obj); + if(Tensile::Debug::Instance().printDataInit()) + usedKeys.insert(key); + } + } + + template + void input(T& obj) + { + EmptyContext ctx; + input(obj, ctx); + } + + void checkUsedKeys() + { + std::unordered_set fileKeys; + for(auto const& pair : objectMap) + fileKeys.insert(pair.first); + + if(usedKeys != fileKeys) + { + for(auto const& k : fileKeys) + if(usedKeys.find(k) == usedKeys.end()) + addError(concatenate("Error: Unused key ", k)); + } + } + + template + typename std::enable_if::value, + void>::type + input(T& obj, Context& ctx) + { + MappingTraits::mapping(*this, obj, ctx); + + if(Tensile::Debug::Instance().printDataInit()) + checkUsedKeys(); + } + + template + typename std::enable_if::value, + void>::type + input(T& obj, Context& ctx) + { + MappingTraits::mapping(*this, obj); + + if(Tensile::Debug::Instance().printDataInit()) + checkUsedKeys(); + } + + template + typename std::enable_if::value, void>::type input(T& obj, + Context& ctx) + { + object.convert(obj); + } + + template + typename std::enable_if::value, void>::type + input(T& obj, Context& ctx) + { + enumFound = 0; + EnumTraits::enumeration(*this, obj); + + if(enumFound != 1) + addError(concatenate("Enum not found!", obj)); + } + + template + typename std::enable_if::value, void>::type + input(T& obj, Context& ctx) + { + assert(object.type == msgpack::type::object_type::ARRAY); + auto result = object.as>(); + + for(size_t i = 0; i < result.size(); i++) + { + MessagePackInput subRef = createSubRef(result[i]); + auto& value = SequenceTraits::element(*this, obj, i); + subRef.input(value); + + if(!subRef.error.empty()) + { + error.insert(error.end(), subRef.error.begin(), subRef.error.end()); + return; + } + } + } + + template + typename std::enable_if::value, void>::type + input(T& obj, Context& ctx) + { + objectToMap(object, objectMap); + + for(auto& element : objectMap) + { + CustomMappingTraits::inputOne(*this, element.first, obj); + } + } + + template + void enumCase(T& member, const char* key, T value) + { + assert(object.type == msgpack::type::object_type::STR); + std::string result; + object.convert(result); + + if(result == key) + { + enumFound++; + member = value; + } + } + }; + + template <> + struct IOTraits + { + template + static void mapRequired(MessagePackInput& io, const char* key, T& obj) + { + io.mapRequired(key, obj); + } + + template + static void mapOptional(MessagePackInput& io, const char* key, T& obj) + { + io.mapOptional(key, obj); + } + + static bool outputting(MessagePackInput& io) + { + return false; + } + + static void setError(MessagePackInput& io, std::string const& msg) + { + io.error.push_back(msg); + } + + static void setContext(MessagePackInput& io, void* ctx) + { + io.context = ctx; + } + + static void* getContext(MessagePackInput& io) + { + return io.context; + } + + template + static void enumCase(MessagePackInput& io, T& member, const char* key, T value) + { + io.enumCase(member, key, value); + } + }; + + template + struct SequenceTraits, IO> + { + static size_t size(MessagePackInput& io, std::vector& v) + { + return v.size(); + } + static T& element(MessagePackInput& io, std::vector& v, size_t index) + { + if(index >= v.size()) + { + size_t n = index - v.size() + 1; + v.insert(v.end(), n, T()); + } + + return v[index]; + } + }; + + template + struct MappingTraits, IO> + { + using obj = MasterContractionLibrary; + + static void mapping(IO& io, std::shared_ptr& o) + { + PointerMappingTraits::mapping(io, o); + } + }; + } +} diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclFwd.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclFwd.hpp new file mode 100644 index 0000000000..b1bd9ed730 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclFwd.hpp @@ -0,0 +1,47 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#ifndef OCL_FWD_HPP +#define OCL_FWD_HPP + +/* +* This file is intended to fwd declare OpenCL objects +* so that we can reference them in header files and +* not have to pull in the OpenCL headers. We want to +* limit use of OpenCL to actual compilation units (cpp) +* where they are needed. +*/ + +// Fwd declarations +namespace cl +{ + class Program; + class Context; + class Device; + class Buffer; +} // namespace cl + +#endif // OCL_FWD_HPP diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclHardware.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclHardware.hpp new file mode 100644 index 0000000000..c16cb75f51 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclHardware.hpp @@ -0,0 +1,43 @@ +#ifndef OCL_HARDWARE_HPP +#define OCL_HARDWARE_HPP + +#include +#include + +namespace Tensile +{ + namespace ocl + { + struct oclDeviceProp_t + { + std::string name; + size_t totalGlobalMem; + size_t sharedMemPerBlock; + int warpSize; + int maxThreadsPerBlock; + int maxThreadsDim[3]; + int maxGridSize[3]; + int clockRate; + int multiProcessorCount; + int pciBusID; + int pciDeviceID; + size_t maxSharedMemoryPerMultiProcessor; + int gcnArch; + }; + + struct OclAMDGPU : public AMDGPU + { + OclAMDGPU() = default; + OclAMDGPU(oclDeviceProp_t const& prop); + + oclDeviceProp_t properties; + }; + + std::shared_ptr GetCurrentDevice(); + std::shared_ptr GetDevice(int deviceId); + std::shared_ptr GetDevice(oclDeviceProp_t const& prop); + + } // namespace ocl +} // namespace Tensile + +#endif // OCL_HARDWARE_HPP diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclSolutionAdapter.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclSolutionAdapter.hpp new file mode 100644 index 0000000000..46d380a70c --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclSolutionAdapter.hpp @@ -0,0 +1,101 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#ifndef OCL_SOLUTION_ADAPTER_HPP +#define OCL_SOLUTION_ADAPTER_HPP + +#include +#include + +#include + +namespace Tensile +{ + namespace ocl + { + class SolutionAdapter final : public Tensile::SolutionAdapter + { + public: + SolutionAdapter(); + SolutionAdapter(bool debug); + SolutionAdapter(bool debug, std::string const& name); + SolutionAdapter(bool debug, + std::string const& name, + cl::Context context, + cl::Device device); + ~SolutionAdapter() final = default; + + std::string name() const final; + + void loadCodeObjectFile(std::string const& path); + void loadCodeObjectBytes(std::vector const& bytes); + + void loadEmbeddedCodeObjects(); + void loadEmbeddedCodeObjects(std::string const& key); + + void launchKernel(KernelInvocation const& kernel); + void launchKernel(KernelInvocation const& kernel, + cl::CommandQueue stream, + cl::Event* timingEvent = nullptr); + + void launchKernels(std::vector const& kernels); + void launchKernels(std::vector const& kernels, + cl::CommandQueue stream, + cl::Event* timingEvent = nullptr); + + void launchKernels(std::vector const& kernels, + cl::CommandQueue stream, + std::vector& timingEvents); + + void initKernel(std::string const& name); + + private: + cl::Kernel getKernel(std::string const& name); + void addModule(std::string const& name, cl::Program const& module); + void addModules(std::string const& groupName, std::vector const& modules); + + std::mutex m_access; + + std::vector m_modules; + cl::Context m_context; + cl::Device m_device; + std::unordered_map m_kernels; + bool m_debug = false; + bool m_debugSkipLaunch = false; + std::string m_name = "OclSolutionAdapter"; + + std::vector m_loadedModuleNames; + + friend std::ostream& operator<<(std::ostream& stream, SolutionAdapter const& adapter); + }; + + std::ostream& operator<<(std::ostream& stream, SolutionAdapter const& adapter); + std::ostream& operator<<(std::ostream& stream, std::shared_ptr const& ptr); + + } // namespace ocl +} // namespace Tensile + +#endif //OCL_SOLUTION_ADAPTER_HPP diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclUtils.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclUtils.hpp new file mode 100644 index 0000000000..28a185c2b3 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ocl/OclUtils.hpp @@ -0,0 +1,76 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#ifndef OCL_UTILS_HPP +#define OCL_UTILS_HPP + +/* +* A place to put utilities to help with OpenCL +* application workflow. +* Approximates Tensile 'module' loads of +* ELF binaries from file and formatting them as +* executable kernels +*/ + +#include +#include + +#define CL_CHECK(x) \ + if(CL_SUCCESS != (x)) \ + { \ + throw std::runtime_error( \ + concatenate("File: ", __FILE__, " (", __LINE__, ") CL Error code: ", (x))); \ + } + +namespace Tensile +{ + + namespace ocl + { + class oclDeviceProp_t; + + cl::Program clModuleLoad(cl::Context context, + std::vector const& devices, + std::string const& path); + cl::Program clModuleLoadData(cl::Context context, + std::vector const& devices, + std::vector const& bytes); + + oclDeviceProp_t clGetDevicePropertiesAMD(); + oclDeviceProp_t clGetDevicePropertiesAMD(int deviceId); + oclDeviceProp_t clGetDevicePropertiesAMD(int deviceId, cl::Context context); + oclDeviceProp_t clGetDevicePropertiesAMD(cl::Device device); + + } // namespace ocl + +} // namespace Tensile + +namespace cl +{ + std::ostream& operator<<(std::ostream& stream, Buffer buffer); +} + +#endif // OCL_UTILS_HPP diff --git a/tensilelite/Tensile/Source/lib/source/AMDGPU.cpp b/tensilelite/Tensile/Source/lib/source/AMDGPU.cpp new file mode 100644 index 0000000000..51fb565116 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/AMDGPU.cpp @@ -0,0 +1,108 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include + +namespace Tensile +{ + TENSILE_API std::string AMDGPU::type() const + { + return Type(); + } + + TENSILE_API AMDGPU::AMDGPU() {} + + TENSILE_API AMDGPU::AMDGPU(AMDGPU::Processor p, int cus, std::string const& name) + : processor(p) + , computeUnitCount(cus) + , deviceName(name) + { + } + + TENSILE_API AMDGPU::~AMDGPU() = default; + + TENSILE_API bool AMDGPU::runsKernelTargeting(AMDGPU::Processor other) const + { + if(other > this->processor) + return false; + if(other == this->processor) + return true; + + if(other == Processor::gfx803) + return false; + + if(other == Processor::gfx900) + return true; + + return false; + } + + std::ostream& operator<<(std::ostream& stream, AMDGPU::Processor p) + { + switch(p) + { + case AMDGPU::Processor::gfx803: + return stream << "gfx803"; + case AMDGPU::Processor::gfx900: + return stream << "gfx900"; + case AMDGPU::Processor::gfx906: + return stream << "gfx906"; + case AMDGPU::Processor::gfx908: + return stream << "gfx908"; + case AMDGPU::Processor::gfx90a: + return stream << "gfx90a"; + case AMDGPU::Processor::gfx1010: + return stream << "gfx1010"; + case AMDGPU::Processor::gfx1011: + return stream << "gfx1011"; + case AMDGPU::Processor::gfx1012: + return stream << "gfx1012"; + case AMDGPU::Processor::gfx1030: + return stream << "gfx1030"; + case AMDGPU::Processor::gfx1100: + return stream << "gfx1100"; + case AMDGPU::Processor::gfx1101: + return stream << "gfx1101"; + case AMDGPU::Processor::gfx1102: + return stream << "gfx1102"; + } + return stream; + } + + TENSILE_API std::string AMDGPU::description() const + { + std::ostringstream rv; + + rv << deviceName << "(" << computeUnitCount << "-CU " << processor << ")"; + + return rv.str(); + } + + TENSILE_API std::ostream& operator<<(std::ostream& stream, AMDGPU g) + { + return stream << g.description(); + } +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/Activation.cpp b/tensilelite/Tensile/Source/lib/source/Activation.cpp new file mode 100644 index 0000000000..baf7483ad7 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/Activation.cpp @@ -0,0 +1,137 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include + +namespace Tensile +{ + std::string ToString(ActivationType d) + { + switch(d) + { + case ActivationType::Abs: + return "Abs"; + case ActivationType::Clippedrelu: + return "Clippedrelu"; + case ActivationType::Exp: + return "Exp"; + case ActivationType::Gelu: + return "Gelu"; + case ActivationType::Leakyrelu: + return "Leakyrelu"; + case ActivationType::Relu: + return "Relu"; + case ActivationType::Sigmoid: + return "Sigmoid"; + case ActivationType::Tanh: + return "Tanh"; + case ActivationType::All: + return "All"; + case ActivationType::None: + return "None"; + default: + return "Invalid"; + } + return "Invalid"; + } + std::ostream& operator<<(std::ostream& stream, const ActivationType& t) + { + return stream << ToString(t); + } + + std::istream& operator>>(std::istream& stream, ActivationType& t) + { + std::string strValue; + stream >> strValue; + if(strValue == ToString(ActivationType::Abs)) + { + t = ActivationType::Abs; + } + else if(strValue == ToString(ActivationType::Clippedrelu)) + { + t = ActivationType::Clippedrelu; + } + else if(strValue == ToString(ActivationType::Exp)) + { + t = ActivationType::Exp; + } + else if(strValue == ToString(ActivationType::Gelu)) + { + t = ActivationType::Gelu; + } + else if(strValue == ToString(ActivationType::Leakyrelu)) + { + t = ActivationType::Leakyrelu; + } + else if(strValue == ToString(ActivationType::Relu)) + { + t = ActivationType::Relu; + } + else if(strValue == ToString(ActivationType::Sigmoid)) + { + t = ActivationType::Sigmoid; + } + else if(strValue == ToString(ActivationType::Tanh)) + { + t = ActivationType::Tanh; + } + else if(strValue == ToString(ActivationType::All)) + { + t = ActivationType::All; + } + else if(strValue == ToString(ActivationType::None)) + { + t = ActivationType::None; + } + else + { + throw std::runtime_error(concatenate("Invalid data type: ", strValue)); + } + return stream; + } + + int getAdditionalArgNum(ActivationType d) + { + std::map argMap; + argMap.insert(std::pair(ActivationType::Clippedrelu, 2)); + argMap.insert(std::pair(ActivationType::Leakyrelu, 1)); + argMap.insert(std::pair(ActivationType::Tanh, 2)); + + if(d == ActivationType::All) + { + int maxArgs = 0; + for(auto iter = argMap.begin(); iter != argMap.end(); iter++) + maxArgs = std::max(maxArgs, iter->second); + return maxArgs; + } + auto iter = argMap.find(d); + if(iter != argMap.end()) + return iter->second; + + return 0; + } +} diff --git a/tensilelite/Tensile/Source/lib/source/ArithmeticUnitTypes.cpp b/tensilelite/Tensile/Source/lib/source/ArithmeticUnitTypes.cpp new file mode 100644 index 0000000000..173373cd81 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/ArithmeticUnitTypes.cpp @@ -0,0 +1,139 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include + +#include + +namespace Tensile +{ + std::map ArithmeticUnitTypeInfo::data; + std::map ArithmeticUnitTypeInfo::typeNames; + + std::string ToString(ArithmeticUnit d) + { + switch(d) + { + case ArithmeticUnit::Any: + return "Any"; + case ArithmeticUnit::MFMA: + return "MFMA"; + case ArithmeticUnit::VALU: + return "VALU"; + + case ArithmeticUnit::Count: + default:; + } + return "Invalid"; + } + + template + void ArithmeticUnitTypeInfo::registerTypeInfo() + { + using T_Info = ArithmeticUnitInfo; + + ArithmeticUnitTypeInfo info; + + info.m_arithmeticUnit = T_Info::Enum; + info.name = T_Info::Name(); + + addInfoObject(info); + } + + void ArithmeticUnitTypeInfo::registerAllTypeInfo() + { + registerTypeInfo(); + registerTypeInfo(); + registerTypeInfo(); + } + + void ArithmeticUnitTypeInfo::registerAllTypeInfoOnce() + { + static int call_once = (registerAllTypeInfo(), 0); + + // Use the variable to quiet the compiler. + if(call_once) + return; + } + + void ArithmeticUnitTypeInfo::addInfoObject(ArithmeticUnitTypeInfo const& info) + { + auto toLower = [](std::string tmp) { + std::transform(tmp.begin(), tmp.end(), tmp.begin(), ::tolower); + return tmp; + }; + + data[info.m_arithmeticUnit] = info; + + // Add some flexibility to names registry. Accept + // lower case versions of the strings + typeNames[info.name] = info.m_arithmeticUnit; + typeNames[toLower(info.name)] = info.m_arithmeticUnit; + } + + ArithmeticUnitTypeInfo const& ArithmeticUnitTypeInfo::Get(int index) + { + return Get(static_cast(index)); + } + + ArithmeticUnitTypeInfo const& ArithmeticUnitTypeInfo::Get(ArithmeticUnit t) + { + registerAllTypeInfoOnce(); + + auto iter = data.find(t); + if(iter == data.end()) + throw std::runtime_error(concatenate("Invalid arithmetic unit: ", static_cast(t))); + + return iter->second; + } + + ArithmeticUnitTypeInfo const& ArithmeticUnitTypeInfo::Get(std::string const& str) + { + registerAllTypeInfoOnce(); + + auto iter = typeNames.find(str); + if(iter == typeNames.end()) + throw std::runtime_error(concatenate("Invalid arithmetic unit: ", str)); + + return Get(iter->second); + } + + std::ostream& operator<<(std::ostream& stream, const ArithmeticUnit& t) + { + return stream << ToString(t); + } + + std::istream& operator>>(std::istream& stream, ArithmeticUnit& t) + { + std::string strValue; + stream >> strValue; + + t = ArithmeticUnitTypeInfo::Get(strValue).m_arithmeticUnit; + + return stream; + } +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp b/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp new file mode 100644 index 0000000000..1ee08433be --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp @@ -0,0 +1,1292 @@ +/** + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace Tensile +{ + std::string ContractionProblem::ZeroPad::description() const + { + std::ostringstream rv; + + rv << "anchorIndex: " << anchorIndex << " anchorPos: " << anchorPos + << " boundIndex: " << boundIndex << " boundPos: " << boundPos + << " padStart: " << padStart << " padEnd: " << padEnd; + + return rv.str(); + } + + ContractionProblem ContractionProblem::GEMM_Strides(bool transA, + bool transB, + DataType aType, + DataType bType, + DataType cType, + DataType dType, + size_t m, + size_t n, + size_t k, + size_t batchSize, + size_t lda, + size_t aStride, + size_t ldb, + size_t bStride, + size_t ldc, + size_t cStride, + size_t ldd, + size_t dStride, + double beta) + { + return GEMM_Strides(transA, + transB, + aType, + bType, + cType, + dType, + m, + n, + k, + batchSize, + lda, + aStride, + 0, + ldb, + bStride, + 0, + ldc, + cStride, + 0, + ldd, + dStride, + 0, + beta); + } + + ContractionProblem ContractionProblem::GEMM_Strides(bool transA, + bool transB, + DataType aType, + DataType bType, + DataType cType, + DataType dType, + size_t m, + size_t n, + size_t k, + size_t batchSize, + size_t lda, + size_t aStride, + size_t aOffset, + size_t ldb, + size_t bStride, + size_t bOffset, + size_t ldc, + size_t cStride, + size_t cOffset, + size_t ldd, + size_t dStride, + size_t dOffset, + double beta) + { + Tensile::ContractionProblem::FreeIndices free(2); + Tensile::ContractionProblem::BoundIndices bound(1); + Tensile::ContractionProblem::BatchIndices batch(1); + + free[0].isA = true; + free[0].i = free[0].c = free[0].d = 0; + free[1].isA = false; + free[1].i = free[1].c = free[1].d = 1; + + batch[0].a = batch[0].b = batch[0].c = batch[0].d = 2; + + TensorDescriptor a, b, c, d; + + if(transA) + { + a = TensorDescriptor(aType, {k, m, batchSize}, {1, lda, aStride}, aOffset); + free[0].i = 1; + bound[0].a = 0; + } + else + { + a = TensorDescriptor(aType, {m, k, batchSize}, {1, lda, aStride}, aOffset); + free[0].i = 0; + bound[0].a = 1; + } + + if(transB) + { + b = TensorDescriptor(bType, {n, k, batchSize}, {1, ldb, bStride}, bOffset); + free[1].i = 0; + bound[0].b = 1; + } + else + { + b = TensorDescriptor(bType, {k, n, batchSize}, {1, ldb, bStride}, bOffset); + free[1].i = 1; + bound[0].b = 0; + } + + c = TensorDescriptor(cType, {m, n, batchSize}, {1, ldc, cStride}, cOffset); + d = TensorDescriptor(dType, {m, n, batchSize}, {1, ldd, dStride}, dOffset); + + TensorOps nop; + + ContractionProblem problem(a, nop, b, nop, c, nop, d, nop, free, batch, bound, beta); + + return problem; + } + + ContractionProblem ContractionProblem::GEMM(bool transA, + bool transB, + size_t m, + size_t n, + size_t k, + size_t lda, + size_t ldb, + size_t ldc, + double beta, + bool colMajor, + size_t batchCount) + { + return GEMM(transA, transB, m, n, k, lda, 0, ldb, 0, ldc, 0, beta, colMajor, batchCount); + } + + ContractionProblem ContractionProblem::GEMM(bool transA, + bool transB, + size_t m, + size_t n, + size_t k, + size_t lda, + size_t offsetA, + size_t ldb, + size_t offsetB, + size_t ldc, + size_t offsetC, + double beta, + bool colMajor, + size_t batchCount) + { + if(colMajor) + throw std::runtime_error("Column major not yet implemented."); + + Tensile::ContractionProblem::FreeIndices free(2); + BoundIndex bound; + + free[0].isA = true; + free[0].i = free[0].c = free[0].d = 0; + free[1].isA = false; + free[1].i = free[1].c = free[1].d = 1; + + TensorDescriptor a, b, c, d; + if(transA) + { + a = TensorDescriptor(DataType::Float, {k, m}, {1, lda}, offsetA); + free[0].i = 1; + bound.a = 0; + } + else + { + a = TensorDescriptor(DataType::Float, {m, k}, {1, lda}, offsetA); + free[0].i = 0; + bound.a = 1; + } + + if(transB) + { + b = TensorDescriptor(DataType::Float, {n, k}, {1, ldb}, offsetB); + free[1].i = 0; + bound.b = 1; + } + else + { + b = TensorDescriptor(DataType::Float, {k, n}, {1, ldb}, offsetB); + free[1].i = 1; + bound.b = 0; + } + + FreeIndices freeIndices{free}; + BatchIndices batchIndices; + BoundIndices boundIndices{bound}; + + d = TensorDescriptor(DataType::Float, {m, n}, {1, ldc}, offsetC); + + a.appendDim(batchCount); + b.appendDim(batchCount); + d.appendDim(batchCount); + + batchIndices.push_back({2, 2, 2, 2}); + + c = d; + + TensorOps nop; + + return ContractionProblem( + a, nop, b, nop, c, nop, d, nop, freeIndices, batchIndices, boundIndices, beta); + } + + ContractionProblem ContractionProblem::GEMM(bool transA, + bool transB, + TensorDescriptor const& a, + TensorOps const& aOps, + TensorDescriptor const& b, + TensorOps const& bOps, + TensorDescriptor const& c, + TensorOps const& cOps, + TensorDescriptor const& d, + TensorOps const& dOps, + double beta) + { + Tensile::ContractionProblem::FreeIndices free(2); + BoundIndex bound; + + free[0].isA = true; + free[0].i = free[0].c = free[0].d = 0; + free[1].isA = false; + free[1].i = free[1].c = free[1].d = 1; + + if(transA) + { + free[0].i = 1; + bound.a = 0; + } + else + { + free[0].i = 0; + bound.a = 1; + } + + if(transB) + { + free[1].i = 0; + bound.b = 1; + } + else + { + free[1].i = 1; + bound.b = 0; + } + + FreeIndices freeIndices{free}; + BatchIndices batchIndices; + BoundIndices boundIndices{bound}; + + batchIndices.push_back({2, 2, 2, 2}); + + return ContractionProblem( + a, aOps, b, bOps, c, cOps, d, cOps, freeIndices, batchIndices, boundIndices, beta); + } + + void ContractionProblem::IdentifierToIndices(std::string const& identifier, + FreeIndices& freeIndices, + BatchIndices& batchIndices, + BoundIndices& boundIndices, + TensorOps& aOps, + TensorOps& bOps, + TensorOps& cOps, + TensorOps& dOps) + { + FreeIndices free; + BatchIndices batch; + BoundIndices bound; + + std::string prefix = "Contraction_"; + if(identifier.find(prefix) != 0) + throw std::runtime_error(concatenate( + "Contraction identifier (", identifier, ") must start with '", prefix, "'.")); + + size_t begin = prefix.size(); + size_t end = identifier.find("_", begin); + size_t nextBegin = end + 1; + + std::string boundStr = identifier.substr(begin, end - begin); + + begin = nextBegin; + end = identifier.find("_", begin); + nextBegin = end + 1; + + if(identifier.at(begin) != 'A') + throw std::runtime_error(concatenate( + "Contraction identifier (", identifier, ")must match 'Contraction_s_A_B_C_D'")); + + if(identifier.at(end - 1) == 'C') + { + aOps.push_back(TensorOp::ComplexConjugate()); + end--; + } + + begin++; + std::string a = identifier.substr(begin, end - begin); + + begin = nextBegin; + end = identifier.find("_", begin); + nextBegin = end + 1; + + if(identifier.at(begin) != 'B') + throw std::runtime_error(concatenate( + "Contraction identifier (", identifier, ")must match 'Contraction_s_A_B_C_D'")); + + if(identifier.at(end - 1) == 'C') + { + bOps.push_back(TensorOp::ComplexConjugate()); + end--; + } + + begin++; + std::string b = identifier.substr(begin, end - begin); + + begin = nextBegin; + end = identifier.find("_", begin); + nextBegin = end + 1; + + if(identifier.at(begin) != 'C') + throw std::runtime_error(concatenate( + "Contraction identifier (", identifier, ")must match 'Contraction_s_A_B_C_D'")); + + if(identifier.at(end - 1) == 'C') + { + cOps.push_back(TensorOp::ComplexConjugate()); + end--; + } + + begin++; + std::string c = identifier.substr(begin, end - begin); + + begin = nextBegin; + end = identifier.find("_", begin); + + if(identifier.at(begin) != 'D') + throw std::runtime_error(concatenate( + "Contraction identifier (", identifier, ")must match 'Contraction_s_A_B_C_D'")); + + if(end != std::string::npos) + throw std::runtime_error(concatenate( + "Contraction identifier (", identifier, ")must match 'Contraction_s_A_B_C_D'")); + + end = identifier.size(); + + if(identifier.at(end - 1) == 'C') + { + dOps.push_back(TensorOp::ComplexConjugate()); + end--; + } + + begin++; + std::string d = identifier.substr(begin, end - begin); + + auto caseInsensitiveCmp = [](char a, char b) { return std::tolower(a) < std::tolower(b); }; + std::set allIndices( + a.begin(), a.end(), caseInsensitiveCmp); + allIndices.insert(b.begin(), b.end()); + allIndices.insert(c.begin(), c.end()); + allIndices.insert(d.begin(), d.end()); + + for(char index : allIndices) + { + char caseInsensitiveIndex[] = {char(std::tolower(index)), char(std::toupper(index))}; + size_t aIndex = a.find_first_of(caseInsensitiveIndex, 0, 2); + size_t bIndex = b.find_first_of(caseInsensitiveIndex, 0, 2); + size_t cIndex = c.find(index); + size_t dIndex = d.find(index); + + if(aIndex != std::string::npos && bIndex != std::string::npos + && cIndex != std::string::npos && dIndex != std::string::npos) + { + batch.push_back(BatchIndex{aIndex, bIndex, cIndex, dIndex}); + } + else if(aIndex != std::string::npos && bIndex != std::string::npos + && cIndex == std::string::npos && dIndex == std::string::npos) + { + bool aMirror = std::isupper(a[aIndex]); + bool bMirror = std::isupper(b[bIndex]); + bound.push_back(BoundIndex{aIndex, bIndex, aMirror, bMirror}); + } + else if(aIndex != std::string::npos && bIndex == std::string::npos + && cIndex != std::string::npos && dIndex != std::string::npos) + { + free.resize(free.size() + 1); + + free.back().isA = true; + free.back().i = aIndex; + free.back().c = cIndex; + free.back().d = dIndex; + } + else if(aIndex == std::string::npos && bIndex != std::string::npos + && cIndex != std::string::npos && dIndex != std::string::npos) + { + free.resize(free.size() + 1); + + free.back().isA = false; + free.back().i = bIndex; + free.back().c = cIndex; + free.back().d = dIndex; + } + } + freeIndices = std::move(free); + batchIndices = std::move(batch); + boundIndices = std::move(bound); + } + + ContractionProblem ContractionProblem::FromIndexSizes(std::string const& operationIdentifier, + std::vector const& indexSizes, + DataType aType, + std::vector const& aStrides, + DataType bType, + std::vector const& bStrides, + DataType cType, + std::vector const& cStrides, + DataType dType, + std::vector const& dStrides, + double beta) + { + return FromIndexSizes(operationIdentifier, + indexSizes, + aType, + aStrides, + 0, + bType, + bStrides, + 0, + cType, + cStrides, + 0, + dType, + dStrides, + 0, + beta); + } + + ContractionProblem ContractionProblem::FromIndexSizes(std::string const& operationIdentifier, + std::vector const& indexSizes, + DataType aType, + std::vector const& aStrides, + size_t aOffset, + DataType bType, + std::vector const& bStrides, + size_t bOffset, + DataType cType, + std::vector const& cStrides, + size_t cOffset, + DataType dType, + std::vector const& dStrides, + size_t dOffset, + double beta) + { + FreeIndices freeIndices; + BatchIndices batchIndices; + BoundIndices boundIndices; + + TensorOps aOps, bOps, cOps, dOps; + + IdentifierToIndices( + operationIdentifier, freeIndices, batchIndices, boundIndices, aOps, bOps, cOps, dOps); + + return FromIndexSizes(freeIndices, + batchIndices, + boundIndices, + indexSizes, + aType, + aStrides, + aOps, + aOffset, + bType, + bStrides, + bOps, + bOffset, + cType, + cStrides, + cOps, + cOffset, + dType, + dStrides, + dOps, + dOffset, + beta); + } + + ContractionProblem ContractionProblem::FromIndexSizes(FreeIndices const& freeIndices, + BatchIndices const& batchIndices, + BoundIndices const& boundIndices, + std::vector const& indexSizes, + DataType aType, + std::vector const& aStrides, + TensorOps const& aOps, + DataType bType, + std::vector const& bStrides, + TensorOps const& bOps, + DataType cType, + std::vector const& cStrides, + TensorOps const& cOps, + DataType dType, + std::vector const& dStrides, + TensorOps const& dOps, + double beta) + { + return FromIndexSizes(freeIndices, + batchIndices, + boundIndices, + indexSizes, + aType, + aStrides, + aOps, + 0, + bType, + bStrides, + bOps, + 0, + cType, + cStrides, + cOps, + 0, + dType, + dStrides, + dOps, + 0, + beta); + } + + ContractionProblem ContractionProblem::FromIndexSizes(FreeIndices const& freeIndices, + BatchIndices const& batchIndices, + BoundIndices const& boundIndices, + std::vector const& indexSizes, + DataType aType, + std::vector const& aStrides, + TensorOps const& aOps, + size_t aOffset, + DataType bType, + std::vector const& bStrides, + TensorOps const& bOps, + size_t bOffset, + DataType cType, + std::vector const& cStrides, + TensorOps const& cOps, + size_t cOffset, + DataType dType, + std::vector const& dStrides, + TensorOps const& dOps, + size_t dOffset, + double beta) + { + size_t maxA = 0; + size_t maxB = 0; + size_t maxC = 0; + size_t maxD = 0; + + // Determine number of dimension for each tensor. + + for(auto const& free : freeIndices) + { + if(free.isA) + maxA = std::max(maxA, free.i); + else + maxB = std::max(maxB, free.i); + maxC = std::max(maxC, free.c); + maxD = std::max(maxD, free.d); + } + + for(auto const& batch : batchIndices) + { + maxA = std::max(maxA, batch.a); + maxB = std::max(maxB, batch.b); + maxC = std::max(maxC, batch.c); + maxD = std::max(maxD, batch.d); + } + + for(auto const& bound : boundIndices) + { + maxA = std::max(maxA, bound.a); + maxB = std::max(maxB, bound.b); + } + + std::vector aSizes(maxA + 1), bSizes(maxB + 1), cSizes(maxC + 1), dSizes(maxD + 1); + + for(auto const& free : freeIndices) + { + size_t indexSize = indexSizes.at(free.d); + if(free.isA) + aSizes[free.i] = indexSize; + else + bSizes[free.i] = indexSize; + + cSizes[free.c] = indexSize; + dSizes[free.d] = indexSize; + } + + for(auto const& batch : batchIndices) + { + size_t indexSize = indexSizes.at(batch.d); + + aSizes[batch.a] = indexSize; + bSizes[batch.b] = indexSize; + cSizes[batch.c] = indexSize; + dSizes[batch.d] = indexSize; + } + + size_t indexIdx = dSizes.size(); + for(auto const& bound : boundIndices) + { + size_t indexSize = indexSizes.at(indexIdx); + + aSizes[bound.a] = indexSize; + bSizes[bound.b] = indexSize; + + indexIdx++; + } + + TensorDescriptor a( + aType, aSizes.begin(), aSizes.end(), aStrides.begin(), aStrides.end(), aOffset); + TensorDescriptor b( + bType, bSizes.begin(), bSizes.end(), bStrides.begin(), bStrides.end(), bOffset); + TensorDescriptor c( + cType, cSizes.begin(), cSizes.end(), cStrides.begin(), cStrides.end(), cOffset); + TensorDescriptor d( + dType, dSizes.begin(), dSizes.end(), dStrides.begin(), dStrides.end(), dOffset); + + return ContractionProblem( + a, aOps, b, bOps, c, cOps, d, dOps, freeIndices, batchIndices, boundIndices, beta); + } + + ContractionProblem::ContractionProblem(TensorDescriptor const& a, + TensorOps const& aOps, + TensorDescriptor const& b, + TensorOps const& bOps, + TensorDescriptor const& c, + TensorOps const& cOps, + TensorDescriptor const& d, + TensorOps const& dOps, + FreeIndices const& freeIndices, + BatchIndices const& batchIndices, + BoundIndices const& boundIndices, + double beta, + size_t workspaceSize) + : m_a(a) + , m_b(b) + , m_c(c) + , m_d(d) + , m_aOps(aOps) + , m_bOps(bOps) + , m_cOps(cOps) + , m_dOps(dOps) + , m_freeIndices(freeIndices) + , m_batchIndices(batchIndices) + , m_boundIndices(boundIndices) + , m_beta(beta) + , m_workspaceSize(workspaceSize) + { + m_betaRestriction = toScalarValueEnum( + m_beta); // Set enum using beta to potentially allow for faster solutions + consistencyCheck(); + normalize(); + } + + size_t ContractionProblem::toAPos(size_t idx) const + { + if(idx >= d().dimensions()) + return boundIndices().at(idx - d().dimensions()).a; + + auto found + = std::find_if(freeIndicesA().begin(), + freeIndicesA().end(), + [idx](const ContractionProblem::FreeIndex& fi) { return fi.d == idx; }); + assert(found != freeIndicesA().end()); + assert(found->isA); + + return found->i; + } + + size_t ContractionProblem::toBPos(size_t idx) const + { + if(idx >= d().dimensions()) + return boundIndices().at(idx - d().dimensions()).b; + + auto found + = std::find_if(freeIndicesB().begin(), + freeIndicesB().end(), + [idx](const ContractionProblem::FreeIndex& fi) { return fi.d == idx; }); + assert(found != freeIndicesB().end()); + assert(!found->isA); + + return found->i; + } + + void ContractionProblem::addAZeroPad(const ZeroPad& zp) + { + m_boundIndices[toBoundsPos(zp.boundIndex)].aZeroPad = zp; + m_boundIndices[toBoundsPos(zp.boundIndex)].aZeroPad.anchorPos = toAPos(zp.anchorIndex); + m_boundIndices[toBoundsPos(zp.boundIndex)].aZeroPad.boundPos = toAPos(zp.boundIndex); + m_aZeroPads.push_back(m_boundIndices[toBoundsPos(zp.boundIndex)].aZeroPad); + } + + void ContractionProblem::addBZeroPad(const ZeroPad& zp) + { + m_boundIndices[toBoundsPos(zp.boundIndex)].bZeroPad = zp; + m_boundIndices[toBoundsPos(zp.boundIndex)].bZeroPad.anchorPos = toBPos(zp.anchorIndex); + m_boundIndices[toBoundsPos(zp.boundIndex)].bZeroPad.boundPos = toBPos(zp.boundIndex); + m_bZeroPads.push_back(m_boundIndices[toBoundsPos(zp.boundIndex)].bZeroPad); + } + + void ContractionProblem::checkPersistentKernelEligibility(ContractionSolution const& solution, + Hardware const& hardware) + { + m_eligibleForPK = true; + + // Get the new WorkGroup numbers under the PK and CU value + auto sizeMapping = solution.sizeMapping; + if(sizeMapping.persistentKernel == 0) + return; + + // Get the normal WorkGroup numbers by sizeMapping MacroTile + dim3 numWG(1, 1, 1); + for(size_t i = 0; i < m_freeIndicesA.size(); i++) + { + numWG.x *= m_freeSizesA.at(i); + } + for(size_t i = 0; i < m_freeIndicesB.size(); i++) + { + numWG.y *= m_freeSizesB.at(i); + } + for(size_t i = 0; i < m_batchIndices.size(); i++) + { + if(sizeMapping.packBatchDims & 0x1) + numWG.x *= m_batchSizes[i]; + if(sizeMapping.packBatchDims & 0x2) + numWG.y *= m_batchSizes[i]; + if(!sizeMapping.packBatchDims) + numWG.z *= m_batchSizes[i]; + } + + numWG.x = CeilDivide(numWG.x, sizeMapping.macroTile.x); + numWG.y = CeilDivide(numWG.y, sizeMapping.macroTile.y); + numWG.y *= sizeMapping.globalSplitU; + + size_t problemTiles = numWG.x * numWG.y; + if(sizeMapping.persistentKernelAlongBatch) + problemTiles *= numWG.z; + + AMDGPU const* pAMDGPU = dynamic_cast(&hardware); + assert(pAMDGPU != nullptr && pAMDGPU->computeUnitCount != 0); + + size_t cuCount = pAMDGPU->computeUnitCount; + size_t finalPKValue = sizeMapping.persistentKernel; + if(finalPKValue == -1) + { + // 1. Get the largest pk value (ex.3) + // which can make the PK.G (ex.3*120=360) <= problemGroups (ex.433) + // 2. Scale by 5/8 (can try 0.5~1, to control the tiles-per-workgroup = 1~2) + finalPKValue = 5 * (problemTiles / cuCount) / 8; + finalPKValue = std::max(finalPKValue, (size_t)1); + } + + size_t persistentGroups = cuCount * finalPKValue; + + // If #PKWG (PK*CUs) >= #TotalTiles, the persistent kernel behaves just like non-PK + m_eligibleForPK = persistentGroups < problemTiles; + } + + void ContractionProblem::normalize() + { + m_maxProblemSize = 0; + + m_batchSizes.resize(m_batchIndices.size()); + m_boundSizes.resize(m_boundIndices.size()); + + m_freeSizesA.reserve(m_freeIndices.size()); + m_freeSizesB.reserve(m_freeIndices.size()); + + m_freeIndicesA.reserve(m_freeIndices.size()); + m_freeIndicesB.reserve(m_freeIndices.size()); + + for(int i = 0; i < m_freeIndices.size(); i++) + { + size_t mySize = m_d.sizes()[m_freeIndices[i].d]; + if(m_freeIndices[i].isA) + { + m_freeIndicesA.push_back(m_freeIndices[i]); + m_freeSizesA.push_back(mySize); + } + else + { + m_freeIndicesB.push_back(m_freeIndices[i]); + m_freeSizesB.push_back(mySize); + } + + m_maxProblemSize = std::max(m_maxProblemSize, mySize); + } + + for(int i = 0; i < m_batchIndices.size(); i++) + { + m_batchSizes[i] = std::max({m_a.sizes()[m_batchIndices[i].a], + m_b.sizes()[m_batchIndices[i].b], + m_c.empty() ? 0 : m_c.sizes()[m_batchIndices[i].c], + m_d.sizes()[m_batchIndices[i].d]}); + } + + for(int i = 0; i < m_boundIndices.size(); i++) + { + m_boundSizes[i] + = std::max(m_a.sizes()[m_boundIndices[i].a], m_b.sizes()[m_boundIndices[i].b]); + + m_maxProblemSize = std::max(m_maxProblemSize, m_boundSizes[i]); + } + + for(auto zp : m_aZeroPads) + m_boundIndices[toBoundsPos(zp.boundIndex)].aZeroPad = zp; + for(auto zp : m_bZeroPads) + m_boundIndices[toBoundsPos(zp.boundIndex)].bZeroPad = zp; + + getIndexNames(m_aNames, m_bNames, m_cNames, m_dNames, m_sumNames); + + m_operationIdentifier = getOperationIdentifier(); + + m_problemSizes.resize(0); + m_problemSizes.reserve(m_c.dimensions() + m_boundSizes.size()); + + m_problemSizes.insert(m_problemSizes.end(), m_c.sizes().begin(), m_c.sizes().end()); + m_problemSizes.insert(m_problemSizes.end(), m_boundSizes.begin(), m_boundSizes.end()); + + m_problemStrides.reserve(m_a.dimensions() + m_b.dimensions() + m_c.dimensions() + + m_d.dimensions()); + m_problemStrides.insert(m_problemStrides.end(), m_a.strides().begin(), m_a.strides().end()); + m_problemStrides.insert(m_problemStrides.end(), m_b.strides().begin(), m_b.strides().end()); + m_problemStrides.insert(m_problemStrides.end(), m_c.strides().begin(), m_c.strides().end()); + m_problemStrides.insert(m_problemStrides.end(), m_d.strides().begin(), m_d.strides().end()); + + m_allocatedElementsNonBatchA = 1; + for(int idx = 0; idx < a().dimensions(); idx++) + { + bool isBatch = m_batchIndices.end() + != std::find_if(m_batchIndices.begin(), + m_batchIndices.end(), + [idx](const ContractionProblem::BatchIndex& bi) { + return bi.a == idx; + }); + if(!isBatch) + m_allocatedElementsNonBatchA += m_a.strides()[idx] * (m_a.sizes()[idx] - 1); + } + + m_allocatedElementsNonBatchB = 1; + for(int idx = 0; idx < b().dimensions(); idx++) + { + bool isBatch = m_batchIndices.end() + != std::find_if(m_batchIndices.begin(), + m_batchIndices.end(), + [idx](const ContractionProblem::BatchIndex& bi) { + return bi.b == idx; + }); + if(!isBatch) + m_allocatedElementsNonBatchB += m_b.strides()[idx] * (m_b.sizes()[idx] - 1); + } + + // CD always contain index0. if this is in the B free indices, then need to + // transposing the output tensor. + m_transposeC01 = freeIndicesB().end() + != std::find_if(freeIndicesB().begin(), + freeIndicesB().end(), + [](const ContractionProblem::FreeIndex& fi) { + return fi.c == 0 /*idx0*/; + }); + } + + void ContractionProblem::consistencyCheck() const + { + std::vector aUseCount(m_a.dimensions(), 0); + std::vector bUseCount(m_b.dimensions(), 0); + std::vector cUseCount(m_c.dimensions(), 0); + std::vector dUseCount(m_d.dimensions(), 0); + + for(FreeIndex const& free : m_freeIndices) + { + if(free.isA) + { + aUseCount[free.i]++; + TENSILE_ASSERT_EXC(free.i < m_a.dimensions()); + TENSILE_ASSERT_EXC(m_a.sizes()[free.i] == m_d.sizes()[free.d]); + } + else + { + bUseCount[free.i]++; + TENSILE_ASSERT_EXC(free.i < m_b.dimensions()); + TENSILE_ASSERT_EXC(m_b.sizes()[free.i] == m_d.sizes()[free.d]); + } + + TENSILE_ASSERT_EXC(free.d < m_d.dimensions()); + dUseCount[free.d]++; + + if(!m_c.empty()) + { + TENSILE_ASSERT_EXC(free.c < m_c.dimensions()); + + cUseCount[free.c]++; + + if(free.isA) + TENSILE_ASSERT_EXC(m_a.sizes()[free.i] == m_c.sizes()[free.c]); + else + TENSILE_ASSERT_EXC(m_b.sizes()[free.i] == m_c.sizes()[free.c]); + } + } + + for(BatchIndex const& batch : m_batchIndices) + { + TENSILE_ASSERT_EXC(batch.a < m_a.dimensions()); + TENSILE_ASSERT_EXC(batch.b < m_b.dimensions()); + TENSILE_ASSERT_EXC(batch.d < m_d.dimensions()); + + aUseCount[batch.a]++; + bUseCount[batch.b]++; + dUseCount[batch.d]++; + + size_t aSize = m_a.sizes()[batch.a]; + size_t bSize = m_b.sizes()[batch.b]; + size_t cSize = 1; + size_t dSize = m_d.sizes()[batch.d]; + + if(!m_c.empty()) + { + TENSILE_ASSERT_EXC(batch.c < m_c.dimensions()); + cUseCount[batch.c]++; + + cSize = m_c.sizes()[batch.c]; + } + + size_t indexSize = std::max({aSize, bSize, cSize, dSize}); + + TENSILE_ASSERT_EXC(aSize == 1 || aSize == indexSize); + TENSILE_ASSERT_EXC(bSize == 1 || bSize == indexSize); + TENSILE_ASSERT_EXC(cSize == 1 || cSize == indexSize); + TENSILE_ASSERT_EXC(dSize == 1 || dSize == indexSize); + } + + for(BoundIndex const& bound : m_boundIndices) + { + TENSILE_ASSERT_EXC(bound.a < m_a.dimensions()); + TENSILE_ASSERT_EXC(bound.b < m_b.dimensions()); + + aUseCount[bound.a]++; + bUseCount[bound.b]++; + + TENSILE_ASSERT_EXC(m_a.sizes()[bound.a] == m_b.sizes()[bound.b]); + } + + for(int aUse : aUseCount) + TENSILE_ASSERT_EXC(aUse == 1); + for(int bUse : bUseCount) + TENSILE_ASSERT_EXC(bUse == 1); + for(int cUse : cUseCount) + TENSILE_ASSERT_EXC(cUse == 1); + for(int dUse : dUseCount) + TENSILE_ASSERT_EXC(dUse == 1); + + for(auto const& op : m_aOps) + if(op.type == TensorOp::Type::ComplexConjugate) + TENSILE_ASSERT_EXC(DataTypeInfo::Get(m_a.dataType()).isComplex); + + for(auto const& op : m_bOps) + if(op.type == TensorOp::Type::ComplexConjugate) + TENSILE_ASSERT_EXC(DataTypeInfo::Get(m_b.dataType()).isComplex); + + for(auto const& op : m_cOps) + if(op.type == TensorOp::Type::ComplexConjugate) + TENSILE_ASSERT_EXC(DataTypeInfo::Get(m_c.dataType()).isComplex); + + for(auto const& op : m_dOps) + if(op.type == TensorOp::Type::ComplexConjugate) + TENSILE_ASSERT_EXC(DataTypeInfo::Get(m_d.dataType()).isComplex); + } + + size_t ContractionProblem::freeSizeA(size_t idx) const + { + return m_freeSizesA.at(idx); + } + + size_t ContractionProblem::freeSizeB(size_t idx) const + { + return m_freeSizesB.at(idx); + } + + size_t ContractionProblem::batchSize(size_t idx) const + { + return m_batchSizes[idx]; + } + + size_t ContractionProblem::boundSize(size_t idx) const + { + return m_boundSizes[idx]; + } + + size_t ContractionProblem::size(size_t idx) const + { + if(idx < c().sizes().size()) + return c().sizes()[idx]; + else + return m_boundSizes.at(idx - c().sizes().size()); + } + + size_t ContractionProblem::flopsPerMac() const + { + return 2 * DataTypeInfo::Get(m_a.dataType()).packing; + } + + size_t ContractionProblem::flopCount() const + { + size_t rv = flopsPerMac(); + + for(auto size : m_freeSizesA) + rv *= size; + + for(auto size : m_freeSizesB) + rv *= size; + + for(auto size : m_batchSizes) + rv *= size; + + for(auto size : m_boundSizes) + rv *= size; + + return rv; + } + + void ContractionProblem::getIndexNames(std::string& aNames, + std::string& bNames, + std::string& cNames, + std::string& dNames, + std::string& sumNames) const + { + aNames.resize(m_a.dimensions(), '_'); + bNames.resize(m_b.dimensions(), '_'); + cNames.resize(m_c.dimensions(), '_'); + dNames.resize(m_d.dimensions(), '_'); + sumNames.resize(m_boundIndices.size(), '_'); + + char name = 'i'; + + for(char& ch : dNames) + { + ch = name; + name++; + } + + for(char& ch : sumNames) + { + ch = name; + name++; + } + + for(auto const& free : m_freeIndices) + { + if(free.isA) + aNames[free.i] = dNames[free.d]; + else + bNames[free.i] = dNames[free.d]; + if(!m_c.empty()) + { + cNames[free.c] = dNames[free.d]; + } + } + + for(auto const& batch : m_batchIndices) + { + aNames[batch.a] = dNames[batch.d]; + bNames[batch.b] = dNames[batch.d]; + if(!m_c.empty()) + cNames[batch.c] = dNames[batch.d]; + } + + for(ptrdiff_t i = 0; i < sumNames.size(); i++) + { + auto const& boundIndex = m_boundIndices[i]; + aNames[boundIndex.a] = boundIndex.aMirror ? std::toupper(sumNames[i]) : sumNames[i]; + bNames[boundIndex.b] = boundIndex.bMirror ? std::toupper(sumNames[i]) : sumNames[i]; + } + + if(m_c.empty() || m_beta == 0.0) + cNames = dNames; + } + + std::string ContractionProblem::getOperationDescription() const + { + std::ostringstream rv; + + rv << "D[" << m_dNames << "] = alpha * ("; + + if(!m_sumNames.empty()) + rv << "Sum[" << m_sumNames << "] "; + + rv << "A[" << m_aNames << "] * B[" << m_bNames << "])"; + + if(!m_c.empty() && m_beta != 0) + { + rv << " + "; + if(m_beta != 1.0) + rv << "beta * "; + rv << "C[" << m_cNames << "]"; + } + + return rv.str(); + } + + std::string ContractionProblem::getOperationIdentifier() const + { + std::string rv = "Contraction_"; + rv += m_sumNames; + rv += "_A"; + rv += m_aNames; + for(auto const& op : m_aOps) + rv += op.suffix(); + + rv += "_B"; + rv += m_bNames; + for(auto const& op : m_bOps) + rv += op.suffix(); + + rv += "_C"; + rv += m_cNames; + for(auto const& op : m_cOps) + rv += op.suffix(); + + rv += "_D"; + rv += m_dNames; + for(auto const& op : m_dOps) + rv += op.suffix(); + + return rv; + } + + std::string ContractionProblem::description() const + { + std::ostringstream rv; + + rv << operationIdentifier() << ",\n" + << "A: " << m_a << ",\n" + << "B: " << m_b << ",\n" + << "C: " << m_c << ",\n" + << "D: " << m_d << "\n"; + + return rv.str(); + } + + TENSILE_API std::ostream& operator<<(std::ostream& stream, + ContractionProblem const& contraction) + { + return stream << contraction.description(); + } + + std::ostream& operator<<(std::ostream& stream, ContractionProblem::FreeIndex const& free) + { + return stream << "{isA=" << free.isA << " i=" << free.i << " c=" << free.c + << " d=" << free.d << "}"; + } + std::ostream& operator<<(std::ostream& stream, ContractionProblem::BatchIndex const& batch) + { + if(batch.a == batch.b && batch.a == batch.c && batch.a == batch.d) + return stream << "{" << batch.a << "}"; + + return stream << "{a=" << batch.a << " b=" << batch.b << " c=" << batch.c + << " d=" << batch.d << "}"; + } + + std::ostream& operator<<(std::ostream& stream, ContractionProblem::BoundIndex const& bound) + { + return stream << "{a=" << bound.a << " b=" << bound.b << "}"; + } + + std::istream& operator>>(std::istream& stream, ContractionProblem::FreeIndex& free) + { + StreamRead comma(","); + return stream >> free.isA >> comma >> free.i >> comma >> free.c >> comma >> free.d; + } + + std::istream& operator>>(std::istream& stream, ContractionProblem::BatchIndex& batch) + { + StreamRead comma(","); + return stream >> batch.a >> comma >> batch.b >> comma >> batch.c >> comma >> batch.d; + } + + std::istream& operator>>(std::istream& stream, ContractionProblem::BoundIndex& bound) + { + StreamRead comma(","); + return stream >> bound.a >> comma >> bound.b; + } + + TENSILE_API ProblemInputs::~ProblemInputs() = default; + ContractionInputs::ContractionInputs() = default; + ContractionInputs::~ContractionInputs() = default; + + template + TypedContractionInputs::TypedContractionInputs() = default; + + template + TypedContractionInputs::~TypedContractionInputs() = default; + + template + TypedContractionInputs::TypedContractionInputs(A const* _a, + B const* _b, + C const* _c, + D* _d, + A const* const* _batchA, + B const* const* _batchB, + C const* const* _batchC, + D* const* _batchD, + Alpha _alpha, + Beta _beta, + A const* _bias, + void* _ws) + : a(_a) + , b(_b) + , c(_c) + , d(_d) + , batchA(_batchA) + , batchB(_batchB) + , batchC(_batchC) + , batchD(_batchD) + , ws(_ws) + , alpha(_alpha) + , beta(_beta) + , bias(_bias) + { + } + + template struct TypedContractionInputs; + template struct TypedContractionInputs; + template struct TypedContractionInputs>; + template struct TypedContractionInputs>; + template struct TypedContractionInputs; + template struct TypedContractionInputs; + template struct TypedContractionInputs; + template struct TypedContractionInputs; + +#ifdef TENSILE_USE_HALF + template struct TypedContractionInputs; + template struct TypedContractionInputs; + template struct TypedContractionInputs; +#endif +#ifdef TENSILE_USE_BF16 + template struct TypedContractionInputs; + template struct TypedContractionInputs; +#endif +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp new file mode 100644 index 0000000000..c693eea94f --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp @@ -0,0 +1,1595 @@ +/** + * Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include +#include +#include + +#include +#include +#include + +namespace Tensile +{ + PerfModel perf; + + int32_t ContractionSolution::staggerUIter(ContractionSolution::Problem const& problem, + ContractionSolution::Inputs const& inputs, + Hardware const& hardware) const + { + uint32_t sizeL = problem.boundSize(0); + + // how many stride-sized clicks to stagger start offset + unsigned int staggerUIter = sizeMapping.staggerU; + + // /DepthU/GSU + int unrollLoopIters = sizeL / sizeMapping.depthU / sizeMapping.globalSplitU; + + unsigned int shifted = 1 << sizeMapping.staggerStrideShift; + + while(staggerUIter > 1) + { + if(unrollLoopIters >= (staggerUIter * shifted)) + break; + + staggerUIter /= 2; // step down to smaller stagger + } + + if(staggerUIter >= 1) + staggerUIter -= 1; + + return staggerUIter; + } + + // Return magic number. If magicShift is 0, compute and return it. + uint32_t ContractionSolution::magicNumberAlg1(uint32_t x, uint32_t* magicShift) const + { + uint64_t magicNum; + *magicShift = 33; + magicNum = (1L << *magicShift) / x + 1; + if((magicNum >> 32) != 0) + { + *magicShift = 31; + magicNum = (1L << *magicShift) / x + 1; + } + + assert(magicNum >> 32 == 0); // ensure magic number fits + + return static_cast(magicNum); + } + + uint32_t ContractionSolution::magicNumberAlg2(uint32_t d, uint32_t* magicShift) const + { + struct mu + { + unsigned M; // Magic number, + int a; // "add" indicator, + int s; + }; // and shift amount. + + struct mu magu; + if(d == 0) + { + // Make dividend of 0 return 0 + magu.M = 0; + magu.a = 0; + magu.s = 0; + } + else + { + // Must have 1 <= d <= 2**32-1. + int p; + unsigned nc, delta, q1, r1, q2, r2; + magu.a = 0; // Initialize "add" indicator. + nc = -1 - (-d) % d; // Unsigned arithmetic here. + p = 31; // Init. p. + q1 = 0x80000000 / nc; // Init. q1 = 2**p/nc. + r1 = 0x80000000 - q1 * nc; // Init. r1 = rem(2**p, nc). + q2 = 0x7FFFFFFF / d; // Init. q2 = (2**p - 1)/d. + r2 = 0x7FFFFFFF - q2 * d; // Init. r2 = rem(2**p - 1, d). + do + { + p = p + 1; + if(r1 >= nc - r1) + { + q1 = 2 * q1 + 1; // Update q1. + r1 = 2 * r1 - nc; + } // Update r1. + else + { + q1 = 2 * q1; + r1 = 2 * r1; + } + if(r2 + 1 >= d - r2) + { + if(q2 >= 0x7FFFFFFF) + magu.a = 1; + q2 = 2 * q2 + 1; // Update q2. + r2 = 2 * r2 + 1 - d; + } // Update r2. + else + { + if(q2 >= 0x80000000) + magu.a = 1; + q2 = 2 * q2; + r2 = 2 * r2 + 1; + } + delta = d - 1 - r2; + } while(p < 64 && (q1 < delta || (q1 == delta && r1 == 0))); + + magu.M = q2 + 1; // Magic number + magu.s = p - 32; // and shift amount to return + } + + *magicShift = magu.s; + const uint32_t abit = 0x80000000; + if(magu.a) + *magicShift |= abit; + + // std::cout << " d=" << d << " M=" << magu.M << " a=" << magu.a << " s=" << + // magu.s << "\n"; + + return magu.M; + } + + uint32_t + ContractionSolution::magicNumber(int magicDivAlg, uint32_t x, uint32_t* magicShift) const + { + if(magicDivAlg == 1) + return magicNumberAlg1(x, magicShift); + else if(magicDivAlg == 2) + return magicNumberAlg2(x, magicShift); + else + throw std::runtime_error("bad magicDivAlg"); + } + + uint32_t ContractionSolution::smallMagicNumber(uint32_t x) const + { + uint64_t magicNum; + const int smallMagicShift = 31; + magicNum = (1L << smallMagicShift) / x + 1; + assert(magicNum >> 32 == 0); // ensure magic number fits + return static_cast(magicNum); + } + + std::vector generatePackedIndicesA(ContractionSolution::Problem const& problem, + size_t packBatchDims) + { + std::vector packedIndices; + + // TODO -move packedIndices calc to problem decode. + for(auto idx = 0; idx < problem.a().dimensions(); idx++) + { + bool isSum = problem.boundIndices().end() + != std::find_if(problem.boundIndices().begin(), + problem.boundIndices().end(), + [idx](const ContractionProblem::BoundIndex& bi) { + return bi.a == idx; + }); + + bool nonPackableBatch = false; + // TODO - base this check on if the batch is SetConstStrideA=0 - if so, + // don't pack + if(!(packBatchDims & 0x1)) + { + nonPackableBatch = problem.batchIndices().end() + != std::find_if(problem.batchIndices().begin(), + problem.batchIndices().end(), + [idx](const ContractionProblem::BatchIndex& bi) { + return bi.a == idx; + }); + } + + if(!isSum && !nonPackableBatch) + packedIndices.push_back(idx); + } + + return packedIndices; + } + + std::vector generatePackedIndicesB(ContractionSolution::Problem const& problem, + size_t packBatchDims) + { + std::vector packedIndices; + + // Pack in all non-summation indices, except don't need magic number for the + // last one + for(auto idx = 0; idx < problem.b().dimensions(); idx++) + { + bool isSum = problem.boundIndices().end() + != std::find_if(problem.boundIndices().begin(), + problem.boundIndices().end(), + [idx](const ContractionProblem::BoundIndex& bi) { + return bi.b == idx; + }); + + bool nonPackableBatch = false; + // TODO - base this check on if the batch is SetConstStrideB=0 - if so, + // don't pack + if(!(packBatchDims & 0x2)) + { + nonPackableBatch = problem.batchIndices().end() + != std::find_if(problem.batchIndices().begin(), + problem.batchIndices().end(), + [idx](const ContractionProblem::BatchIndex& bi) { + return bi.b == idx; + }); + } + + if(!isSum && !nonPackableBatch) + packedIndices.push_back(idx); + } + + return packedIndices; + } + + template + KernelInvocation + ContractionSolution::generateSingleCall(ContractionSolution::Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const + { + TENSILE_ASSERT_EXC(sizeMapping.workGroupMapping >= 0); + + TensorDescriptor const& a = problem.a(); + TensorDescriptor const& b = problem.b(); + TensorDescriptor const& c = problem.c(); + TensorDescriptor const& d = problem.d(); + + KernelInvocation rv; + + rv.args = KernelArguments(T_Debug); + + rv.args.reserve(1024, 128); + + rv.kernelName = kernelName; + + rv.workGroupSize.x = sizeMapping.workGroupSize.x * sizeMapping.workGroupSize.y + * sizeMapping.workGroupSize.z; + rv.workGroupSize.y = 1; + rv.workGroupSize.z = 1; + + rv.numWorkGroups.x = 1; + rv.numWorkGroups.y = 1; + + for(size_t i = 0; i < problem.freeIndicesA().size(); i++) + { + rv.numWorkGroups.x *= problem.freeSizeA(i); + } + for(size_t i = 0; i < problem.freeIndicesB().size(); i++) + { + rv.numWorkGroups.y *= problem.freeSizeB(i); + } + + rv.numWorkGroups.z = 1; + for(size_t i = 0; i < problem.batchIndices().size(); i++) + { + if(sizeMapping.packBatchDims & 0x1) + rv.numWorkGroups.x *= problem.batchSize(i); + if(sizeMapping.packBatchDims & 0x2) + rv.numWorkGroups.y *= problem.batchSize(i); + if(!sizeMapping.packBatchDims) + rv.numWorkGroups.z *= problem.batchSize(i); + } + + if(problem.transposeC01()) + std::swap(rv.numWorkGroups.x, rv.numWorkGroups.y); + + rv.numWorkGroups.x = CeilDivide(rv.numWorkGroups.x, sizeMapping.macroTile.x); + rv.numWorkGroups.y = CeilDivide(rv.numWorkGroups.y, sizeMapping.macroTile.y); + + uint32_t problemNumGroupTiles0 = rv.numWorkGroups.x; + uint32_t problemNumGroupTiles1 = rv.numWorkGroups.y; + // used only when persistent kernel along batch + uint32_t problemNumGroupTiles2 = rv.numWorkGroups.z; + + rv.numWorkGroups.y *= sizeMapping.globalSplitU; + + if(sizeMapping.persistentKernel != 0) + { + AMDGPU const* pAMDGPU = dynamic_cast(&hardware); + assert(pAMDGPU != nullptr && pAMDGPU->computeUnitCount != 0); + + size_t cuCount = pAMDGPU->computeUnitCount; + size_t finalPKValue = sizeMapping.persistentKernel; + size_t problemGroups = rv.numWorkGroups.x * rv.numWorkGroups.y; + if(sizeMapping.persistentKernelAlongBatch) + { + problemGroups *= rv.numWorkGroups.z; + rv.numWorkGroups.z = 1; + } + + if(finalPKValue == -1) + { + // 1. Get the largest pk value (ex.3) + // which can make the PK.G (ex.3*120=360) <= problemGroups (ex.433) + // 2. Scale by 5/8 (can try 0.5~1, to control the tiles-per-workgroup = 1~2) + finalPKValue = 5 * (problemGroups / cuCount) / 8; + finalPKValue = std::max(finalPKValue, (size_t)1); + //std::cout << "final persistent kernel value: " << finalPKValue << std::endl; + } + + size_t persistentGroups = cuCount * finalPKValue; + rv.numWorkGroups.x = std::min(persistentGroups, problemGroups); + rv.numWorkGroups.y = 1; + } + + rv.numWorkItems.x = rv.workGroupSize.x * rv.numWorkGroups.x; + rv.numWorkItems.y = rv.workGroupSize.y * rv.numWorkGroups.y; + rv.numWorkItems.z = rv.workGroupSize.z * rv.numWorkGroups.z; + + if(debugKernel) + { + rv.args.appendUnbound("debugBuffer"); + } + + rv.sharedMemBytes = 0; + + if(!isSourceKernel()) + { + uint64_t tensor2dSizeC = c.totalAllocatedElements(); + uint64_t tensor2dSizeA = (sizeMapping.packBatchDims & 0x1) + ? a.totalAllocatedElements() + : problem.allocatedElementsNonBatchA(); + uint64_t tensor2dSizeB = (sizeMapping.packBatchDims & 0x2) + ? b.totalAllocatedElements() + : problem.allocatedElementsNonBatchB(); + + rv.args.append("tensor2dSizeC", tensor2dSizeC); + rv.args.append("tensor2dSizeA", tensor2dSizeA); + rv.args.append("tensor2dSizeB", tensor2dSizeB); + } + + if(sizeMapping.globalAccumulation) + { + rv.args.append("ws_d", inputs.ws); + rv.args.append("ws_c", inputs.ws); + } + else if(problemType.stridedBatched) + { + rv.args.append("d", inputs.d); + rv.args.append("c", inputs.c); + } + else + { + rv.args.append("batchD", inputs.batchD); + rv.args.append("batchC", inputs.batchC); + } + + if(problemType.stridedBatched) + { + rv.args.append("a", inputs.a); + rv.args.append("b", inputs.b); + } + else + { + rv.args.append("batchA", inputs.batchA); + rv.args.append("batchB", inputs.batchB); + } + + rv.args.append("alpha", inputs.alpha); + if(std::is_same::value && !isSourceKernel()) + rv.args.append("alpha_2", inputs.alpha); + + if(problemType.useBeta) + { + rv.args.append("beta", inputs.beta); + if(std::is_same::value && !isSourceKernel()) + rv.args.append("beta_2", inputs.beta); + } + + if(problemType.useBias && (sizeMapping.globalSplitU == 1)) + { + rv.args.append("bias", inputs.bias); + } + + if(!isSourceKernel()) + { + if((problem.activationType() != ActivationType::None) && sizeMapping.activationFused + && (sizeMapping.globalSplitU == 1)) + { + for(int i = 0; i < inputs.activationArgs.size(); i++) + { + std::string name = "activation_" + std::to_string(i); + if(problem.activationHPA()) // Same as hpa type. + { + rv.args.append( + name.c_str(), + static_cast(inputs.activationArgs[i])); + } + else if(std::is_same::value) + { + rv.args.append((name + "_pk").c_str(), + inputs.activationArgs[i]); + rv.args.append(name.c_str(), + inputs.activationArgs[i]); + } + else + { + if(std::is_same::value) + { + // BFloat16 to float32. + rv.args.append((name + "_append").c_str(), + static_cast(0)); + } + rv.args.append(name.c_str(), + inputs.activationArgs[i]); + } + } + if(problem.activationType() == ActivationType::All) + { + rv.args.append("activationType", + static_cast(problem.activationEnumArg())); + } + } + } + + size_t startStrideCD = problemType.useInitialStridesCD ? 0 : 1; + size_t startStrideAB = problemType.useInitialStridesAB ? 0 : 1; + + if(sizeMapping.globalAccumulation) + { + size_t wsStride = startStrideCD ? d.sizes()[0] : 1; + for(size_t i = startStrideCD; i < d.dimensions(); i++) + { + rv.args.append(concatenate_if("strideW_D", i), wsStride); + wsStride *= d.sizes()[i]; + } + + wsStride = startStrideCD ? d.sizes()[0] : 1; + for(size_t i = startStrideCD; i < c.dimensions(); i++) + { + rv.args.append(concatenate_if("strideW_C", i), wsStride); + wsStride *= d.sizes()[i]; + } + } + else + { + for(size_t i = startStrideCD; i < d.dimensions(); i++) + rv.args.append(concatenate_if("strideD", i), d.strides()[i]); + + for(size_t i = startStrideCD; i < c.dimensions(); i++) + rv.args.append(concatenate_if("strideC", i), c.strides()[i]); + } + + for(size_t i = startStrideAB; i < a.dimensions(); i++) + rv.args.append(concatenate_if("strideA", i), a.strides()[i]); + + for(size_t i = startStrideAB; i < b.dimensions(); i++) + rv.args.append(concatenate_if("strideB", i), b.strides()[i]); + + { + int idx = 0; + for(auto size : problem.problemSizes()) + { + rv.args.append(concatenate_if("size_", idx), size); + idx++; + } + } + + if(sizeMapping.packSummationDims) + // boundIndices are ordered with unroll last. + // Magic numbers for all but first are needed to unpack other dims. + for(auto si = 1; si < problem.boundIndices().size(); si++) + { + auto numIter = problem.boundSize(si); + bool isUnroll = si == problem.boundIndices().size() - 1; + if(isUnroll) + { + numIter = numIter / sizeMapping.depthU / sizeMapping.globalSplitU + * sizeMapping.depthU; + } + uint32_t magicShift; + rv.args.append( + concatenate_if("magicNumberNumIter_", si), + magicNumber(sizeMapping.magicDivAlg, numIter, &magicShift)); + rv.args.append(concatenate_if("magicShiftNumIter_", si), + magicShift); + + if(isUnroll and sizeMapping.globalSplitU > 1) + { + // compute magic number for gsu remainder iterations: + // Kernel will select whether to use above or remainder portion based on work-group assignment + rv.args.append( + concatenate_if("magicNumberNumIter_GsuRemainder"), + magicNumber( + sizeMapping.magicDivAlg, numIter + sizeMapping.depthU, &magicShift)); + rv.args.append( + concatenate_if("magicShiftNumIter_GsuRemainder"), magicShift); + } + } + + if(problem.freeIndicesA().size() > 1 || sizeMapping.packBatchDims & 0x1) + { + std::vector packedIndices + = generatePackedIndicesA(problem, sizeMapping.packBatchDims); + + // Pack in all non-summation indices, except don't need magic number for the + // last one + for(auto pi = packedIndices.begin(); pi != packedIndices.end() - 1; pi++) + { + auto idx = *pi; + auto size = a.sizes()[idx]; + uint32_t magicShift; + rv.args.append(concatenate_if("magicNumberSizeA_", idx), + magicNumber(sizeMapping.magicDivAlg, size, &magicShift)); + rv.args.append(concatenate_if("magicShiftSizeA_", idx), + magicShift); + } + } + if(problem.freeIndicesB().size() > 1 || sizeMapping.packBatchDims & 0x2) + { + std::vector packedIndices + = generatePackedIndicesB(problem, sizeMapping.packBatchDims); + + // Pack in all non-summation indices, except don't need magic number for the + // last one + for(auto pi = packedIndices.begin(); pi != packedIndices.end() - 1; pi++) + { + auto idx = *pi; + auto size = b.sizes()[idx]; + uint32_t magicShift; + rv.args.append(concatenate_if("magicNumberSizeB_", idx), + magicNumber(sizeMapping.magicDivAlg, size, &magicShift)); + rv.args.append(concatenate_if("magicShiftSizeB_", idx), + magicShift); + } + } + + for(auto si : problem.boundIndices()) + { + if(si.aZeroPad.valid()) + { + rv.args.append(concatenate_if("padStartA_", si.a), + si.aZeroPad.padStart); + rv.args.append(concatenate_if("padEndA_", si.a), + si.aZeroPad.padEnd); + } + if(si.bZeroPad.valid()) + { + rv.args.append(concatenate_if("padStartB_", si.b), + si.bZeroPad.padStart); + rv.args.append(concatenate_if("padEndB_", si.b), + si.bZeroPad.padEnd); + } + } + + rv.args.append("staggerUIter", staggerUIter(problem, inputs, hardware)); + + rv.args.append("problemNumGroupTiles0", problemNumGroupTiles0); + rv.args.append("problemNumGroupTiles1", problemNumGroupTiles1); + + if(!isSourceKernel()) + { + uint32_t numFullBlocks = problemNumGroupTiles1; + uint32_t wgmRemainder1 = 0; + uint32_t magicNumberWgmRemainder1 = 0; + + // conditional args, aligned with KernelWriterAssembly.py + if(sizeMapping.persistentKernel != 0) + { + uint32_t magicShift; + rv.args.append("magicNumberProblemNumGroupTiles0", + magicNumber(2, problemNumGroupTiles0, &magicShift)); + rv.args.append("magicShiftProblemNumGroupTiles0", magicShift); + rv.args.append("gridNumWorkGroups0", rv.numWorkGroups.x); + } + + if(sizeMapping.persistentKernelAlongBatch) + { + uint32_t numGroupTiles0x1 = problemNumGroupTiles0 * problemNumGroupTiles1; + uint32_t magicShift; + + rv.args.append("problemNumGroupTiles2", problemNumGroupTiles2); + rv.args.append("magicNumberProblemNumGroupTiles0By1", + magicNumber(2, numGroupTiles0x1, &magicShift)); + rv.args.append("magicShiftProblemNumGroupTiles0By1", magicShift); + } + + if(sizeMapping.workGroupMapping != 0) + { + numFullBlocks = problemNumGroupTiles1 / sizeMapping.workGroupMapping; + wgmRemainder1 = problemNumGroupTiles1 % sizeMapping.workGroupMapping; + if(wgmRemainder1 == 0) + wgmRemainder1 = sizeMapping.workGroupMapping; + magicNumberWgmRemainder1 = smallMagicNumber(wgmRemainder1); + } + + rv.args.append("numFullBlocks", numFullBlocks); + rv.args.append("wgmRemainder1", wgmRemainder1); + rv.args.append("magicNumberWgmRemainder1", magicNumberWgmRemainder1); + } + + rv.args.append("offsetD", d.offset()); + rv.args.append("offsetC", c.offset()); + rv.args.append("offsetA", a.offset()); + rv.args.append("offsetB", b.offset()); + + if(!isSourceKernel()) + { + rv.args.append("pad", 0); + } + + return rv; + } + + bool ContractionSolution::isSourceKernel() const + { + return sizeMapping.sourceKernel; + } + + template + KernelInvocation ContractionSolution::generateBetaOnlyCall(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const + { + TensorDescriptor const& c = problem.c(); + TensorDescriptor const& d = problem.d(); + + KernelInvocation rv; + + rv.args = KernelArguments(T_Debug); + + rv.args.reserve(512, 64); + + rv.kernelName = betaOnlyKernelName(problem, inputs, hardware); + + rv.workGroupSize.x = 256; + rv.workGroupSize.y = 1; + rv.workGroupSize.z = 1; + + size_t wiX = 1; + size_t wiY = 1; + size_t wiZ = 1; + for(size_t i = 0; i < problem.freeIndicesA().size(); i++) + wiX *= problem.freeSizeA(i); + for(size_t i = 0; i < problem.freeIndicesB().size(); i++) + wiY *= problem.freeSizeB(i); + for(size_t i = 0; i < problem.batchIndices().size(); i++) + wiZ *= problem.batchSize(i); + + rv.numWorkGroups.x = CeilDivide(wiX * wiY * wiZ, rv.workGroupSize.x); + rv.numWorkGroups.y = 1; + rv.numWorkGroups.z = 1; + + rv.numWorkItems.x = rv.workGroupSize.x * rv.numWorkGroups.x; + rv.numWorkItems.y = rv.workGroupSize.y * rv.numWorkGroups.y; + rv.numWorkItems.z = rv.workGroupSize.z * rv.numWorkGroups.z; + + if(sizeMapping.globalAccumulation) + rv.args.append("WS", inputs.ws); + else if(problemType.stridedBatched) + rv.args.append("D", inputs.d); + else + rv.args.append("batchD", inputs.batchD); + + if(problemType.stridedBatched) + rv.args.append("C", inputs.c); + else + rv.args.append("batchC", inputs.batchC); + + if(problemType.useBeta && sizeMapping.globalAccumulation == 0) + { + rv.args.append("bias", inputs.bias); + } + + if(sizeMapping.globalAccumulation) + { + size_t stride = d.sizes()[0]; + for(size_t i = 1; i < d.dimensions(); i++) + { + rv.args.append(concatenate_if("strideW", i), + d.sizes()[i] == 1 ? 0 : stride); + stride *= d.sizes()[i]; + } + } + else + { + for(size_t i = 1; i < d.dimensions(); i++) + rv.args.append(concatenate_if("strideD", i), + d.sizes()[i] == 1 ? 0 : d.strides()[i]); + } + + for(size_t i = 1; i < c.dimensions(); i++) + rv.args.append(concatenate_if("strideC", i), + c.sizes()[i] == 1 ? 0 : c.strides()[i]); + + int idx = 0; + for(auto size : problem.d().sizes()) + { + rv.args.append(concatenate_if("size_", idx), size); + idx++; + } + + rv.args.append("offsetD", d.offset()); + rv.args.append("offsetC", c.offset()); + + rv.args.append("beta", inputs.beta); + + return rv; + } + + template + std::string ContractionSolution::betaOnlyKernelName(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const + { + std::string name = concatenate( + "C", problem.cNames(), "_", TypeInfo::Abbrev()); + + if(!problemType.stridedBatched) + { + name += "_GB"; + } + + if(problemType.useBias && (sizeMapping.globalAccumulation == 0)) + { + name += "_Bias"; + } + + if(sizeMapping.globalAccumulation) + { + name += "_GA"; + } + + return name; + } + + template + KernelInvocation ContractionSolution::generateOutputConversionCall( + Problem const& problem, TypedInputs const& inputs, Hardware const& hardware) const + { + TensorDescriptor const& c = problem.c(); + TensorDescriptor const& d = problem.d(); + + KernelInvocation rv; + + rv.args = KernelArguments(T_Debug); + + rv.args.reserve(512, 64); + + rv.kernelName = outputConversionKernelName(problem, inputs, hardware); + + rv.workGroupSize.x = 256; + rv.workGroupSize.y = 1; + rv.workGroupSize.z = 1; + + size_t wiX = 1; + size_t wiY = 1; + size_t wiZ = 1; + for(size_t i = 0; i < problem.freeIndicesA().size(); i++) + wiX *= problem.freeSizeA(i); + for(size_t i = 0; i < problem.freeIndicesB().size(); i++) + wiY *= problem.freeSizeB(i); + for(size_t i = 0; i < problem.batchIndices().size(); i++) + wiZ *= problem.batchSize(i); + + rv.numWorkGroups.x = CeilDivide(wiX * wiY * wiZ, rv.workGroupSize.x); + rv.numWorkGroups.y = 1; + rv.numWorkGroups.z = 1; + + rv.numWorkItems.x = rv.workGroupSize.x * rv.numWorkGroups.x; + rv.numWorkItems.y = rv.workGroupSize.y * rv.numWorkGroups.y; + rv.numWorkItems.z = rv.workGroupSize.z * rv.numWorkGroups.z; + + if(problemType.stridedBatched) + rv.args.append("D", inputs.d); + else + rv.args.append("batchD", inputs.batchD); + + rv.args.append("WS", inputs.ws); + + if(problemType.stridedBatched) + rv.args.append("C", inputs.c); + else + rv.args.append("batchC", inputs.batchC); + + if(problemType.useBias) + { + rv.args.append("bias", inputs.bias); + } + + if(sizeMapping.globalAccumulation == 2) + rv.args.append("alpha", inputs.alpha); + else + rv.args.append("alpha", 1.0f); + + if(sizeMapping.globalAccumulation == 2 and problemType.useBeta) + rv.args.append("beta", inputs.beta); + else + rv.args.append("beta", 0.0f); + + if((problem.activationType() != ActivationType::None) && sizeMapping.activationFused) + { + for(int i = 0; i < inputs.activationArgs.size(); i++) + { + std::string name = "activation_" + std::to_string(i); + if(problem.activationHPA()) // Same as hpa type. + { + rv.args.append( + name.c_str(), + static_cast(inputs.activationArgs[i])); + } + else + { + constexpr bool needCast = std::is_same(); + using castT = std::conditional_t; + rv.args.append(name.c_str(), + static_cast(inputs.activationArgs[i])); + } + } + if(problem.activationType() == ActivationType::All) + { + rv.args.append("activationType", + static_cast(problem.activationEnumArg())); + } + } + + for(size_t i = 1; i < d.dimensions(); i++) + rv.args.append(concatenate_if("strideD", i), d.strides()[i]); + + uint32_t wsStride = d.sizes()[0]; + for(size_t i = 1; i < d.dimensions(); i++) + { + rv.args.append(concatenate_if("strideW", i), wsStride); + wsStride *= d.sizes()[i]; + } + + for(size_t i = 1; i < c.dimensions(); i++) + rv.args.append(concatenate_if("strideC", i), c.strides()[i]); + + int idx = 0; + for(auto size : problem.d().sizes()) + { + rv.args.append(concatenate_if("size_", idx), size); + idx++; + } + + rv.args.append("offsetD", d.offset()); + rv.args.append("offsetC", c.offset()); + + if(sizeMapping.globalAccumulation == 1) + rv.args.append("gsu", 1); + else + rv.args.append("gsu", sizeMapping.globalSplitU); + + return rv; + } + + template + std::string ContractionSolution::outputConversionKernelName(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const + { + std::string name = concatenate( + "C", problem.cNames(), "_", TypeInfo::Abbrev()); + + if(!problemType.stridedBatched) + { + name += "_GB"; + } + + if(problemType.useBias) + { + name += "_Bias"; + } + + if(problem.activationType() != ActivationType::None) + { + if(problem.activationType() == ActivationType::All) + name += "_A"; + else + { + std::string actName = ToString(problem.activationType()); + std::transform(actName.begin(), actName.end(), actName.begin(), ::toupper); + name += actName; + } + } + if(problem.activationHPA()) + { + name += "h"; + } + + name += "_PostGSU"; + + return name; + } + + template + KernelInvocation ContractionSolution::generateActivationOnlyCall(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const + { + TensorDescriptor const& d = problem.d(); + + KernelInvocation rv; + + rv.args = KernelArguments(T_Debug); + + rv.args.reserve(512, 64); + + rv.kernelName = activationOnlyKernelName(problem, inputs, hardware); + + rv.workGroupSize.x = 256; + rv.workGroupSize.y = 1; + rv.workGroupSize.z = 1; + + size_t wiX = 1; + size_t wiY = 1; + size_t wiZ = 1; + for(size_t i = 0; i < problem.freeIndicesA().size(); i++) + wiX *= problem.freeSizeA(i); + for(size_t i = 0; i < problem.freeIndicesB().size(); i++) + wiY *= problem.freeSizeB(i); + for(size_t i = 0; i < problem.batchIndices().size(); i++) + wiZ *= problem.batchSize(i); + + rv.numWorkGroups.x = CeilDivide(wiX * wiY * wiZ, rv.workGroupSize.x); + rv.numWorkGroups.y = 1; + rv.numWorkGroups.z = 1; + + rv.numWorkItems.x = rv.workGroupSize.x * rv.numWorkGroups.x; + rv.numWorkItems.y = rv.workGroupSize.y * rv.numWorkGroups.y; + rv.numWorkItems.z = rv.workGroupSize.z * rv.numWorkGroups.z; + + if(problemType.stridedBatched) + rv.args.append("D", inputs.d); + else + rv.args.append("batchD", inputs.batchD); + + if(problem.activationType() != ActivationType::None) + { + if(problem.activationType() == ActivationType::All) + { + rv.args.append("activationType", + static_cast(problem.activationEnumArg())); + } + for(int i = 0; i < inputs.activationArgs.size(); i++) + { + std::string name = "activation_" + std::to_string(i); + if(problem.activationHPA()) // Same as hpa type. + { + rv.args.append( + name.c_str(), + static_cast(inputs.activationArgs[i])); + } + else + { + constexpr bool needCast = std::is_same(); + using castT = std::conditional_t; + rv.args.append(name.c_str(), + static_cast(inputs.activationArgs[i])); + } + } + } + + if(sizeMapping.globalAccumulation) + { + size_t stride = d.sizes()[0]; + for(size_t i = 1; i < d.dimensions(); i++) + { + rv.args.append(concatenate_if("strideW", i), + d.sizes()[i] == 1 ? 0 : stride); + stride *= d.sizes()[i]; + } + } + else + { + for(size_t i = 1; i < d.dimensions(); i++) + rv.args.append(concatenate_if("strideD", i), + d.sizes()[i] == 1 ? 0 : d.strides()[i]); + } + + int idx = 0; + for(auto size : problem.d().sizes()) + { + rv.args.append(concatenate_if("size_", idx), size); + idx++; + } + + rv.args.append("offsetD", d.offset()); + + return rv; + } + + template + std::string ContractionSolution::activationOnlyKernelName(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const + { + std::string name = concatenate( + "D", problem.cNames(), "_", TypeInfo::Abbrev()); + if(problem.activationType() != ActivationType::None) + { + if(problem.activationType() == ActivationType::All) + name += "_A"; + else + { + std::string actName = ToString(problem.activationType()); + std::transform(actName.begin(), actName.end(), actName.begin(), ::toupper); + name += actName; + } + } + if(problem.activationHPA()) + { + name += "h"; + } + + return name; + } + + template + std::vector ContractionSolution::solveTyped(Problem const& problem, + TypedInputs const& inputs, + Hardware const& hardware) const + { + bool debug = Debug::Instance().printKernelArguments() || this->kernelArgsLog; + + int boundSize = 1; + for(size_t i = 0; i < problem.boundIndices().size(); i++) + boundSize *= problem.boundSize(i); + + // Check for nullptrs if alpha is non-zero. + if(((inputs.alpha != static_cast(0)) && (boundSize != 0)) + && ((problem.stridedBatched() && (inputs.a == nullptr || inputs.b == nullptr)) + || (!problem.stridedBatched() + && (inputs.batchA == nullptr || inputs.batchB == nullptr)))) + { + std::string matrixID = inputs.a == nullptr ? "A" : "B"; + std::string msg = std::string("Unsupported nullptr for ") + matrixID + + std::string(" when (Alpha !=0) && (K != 0)\n"); + throw std::runtime_error(msg.c_str()); + } + + // Check if alpha matches problem definition + if(problem.alphaRestriction() != ScalarValue::Any + && problem.alphaRestriction() != toScalarValueEnum(inputs.alpha)) + { + std::stringstream inputValue; + inputValue << inputs.alpha; + std::string msg = std::string("Alpha value ") + inputValue.str() + + std::string(" doesn't match that set in problem: ") + + ToString(problem.alphaRestriction()); + throw std::runtime_error(msg.c_str()); + } + + // Check if beta matches problem definition + if(problem.betaRestriction() != ScalarValue::Any + && problem.betaRestriction() != toScalarValueEnum(inputs.beta)) + { + std::stringstream inputValue; + inputValue << inputs.beta; + std::string msg = std::string("Beta value ") + inputValue.str() + + std::string(" doesn't match that set in problem: ") + + ToString(problem.betaRestriction()); + throw std::runtime_error(msg.c_str()); + } + + if(problem.cEqualsD() && inputs.c != inputs.d) + throw std::runtime_error( + "ContractionProblem has cEqualsD set, but pointers for c and d are not equal"); + + std::vector rv; + + if(sizeMapping.globalSplitU > 1 && sizeMapping.globalAccumulation != 2) + { + if(debug) + rv.push_back(generateBetaOnlyCall(problem, inputs, hardware)); + else + rv.push_back(generateBetaOnlyCall(problem, inputs, hardware)); + } + + if(debug) + rv.push_back(generateSingleCall(problem, inputs, hardware)); + else + rv.push_back(generateSingleCall(problem, inputs, hardware)); + + if(sizeMapping.globalAccumulation) + { + if(debug) + rv.push_back( + generateOutputConversionCall(problem, inputs, hardware)); + else + rv.push_back( + generateOutputConversionCall(problem, inputs, hardware)); + } + + if(((sizeMapping.globalSplitU > 1 && (!sizeMapping.globalAccumulation)) + || (!sizeMapping.activationFused)) + && (problem.activationType() != ActivationType::None)) + { + if(debug) + rv.push_back( + generateActivationOnlyCall(problem, inputs, hardware)); + else + rv.push_back( + generateActivationOnlyCall(problem, inputs, hardware)); + } + + return rv; + } + + std::vector + ContractionSolution::solve(ContractionSolution::Problem const& problem, + ContractionSolution::Inputs const& inputs, + Hardware const& hardware) const + { + if(Debug::Instance().printWinningKernelName()) + std::cout << "Running kernel: " << this->KernelName() << std::endl; + + // retreive alpha/beta type set via setAlpha/BetaType() + auto alphaType = problem.alphaType(); + auto betaType = problem.betaType(); + + // TODO: Some gtests are passing the "problem" without actually defining the + // alpha/beta type (alphaType and betaType remain None). + // Until we fix those gtests, we need to keep this condition to adjust the missing + // alpha/beta data types. + if(alphaType == DataType::None) + { + alphaType + = problemType.aType == DataType::BFloat16 ? DataType::Float : problemType.dType; + } + if(betaType == DataType::None) + { + betaType = alphaType; + } + + auto contractionInputsTypeId = ContractionInputs::TypeId(problemType.aType, + problemType.bType, + problemType.cType, + problemType.dType, + alphaType, + betaType); + + switch(contractionInputsTypeId) + { + case ContractionInputs_S_S_S::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } + case ContractionInputs_D_D_D::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } + case ContractionInputs_C_C_C::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } + case ContractionInputs_Z_Z_Z::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } +#ifdef TENSILE_USE_HALF + case ContractionInputs_H_H_H::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } + case ContractionInputs_H_H_S::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } + case ContractionInputs_H_S_S::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } +#endif // TENSILE_USE_HALF + case ContractionInputs_I8x4_I32_I32::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } + case ContractionInputs_I32_I32_I32::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } + case ContractionInputs_I8_I8_I32::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } + case ContractionInputs_I8_I32_I32::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } +#ifdef TENSILE_USE_BF16 + case ContractionInputs_B_B_S::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } + case ContractionInputs_B_S_S::TypeId(): + { + auto const& typedInputs = dynamic_cast(inputs); + return solveTyped(problem, typedInputs, hardware); + } +#endif // TENSILE_USE_BF16 + + default:; + } + throw std::runtime_error("Data type not implemented."); + } + + ContractionSolution::StaticPerformanceModel + ContractionSolution::staticPerformanceModel(double M, + double N, + double K, + double NumBatches, + double MT0, + double MT1, + double NumCUs, + double TotalGranularity, + int GlobalSplitU) const + { + StaticPerformanceModel spm; + + int beta = (int)problemType.useBeta; + int betaReads = 0, betaWrites = 0; + if(GlobalSplitU == 1) + { + if(beta != 0.0) + betaReads = 1.0; + } + else + { + if(beta == 0) + betaWrites = 1; // zero output + else if(beta != 1.0) // if 1.0, just atomic update output + { + // if not 1.0, read, scale, write, then atomic update in kernel + betaReads = 1; // initial read for scale + betaWrites = 1; // writeback after scale + } + } + + auto aInfo = DataTypeInfo::Get(problemType.aType); + auto bInfo = DataTypeInfo::Get(problemType.bType); + auto cInfo = DataTypeInfo::Get(problemType.cType); + auto dInfo = DataTypeInfo::Get(problemType.dType); + + spm.memReadBytesA = (NumBatches * M * N * K) / MT1 * aInfo.elementSize; + spm.memReadBytesB = (NumBatches * M * N * K) / MT0 * bInfo.elementSize; + spm.memReadBytesC = (NumBatches * M * N) * betaReads * cInfo.elementSize; + + if(GlobalSplitU == 1) + spm.memWriteBytesD = (NumBatches * M * N) * (1 + betaWrites) * dInfo.elementSize; + else + { + bool hardwareAtomic = false; // TODO-model + double atomicOperations = hardwareAtomic ? 2 : 3; // read-mod-write or cas //TODO-model + double atomicCollisions = 1.0; // TODO-could be based on K, GSU + spm.memWriteBytesD = (NumBatches * M * N) + * (betaWrites + atomicOperations * atomicCollisions) + * dInfo.elementSize; + } + spm.memReadBytes = spm.memReadBytesA + spm.memReadBytesB + spm.memReadBytesC; + spm.memGlobalReads = spm.memReadBytesA / aInfo.elementSize + + spm.memReadBytesB / bInfo.elementSize + + spm.memReadBytesC / cInfo.elementSize; + spm.memGlobalWrites = spm.memWriteBytesD / dInfo.elementSize; + + return spm; + } + + size_t ContractionSolution::requiredWorkspaceSize(Problem const& problem) const + { + size_t size = 0; + + size += problem.d().totalLogicalElements() * sizeMapping.workspaceSizePerElemC; + + return size; + } + + ContractionSolution::Granularities ContractionSolution::computeGranularities( + Hardware const& hardware, double M, double N, double K, double NumBatches) const + { + ContractionSolution::Granularities granularities; + + double MT0 = sizeMapping.macroTile.x; + double MT1 = sizeMapping.macroTile.y; + + AMDGPU const* pAMDGPU = dynamic_cast(&hardware); + assert(pAMDGPU); + + double NumCUs = pAMDGPU->computeUnitCount; + double wavefrontSize = pAMDGPU->wavefrontSize; + double simdPerCu = pAMDGPU->simdPerCu; + + double GlobalSplitU = sizeMapping.globalSplitU; + double LocalSplitU = sizeMapping.workGroupSize.z; + + granularities.MT0 = MT0; + granularities.MT1 = MT1; + granularities.GSU = GlobalSplitU; + granularities.LSU = LocalSplitU; + granularities.CUs = NumCUs; + + granularities.numTiles0 = M / MT0; + granularities.numTiles1 = N / MT1; + + granularities.tile0Granularity = granularities.numTiles0 / ceil(granularities.numTiles0); + granularities.tile1Granularity = granularities.numTiles1 / ceil(granularities.numTiles1); + + granularities.tilesPerCu + = (NumBatches * ceil(granularities.numTiles0) * ceil(granularities.numTiles1)) + / (NumCUs / GlobalSplitU / LocalSplitU); + + granularities.totalTiles = ceil(granularities.numTiles0) * ceil(granularities.numTiles1); + granularities.natTilesPerCu = NumBatches * granularities.totalTiles / NumCUs; + granularities.suTilesPerCu = (granularities.totalTiles * GlobalSplitU) / NumCUs; + granularities.suCuGranularity + = granularities.suTilesPerCu / ceil(granularities.suTilesPerCu); + + granularities.waveGranularity = std::min( + 1.00, + static_cast(floor(granularities.tilesPerCu + 1.0) * sizeMapping.workGroupSize.x + * sizeMapping.workGroupSize.y * sizeMapping.workGroupSize.z) + / pAMDGPU->wavefrontSize / pAMDGPU->simdPerCu); + + granularities.waves + = ceil((sizeMapping.workGroupSize.x * sizeMapping.workGroupSize.y) / wavefrontSize); + + granularities.suWavesPerSimdx2 + = (granularities.suTilesPerCu * granularities.waves) / (2 * simdPerCu); + granularities.suWaveGranularity + = granularities.suWavesPerSimdx2 * ceil(granularities.suWavesPerSimdx2); + + double nat_tiles_per_cu + = NumBatches * ceil(granularities.numTiles0) * ceil(granularities.numTiles1) / NumCUs; + granularities.natCuGranularity = ceil(nat_tiles_per_cu) * ceil(nat_tiles_per_cu) / NumCUs; + + granularities.cuGranularity = granularities.tilesPerCu / ceil(granularities.tilesPerCu); + + granularities.totalGranularity + = granularities.tile0Granularity * granularities.tile1Granularity + * granularities.cuGranularity * granularities.waveGranularity; + + granularities.totalTileAwareGranularity + = granularities.tile0Granularity * granularities.tile1Granularity + * granularities.suCuGranularity * granularities.suWaveGranularity; + + return granularities; + } + + ContractionSolution::ProjectedPerformance + ContractionSolution::projectedPerformance(Problem const& problem, + Hardware const& hardware) const + { + ProjectedPerformance pp; + + double M = 1.0, N = 1.0; + if(problem.freeIndicesA().size() > 1 || sizeMapping.packBatchDims & 0x1) + { + std::vector packedIndices + = generatePackedIndicesA(problem, sizeMapping.packBatchDims); + for(auto pi = packedIndices.begin(); pi != packedIndices.end(); pi++) + M *= problem.a().sizes()[*pi]; + } + else + M = problem.freeSizeA(0); + + if(problem.freeIndicesB().size() > 1 || sizeMapping.packBatchDims & 0x2) + { + std::vector packedIndices + = generatePackedIndicesB(problem, sizeMapping.packBatchDims); + for(auto pi = packedIndices.begin(); pi != packedIndices.end(); pi++) + N *= problem.b().sizes()[*pi]; + } + else + N = problem.freeSizeB(0); + + double NumBatches = 1; + if(sizeMapping.packBatchDims == 0) + { + for(size_t i = 0; i < problem.batchIndices().size(); i++) + NumBatches *= problem.batchSize(i); + } + double K = problem.boundSize(0); // TODO - fix for multiple summations + + pp.granularities = ContractionSolution::computeGranularities(hardware, M, N, K, NumBatches); + + auto it = ideals.begin(); + + int closestKMeasure = std::numeric_limits::max(); + double closestKPerformance = 0.0; + + while(it != ideals.end()) + { + int myK = it->first; + int myMeasure = std::abs(myK - K); + if(myMeasure < closestKMeasure) + { + closestKMeasure = myMeasure; + closestKPerformance = it->second; + } + it++; + } + + double MT0 = pp.granularities.MT0; + double MT1 = pp.granularities.MT1; + double NumCUs = pp.granularities.CUs; + + double GlobalSplitU = pp.granularities.GSU; + double IdealGranularityPerf = closestKPerformance; + + pp.staticModel = staticPerformanceModel( + M, N, K, NumBatches, MT0, MT1, NumCUs, pp.granularities.totalGranularity, GlobalSplitU); + + pp.speedGFlops = IdealGranularityPerf * pp.granularities.totalGranularity; + pp.CUs = NumCUs; + + return pp; + } + + ContractionSolution::TAMetricProblemScore ContractionSolution::computeProblemScore( + Hardware const& hardware, double M, double N, double K, double NumBatches) const + { + ContractionSolution::TAMetricProblemScore pp; + pp.granularites = ContractionSolution::computeGranularities(hardware, M, N, K, NumBatches); + + pp.M = M; + pp.N = N; + pp.K = K; + + double slope = linearModel.slope; + double intercept = linearModel.intercept; + double perf_max = linearModel.max; + + double sum_value = K; + double sum_perf0 = sum_value / (intercept + (slope * sum_value)); + pp.summationPerformance = 1000.0 * sum_perf0 / perf_max; + + return pp; + } + + double ContractionSolution::computeTileAwareMetric( + ContractionSolution::TAMetricProblemScore pp, + ContractionSolution::TAMetricProblemScore ppReference) const + { + double tile0GranularityDim = abs(log(ppReference.granularites.tile0Granularity) + - log(pp.granularites.tile0Granularity)); + double metric = tile0GranularityDim; + + double tile1GranularityDim = abs(log(ppReference.granularites.tile1Granularity) + - log(pp.granularites.tile1Granularity)); + metric += tile1GranularityDim; + + double natCuGranularityDim = abs(log(ppReference.granularites.natCuGranularity) + - log(pp.granularites.natCuGranularity)); + metric += natCuGranularityDim; + + double suCuGranularityDim = abs(log(ppReference.granularites.suCuGranularity) + - log(pp.granularites.suCuGranularity)); + metric += suCuGranularityDim; + + double suWaveGranularityDim = abs(log(ppReference.granularites.suWaveGranularity) + - log(pp.granularites.suWaveGranularity)); + metric += suWaveGranularityDim; + + double natTilesPerCuDim + = abs(log(ppReference.granularites.natTilesPerCu) - log(pp.granularites.natTilesPerCu)); + metric += natTilesPerCuDim; + + double suTilesPerCuDim + = abs(log(ppReference.granularites.suTilesPerCu) - log(pp.granularites.suTilesPerCu)); + metric += suTilesPerCuDim; + + double summationPerformanceDim + = abs(ppReference.summationPerformance - pp.summationPerformance); + metric += summationPerformanceDim; + + return metric; + } + + double ContractionSolution::computeTAMScore(Problem const& problem, + Hardware const& hardware, + double model_M, + double model_N, + double model_K, + double model_NumBatches) const + { + double M = 1.0, N = 1.0; + if(problem.freeIndicesA().size() > 1 || sizeMapping.packBatchDims & 0x1) + { + std::vector packedIndices + = generatePackedIndicesA(problem, sizeMapping.packBatchDims); + for(auto pi = packedIndices.begin(); pi != packedIndices.end(); pi++) + M *= problem.a().sizes()[*pi]; + } + else + M = problem.freeSizeA(0); + + if(problem.freeIndicesB().size() > 1 || sizeMapping.packBatchDims & 0x2) + { + std::vector packedIndices + = generatePackedIndicesB(problem, sizeMapping.packBatchDims); + for(auto pi = packedIndices.begin(); pi != packedIndices.end(); pi++) + N *= problem.b().sizes()[*pi]; + } + else + N = problem.freeSizeB(0); + + double NumBatches = 1; + if(sizeMapping.packBatchDims == 0) + { + for(size_t i = 0; i < problem.batchIndices().size(); i++) + NumBatches *= problem.batchSize(i); + } + double K = problem.boundSize(0); // TODO - fix for multiple summations + + ContractionSolution::TAMetricProblemScore pp + = computeProblemScore(hardware, M, N, K, NumBatches); + + ContractionSolution::TAMetricProblemScore ppReference + = computeProblemScore(hardware, model_M, model_N, model_K, model_NumBatches); + + double distance = computeTileAwareMetric(pp, ppReference); + + return distance; + } + + std::ostream& operator<<(std::ostream& stream, + ContractionSolution::StaticPerformanceModel const& spm) + { + return stream << " memReadBytesA=" << spm.memReadBytesA + << " memReadBytesB=" << spm.memReadBytesB + << " memReadBytesC=" << spm.memReadBytesC + << " memWriteBytesD=" << spm.memWriteBytesD; + } + + std::ostream& operator<<(std::ostream& stream, + ContractionSolution::ProjectedPerformance const& pp) + { + return stream << " numTiles0=" << pp.granularities.numTiles0 + << " numTiles1=" << pp.granularities.numTiles1 + << " tilesPerCu=" << pp.granularities.tilesPerCu + + << " totalGranularity=" << pp.granularities.totalGranularity + << " tile0Granularity=" << pp.granularities.tile0Granularity + << " tile1Granularity=" << pp.granularities.tile1Granularity + << " cuGranularity=" << pp.granularities.cuGranularity + << " waveGranularity=" << pp.granularities.waveGranularity + + << " speedGFlops=" << pp.speedGFlops + + << " staticModel=[ " << pp.staticModel << " ]"; + } + + std::ostream& operator<<(std::ostream& stream, BufferLoadCheckPacket const& st) + { + return stream << " shiftPtrElemA=" << st.shiftPtrElemA + << " shiftPtrElemB=" << st.shiftPtrElemB << " depthUorMT0=" << st.depthUorMT0 + << " depthUorMT1=" << st.depthUorMT1; + } + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/DataTypes.cpp b/tensilelite/Tensile/Source/lib/source/DataTypes.cpp new file mode 100644 index 0000000000..c6dd94fa7b --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/DataTypes.cpp @@ -0,0 +1,216 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include + +#include + +namespace Tensile +{ + std::map DataTypeInfo::data; + std::map DataTypeInfo::typeNames; + + std::string ToString(DataType d) + { + switch(d) + { + case DataType::Float: + return "Float"; + case DataType::Double: + return "Double"; + case DataType::ComplexFloat: + return "ComplexFloat"; + case DataType::ComplexDouble: + return "ComplexDouble"; + case DataType::Half: + return "Half"; + case DataType::Int8x4: + return "Int8x4"; + case DataType::Int32: + return "Int32"; + case DataType::BFloat16: + return "BFloat16"; + case DataType::Int8: + return "Int8"; + + case DataType::Count:; + } + return "Invalid"; + } + + std::string TypeAbbrev(DataType d) + { + switch(d) + { + case DataType::Float: + return "S"; + case DataType::Double: + return "D"; + case DataType::ComplexFloat: + return "C"; + case DataType::ComplexDouble: + return "Z"; + case DataType::Half: + return "H"; + case DataType::Int8x4: + return "4xi8"; + case DataType::Int32: + return "I"; + case DataType::BFloat16: + return "B"; + case DataType::Int8: + return "I8"; + + case DataType::Count:; + } + return "Invalid"; + } + + template + void DataTypeInfo::registerTypeInfo() + { + using T_Info = TypeInfo; + + DataTypeInfo info; + + info.dataType = T_Info::Enum; + info.name = T_Info::Name(); + info.abbrev = T_Info::Abbrev(); + + info.packing = T_Info::Packing; + info.elementSize = T_Info::ElementSize; + info.segmentSize = T_Info::SegmentSize; + + info.isComplex = T_Info::IsComplex; + info.isIntegral = T_Info::IsIntegral; + + addInfoObject(info); + } + + void DataTypeInfo::registerAllTypeInfo() + { + registerTypeInfo(); + registerTypeInfo(); + registerTypeInfo>(); + registerTypeInfo>(); + registerTypeInfo(); + registerTypeInfo(); + registerTypeInfo(); + registerTypeInfo(); + registerTypeInfo(); + } + + void DataTypeInfo::registerAllTypeInfoOnce() + { + static int call_once = (registerAllTypeInfo(), 0); + + // Use the variable to quiet the compiler. + if(call_once) + return; + } + + void DataTypeInfo::addInfoObject(DataTypeInfo const& info) + { + data[info.dataType] = info; + typeNames[info.name] = info.dataType; + } + + DataTypeInfo const& DataTypeInfo::Get(int index) + { + return Get(static_cast(index)); + } + + DataTypeInfo const& DataTypeInfo::Get(DataType t) + { + registerAllTypeInfoOnce(); + + auto iter = data.find(t); + if(iter == data.end()) + throw std::runtime_error(concatenate("Invalid data type: ", static_cast(t))); + + return iter->second; + } + + DataTypeInfo const& DataTypeInfo::Get(std::string const& str) + { + registerAllTypeInfoOnce(); + + auto iter = typeNames.find(str); + if(iter == typeNames.end()) + throw std::runtime_error(concatenate("Invalid data type: ", str)); + + return Get(iter->second); + } + + std::ostream& operator<<(std::ostream& stream, const DataType& t) + { + return stream << ToString(t); + } + + std::istream& operator>>(std::istream& stream, DataType& t) + { + std::string strValue; + stream >> strValue; + +#if 1 + t = DataTypeInfo::Get(strValue).dataType; + +#else + + if(strValue == ToString(DataType::Float)) + t = DataType::Float; + else if(strValue == ToString(DataType::Double)) + t = DataType::Double; + else if(strValue == ToString(DataType::ComplexFloat)) + t = DataType::ComplexFloat; + else if(strValue == ToString(DataType::ComplexDouble)) + t = DataType::ComplexDouble; + else if(strValue == ToString(DataType::Half)) + t = DataType::Half; + else if(strValue == ToString(DataType::Int8x4)) + t = DataType::Int8x4; + else if(strValue == ToString(DataType::Int32)) + t = DataType::Int32; + else if(strValue == ToString(DataType::Int8)) + t = DataType::Int8; + else if(std::all_of(strValue.begin(), strValue.end(), isdigit)) + { + int value = atoi(strValue.c_str()); + if(value >= 0 && value < static_cast(DataType::Count)) + t = static_cast(value); + else + throw std::runtime_error(concatenate("Can't convert ", strValue, " to DataType.")); + } + else + { + throw std::runtime_error(concatenate("Can't convert ", strValue, " to DataType.")); + } +#endif + + return stream; + } +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/Debug.cpp b/tensilelite/Tensile/Source/lib/source/Debug.cpp new file mode 100644 index 0000000000..0ce5c8d472 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/Debug.cpp @@ -0,0 +1,166 @@ +/** + * Copyright 2019-2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include + +#ifndef DEBUG_SM +#define DEBUG_SM 0 +#endif + +#ifndef DEBUG_SM2 +#define DEBUG_SM2 0 +#endif + +namespace Tensile +{ + std::once_flag debug_init; + + bool Debug::printPropertyEvaluation() const + { + return m_value & (0x2 | 0x4); + } + + bool Debug::printDeviceSelection() const + { + return m_value & 0x8; + } + + bool Debug::printPredicateEvaluation() const + { + return m_value & 0x10; + } + + bool Debug::printCodeObjectInfo() const + { + return m_value & 0x20; + } + + bool Debug::printKernelArguments() const + { + return m_value & 0x40; + } + + bool Debug::printTensorInfo() const + { + return m_value & 0x80; + } + + bool Debug::printConvolutionReference1() const + { + return m_value & 0x100; + } + bool Debug::printConvolutionReference2() const + { + return m_value & 0x200; + } + bool Debug::printConvolutionReference3() const + { + return m_value & 0x400; + } + + bool Debug::printTensorModeHex() const + { + return m_value & 0x800; + } + + bool Debug::printDataInit() const + { + return m_value & 0x1000; + } + + bool Debug::printLibraryVersion() const + { + return m_value & 0x2000; + } + + bool Debug::printLookupEfficiency() const + { + return m_value & 0x4000; + } + + bool Debug::printWinningKernelName() const + { + return m_value & 0x8000; + } + + bool Debug::printSolutionSelectionTime() const + { + return m_value & 0x10000; + } + + bool Debug::naivePropertySearch() const + { + return m_naivePropertySearch; + } + + bool Debug::skipKernelLaunch() const + { + return m_value2 & 0x1; + } + + bool Debug::enableDebugSelection() const + { + return m_debugSelection; + } + + std::string Debug::getMetric() const + { + return m_metric; + } + + int Debug::getSolutionIndex() const + { + return m_solution_index; + } + + Debug::Debug() + : m_value(DEBUG_SM) + , m_value2(DEBUG_SM2) + { + const char* db = std::getenv("TENSILE_DB"); + if(db) + m_value = strtol(db, nullptr, 0); + + const char* db2 = std::getenv("TENSILE_DB2"); + if(db2) + m_value2 = strtol(db2, nullptr, 0); + + const char* naive = std::getenv("TENSILE_NAIVE_SEARCH"); + if(naive) + m_naivePropertySearch = strtol(naive, nullptr, 0) != 0; + + const char* db_select = std::getenv("TENSILE_TAM_SELECTION_ENABLE"); + if(db_select) + m_debugSelection = strtol(db_select, nullptr, 0) != 0; + + const char* solution_index = std::getenv("TENSILE_SOLUTION_INDEX"); + if(solution_index) + m_solution_index = strtol(solution_index, nullptr, 0); + + const char* tensile_metric = std::getenv("TENSILE_METRIC"); + if(tensile_metric) + m_metric = tensile_metric; + } + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/EmbeddedData.cpp b/tensilelite/Tensile/Source/lib/source/EmbeddedData.cpp new file mode 100644 index 0000000000..53cd67be19 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/EmbeddedData.cpp @@ -0,0 +1,27 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include diff --git a/tensilelite/Tensile/Source/lib/source/EmbeddedLibrary.cpp b/tensilelite/Tensile/Source/lib/source/EmbeddedLibrary.cpp new file mode 100644 index 0000000000..bac58deb36 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/EmbeddedLibrary.cpp @@ -0,0 +1,53 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#ifdef TENSILE_DEFAULT_SERIALIZATION + +#include + +#include +#include + +namespace Tensile +{ + template + std::shared_ptr> + EmbeddedLibrary::NewLibrary(std::string const& key) + { + auto const& data = EmbeddedData>::Get(key); + if(data.size() == 0) + return nullptr; + if(data.size() > 1) + throw std::runtime_error(concatenate("Expected one data item, found ", data.size())); + + return LoadLibraryData(data[0]); + } + + template std::shared_ptr> + EmbeddedLibrary::NewLibrary(std::string const&); +} // namespace Tensile + +#endif diff --git a/tensilelite/Tensile/Source/lib/source/KernelArguments.cpp b/tensilelite/Tensile/Source/lib/source/KernelArguments.cpp new file mode 100644 index 0000000000..f9161f7a93 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/KernelArguments.cpp @@ -0,0 +1,248 @@ +/** + * Copyright 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include +#include + +namespace Tensile +{ + std::ostream& operator<<(std::ostream& stream, const KernelArguments& t) + { + size_t prevOffset = 0; + for(auto const& name : t.m_names) + { + auto const& iter = t.m_argRecords.find(name); + + if(iter == t.m_argRecords.end()) + throw std::runtime_error("Argument " + name + " not found in record."); + + auto const& record = iter->second; + + size_t offset = std::get(record); + size_t size = std::get(record); + + if(prevOffset != offset) + { + stream << "[" << prevOffset << ".." << offset - 1 << "] " << std::endl; + } + + stream << "[" << offset << ".." << offset + size - 1 << "] " << name << ":"; + + if(std::get(record)) + { + auto oldFill = stream.fill(); + auto oldWidth = stream.width(); + stream << std::hex; + for(size_t i = offset; i < offset + size; i++) + stream << " " << std::setfill('0') << std::setw(2) + << static_cast(t.m_data[i]); + stream << std::dec; + stream.fill(oldFill); + stream.width(oldWidth); + + if(t.m_log) + { + stream << " (" << std::get(record) << ")"; + } + } + else + { + stream << " "; + } + + stream << std::endl; + + prevOffset = offset + size; + } + + return stream; + } + + KernelArguments::KernelArguments(bool log) + : m_log(log) + { + } + + KernelArguments::~KernelArguments() {} + + void KernelArguments::reserve(size_t bytes, size_t count) + { + m_data.reserve(bytes); + m_names.reserve(count); + if(m_log) + m_argRecords.reserve(count); + } + + bool KernelArguments::isFullyBound() const + { + if(!m_log) + return true; + + for(auto const& record : m_argRecords) + { + if(!std::get(record.second)) + return false; + } + + return true; + } + + void const* KernelArguments::data() const + { + if(!isFullyBound()) + throw std::runtime_error("Arguments not fully bound."); + + return reinterpret_cast(m_data.data()); + } + + size_t KernelArguments::size() const + { + return m_data.size(); + } + + KernelArguments::const_iterator::const_iterator(KernelArguments const& args) + : m_args(args) + , m_currentArg(args.m_names.begin()) + { + if(!args.m_log) + { + throw std::runtime_error("KernelArguments::const_iterator requires m_log=true"); + } + assignCurrentArg(); + } + + KernelArguments::const_iterator::const_iterator(KernelArguments const& args, + std::string const& name) + : m_args(args) + , m_currentArg(args.m_names.begin()) + { + if(!args.m_log) + { + throw std::runtime_error("KernelArguments::const_iterator requires m_log=true"); + } + + if(name.empty()) + { + m_currentArg = m_args.m_names.end(); + } + else + { + while(m_currentArg != args.m_names.end() && *m_currentArg != name) + { + m_currentArg++; + } + } + assignCurrentArg(); + } + + auto KernelArguments::const_iterator::operator++() -> const_iterator& + { + if(m_currentArg != m_args.m_names.end()) + { + ++m_currentArg; + assignCurrentArg(); + } + return *this; + } + + auto KernelArguments::const_iterator::operator++(int) -> const_iterator + { + if(m_currentArg != m_args.m_names.end()) + { + auto ret = *this; + ++m_currentArg; + assignCurrentArg(); + return ret; + } + return *this; + } + + bool KernelArguments::const_iterator::operator==(const const_iterator& rhs) const + { + return m_value == rhs.m_value; + } + + bool KernelArguments::const_iterator::operator!=(const const_iterator& rhs) const + { + return m_value != rhs.m_value; + } + + auto KernelArguments::const_iterator::operator*() const + -> decltype(const_iterator::m_value) const& + { + return m_value; + } + + auto KernelArguments::const_iterator::operator->() const + -> decltype(const_iterator::m_value) const* + { + return &m_value; + } + + void KernelArguments::const_iterator::reset() + { + m_currentArg = m_args.m_names.begin(); + assignCurrentArg(); + } + + void KernelArguments::const_iterator::assignCurrentArg() + { + if(m_currentArg != m_args.m_names.end()) + { + auto const& iter = m_args.m_argRecords.find(*m_currentArg); + + if(iter == m_args.m_argRecords.end()) + { + throw std::runtime_error("Argument " + *m_currentArg + " not found in record."); + } + + auto const& record = iter->second; + + if(!m_args.isFullyBound()) + { + throw std::runtime_error("Arguments not fully bound."); + } + + m_value = std::make_pair( + static_cast(m_args.m_data.data() + + std::get(record)), + (size_t)std::get(record)); + } + else + { + m_value = std::make_pair((void const*)nullptr, (size_t)0); + } + } + + auto KernelArguments::begin() const -> const_iterator + { + return const_iterator(*this); + } + + auto KernelArguments::end() const -> const_iterator + { + return const_iterator(*this, ""); + } + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/KernelLanguageTypes.cpp b/tensilelite/Tensile/Source/lib/source/KernelLanguageTypes.cpp new file mode 100644 index 0000000000..10e8cc24e8 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/KernelLanguageTypes.cpp @@ -0,0 +1,159 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include + +#include + +namespace Tensile +{ + std::map KernelLanguageTypeInfo::data; + std::map KernelLanguageTypeInfo::typeNames; + + std::string ToString(KernelLanguage d) + { + switch(d) + { + case KernelLanguage::Any: + return "Any"; + case KernelLanguage::Assembly: + return "Assembly"; + case KernelLanguage::Source: + return "Source"; + + case KernelLanguage::Count: + default:; + } + return "Invalid"; + } + + std::string TypeAbbrev(KernelLanguage d) + { + switch(d) + { + case KernelLanguage::Any: + return "Any"; + case KernelLanguage::Assembly: + return "Asm"; + case KernelLanguage::Source: + return "Src"; + + case KernelLanguage::Count: + default:; + } + return "Invalid"; + } + + template + void KernelLanguageTypeInfo::registerTypeInfo() + { + using T_Info = KernelLanguageInfo; + + KernelLanguageTypeInfo info; + + info.m_kernelLanguage = T_Info::Enum; + info.name = T_Info::Name(); + info.abbrev = T_Info::Abbrev(); + + addInfoObject(info); + } + + void KernelLanguageTypeInfo::registerAllTypeInfo() + { + registerTypeInfo(); + registerTypeInfo(); + registerTypeInfo(); + } + + void KernelLanguageTypeInfo::registerAllTypeInfoOnce() + { + static int call_once = (registerAllTypeInfo(), 0); + + // Use the variable to quiet the compiler. + if(call_once) + return; + } + + void KernelLanguageTypeInfo::addInfoObject(KernelLanguageTypeInfo const& info) + { + auto toLower = [](std::string tmp) { + std::transform(tmp.begin(), tmp.end(), tmp.begin(), ::tolower); + return tmp; + }; + + data[info.m_kernelLanguage] = info; + + // Add some flexibility to names registry. Accept abbreviations and + // lower case versions of the strings + typeNames[info.name] = info.m_kernelLanguage; + typeNames[toLower(info.name)] = info.m_kernelLanguage; + typeNames[info.abbrev] = info.m_kernelLanguage; + typeNames[toLower(info.abbrev)] = info.m_kernelLanguage; + } + + KernelLanguageTypeInfo const& KernelLanguageTypeInfo::Get(int index) + { + return Get(static_cast(index)); + } + + KernelLanguageTypeInfo const& KernelLanguageTypeInfo::Get(KernelLanguage t) + { + registerAllTypeInfoOnce(); + + auto iter = data.find(t); + if(iter == data.end()) + throw std::runtime_error(concatenate("Invalid kernel language: ", static_cast(t))); + + return iter->second; + } + + KernelLanguageTypeInfo const& KernelLanguageTypeInfo::Get(std::string const& str) + { + registerAllTypeInfoOnce(); + + auto iter = typeNames.find(str); + if(iter == typeNames.end()) + throw std::runtime_error(concatenate("Invalid kernel language: ", str)); + + return Get(iter->second); + } + + std::ostream& operator<<(std::ostream& stream, const KernelLanguage& t) + { + return stream << ToString(t); + } + + std::istream& operator>>(std::istream& stream, KernelLanguage& t) + { + std::string strValue; + stream >> strValue; + + t = KernelLanguageTypeInfo::Get(strValue).m_kernelLanguage; + + return stream; + } +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/PerformanceMetricTypes.cpp b/tensilelite/Tensile/Source/lib/source/PerformanceMetricTypes.cpp new file mode 100644 index 0000000000..2c4d6ecb94 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/PerformanceMetricTypes.cpp @@ -0,0 +1,160 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include + +#include + +namespace Tensile +{ + std::map PerformanceMetricTypeInfo::data; + std::map PerformanceMetricTypeInfo::typeNames; + + std::string ToString(PerformanceMetric d) + { + switch(d) + { + case PerformanceMetric::Auto: + return "Auto"; + case PerformanceMetric::CUEfficiency: + return "CUEfficiency"; + case PerformanceMetric::DeviceEfficiency: + return "DeviceEfficiency"; + + case PerformanceMetric::Count: + default:; + } + return "Invalid"; + } + + std::string TypeAbbrev(PerformanceMetric d) + { + switch(d) + { + case PerformanceMetric::Auto: + return "Auto"; + case PerformanceMetric::CUEfficiency: + return "CUEff"; + case PerformanceMetric::DeviceEfficiency: + return "DvEff"; + + case PerformanceMetric::Count: + default:; + } + return "Invalid"; + } + + template + void PerformanceMetricTypeInfo::registerTypeInfo() + { + using T_Info = PerformanceMetricInfo; + + PerformanceMetricTypeInfo info; + + info.m_performanceMetric = T_Info::Enum; + info.name = T_Info::Name(); + info.abbrev = T_Info::Abbrev(); + + addInfoObject(info); + } + + void PerformanceMetricTypeInfo::registerAllTypeInfo() + { + registerTypeInfo(); + registerTypeInfo(); + registerTypeInfo(); + } + + void PerformanceMetricTypeInfo::registerAllTypeInfoOnce() + { + static int call_once = (registerAllTypeInfo(), 0); + + // Use the variable to quiet the compiler. + if(call_once) + return; + } + + void PerformanceMetricTypeInfo::addInfoObject(PerformanceMetricTypeInfo const& info) + { + auto toLower = [](std::string tmp) { + std::transform(tmp.begin(), tmp.end(), tmp.begin(), ::tolower); + return tmp; + }; + + data[info.m_performanceMetric] = info; + + // Add some flexibility to names registry. Accept abbreviations and + // lower case versions of the strings + typeNames[info.name] = info.m_performanceMetric; + typeNames[toLower(info.name)] = info.m_performanceMetric; + typeNames[info.abbrev] = info.m_performanceMetric; + typeNames[toLower(info.abbrev)] = info.m_performanceMetric; + } + + PerformanceMetricTypeInfo const& PerformanceMetricTypeInfo::Get(int index) + { + return Get(static_cast(index)); + } + + PerformanceMetricTypeInfo const& PerformanceMetricTypeInfo::Get(PerformanceMetric t) + { + registerAllTypeInfoOnce(); + + auto iter = data.find(t); + if(iter == data.end()) + throw std::runtime_error( + concatenate("Invalid performance metric: ", static_cast(t))); + + return iter->second; + } + + PerformanceMetricTypeInfo const& PerformanceMetricTypeInfo::Get(std::string const& str) + { + registerAllTypeInfoOnce(); + + auto iter = typeNames.find(str); + if(iter == typeNames.end()) + throw std::runtime_error(concatenate("Invalid performance metric: ", str)); + + return Get(iter->second); + } + + std::ostream& operator<<(std::ostream& stream, const PerformanceMetric& t) + { + return stream << ToString(t); + } + + std::istream& operator>>(std::istream& stream, PerformanceMetric& t) + { + std::string strValue; + stream >> strValue; + + t = PerformanceMetricTypeInfo::Get(strValue).m_performanceMetric; + + return stream; + } +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/ScalarValueTypes.cpp b/tensilelite/Tensile/Source/lib/source/ScalarValueTypes.cpp new file mode 100644 index 0000000000..ce7c96b707 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/ScalarValueTypes.cpp @@ -0,0 +1,139 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include + +#include + +namespace Tensile +{ + std::map ScalarValueTypeInfo::data; + std::map ScalarValueTypeInfo::typeNames; + + std::string ToString(ScalarValue d) + { + switch(d) + { + case ScalarValue::Any: + return "Any"; + case ScalarValue::One: + return "1"; + case ScalarValue::NegativeOne: + return "-1"; + + case ScalarValue::Count: + default:; + } + return "Invalid"; + } + + template + void ScalarValueTypeInfo::registerTypeInfo() + { + using T_Info = ScalarValueInfo; + + ScalarValueTypeInfo info; + + info.m_value = T_Info::Enum; + info.name = T_Info::Name(); + + addInfoObject(info); + } + + void ScalarValueTypeInfo::registerAllTypeInfo() + { + registerTypeInfo(); + registerTypeInfo(); + registerTypeInfo(); + } + + void ScalarValueTypeInfo::registerAllTypeInfoOnce() + { + static int call_once = (registerAllTypeInfo(), 0); + + // Use the variable to quiet the compiler. + if(call_once) + return; + } + + void ScalarValueTypeInfo::addInfoObject(ScalarValueTypeInfo const& info) + { + auto toLower = [](std::string tmp) { + std::transform(tmp.begin(), tmp.end(), tmp.begin(), ::tolower); + return tmp; + }; + + data[info.m_value] = info; + + // Add some flexibility to names registry. Accept + // lower case versions of the strings + typeNames[info.name] = info.m_value; + typeNames[toLower(info.name)] = info.m_value; + } + + ScalarValueTypeInfo const& ScalarValueTypeInfo::Get(int index) + { + return Get(static_cast(index)); + } + + ScalarValueTypeInfo const& ScalarValueTypeInfo::Get(ScalarValue t) + { + registerAllTypeInfoOnce(); + + auto iter = data.find(t); + if(iter == data.end()) + throw std::runtime_error(concatenate("Invalid scalar value: ", static_cast(t))); + + return iter->second; + } + + ScalarValueTypeInfo const& ScalarValueTypeInfo::Get(std::string const& str) + { + registerAllTypeInfoOnce(); + + auto iter = typeNames.find(str); + if(iter == typeNames.end()) + throw std::runtime_error(concatenate("Invalid scalar value: ", str)); + + return Get(iter->second); + } + + std::ostream& operator<<(std::ostream& stream, const ScalarValue& t) + { + return stream << ToString(t); + } + + std::istream& operator>>(std::istream& stream, ScalarValue& t) + { + std::string strValue; + stream >> strValue; + + t = ScalarValueTypeInfo::Get(strValue).m_value; + + return stream; + } +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/Tensile.cpp b/tensilelite/Tensile/Source/lib/source/Tensile.cpp new file mode 100644 index 0000000000..8a80f0c40b --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/Tensile.cpp @@ -0,0 +1,102 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include + +#include +#include + +#ifdef TENSILE_DEFAULT_SERIALIZATION +#ifdef TENSILE_YAML +#include +#endif + +#ifdef TENSILE_MSGPACK +#include +#endif +#endif + +namespace Tensile +{ + + TENSILE_API Problem::~Problem() = default; + TENSILE_API Hardware::Hardware() = default; + TENSILE_API Hardware::~Hardware() = default; + TENSILE_API Solution::~Solution() = default; + TENSILE_API SolutionAdapter::~SolutionAdapter() = default; + +#ifdef TENSILE_DEFAULT_SERIALIZATION + template + std::shared_ptr> + LoadLibraryFile(std::string const& filename) + { + std::shared_ptr> rv; + +#ifdef TENSILE_MSGPACK + rv = MessagePackLoadLibraryFile(filename); + if(rv) + return rv; +#endif + +#ifdef TENSILE_YAML + rv = LLVMLoadLibraryFile(filename); + if(rv) + return rv; +#endif + + // Failed to load library, return nullptr. + return nullptr; + } + + template + std::shared_ptr> + LoadLibraryData(std::vector const& data) + { + std::shared_ptr> rv; + +#ifdef TENSILE_MSGPACK + rv = MessagePackLoadLibraryData(data); + if(rv) + return rv; +#endif + +#ifdef TENSILE_YAML + rv = LLVMLoadLibraryData(data); + if(rv) + return rv; +#endif + + // Failed to load library, return nullptr. + return nullptr; + } + + template std::shared_ptr> + LoadLibraryFile(std::string const& filename); + + template std::shared_ptr> + LoadLibraryData(std::vector const& data); +#endif +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/TensorDescriptor.cpp b/tensilelite/Tensile/Source/lib/source/TensorDescriptor.cpp new file mode 100644 index 0000000000..6012bfd9fc --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/TensorDescriptor.cpp @@ -0,0 +1,121 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace Tensile +{ + const size_t TensorDescriptor::UseDefaultStride = static_cast(-1); + + bool TensorDescriptor::operator==(const TensorDescriptor& rhs) const + { + return m_dataType == rhs.m_dataType && m_sizes == rhs.m_sizes && m_strides == rhs.m_strides + && m_offset == rhs.m_offset; + } + + bool TensorDescriptor::operator!=(const TensorDescriptor& rhs) const + { + return !(*this == rhs); + } + + void TensorDescriptor::appendDim(size_t size) + { + appendDim(size, m_totalAllocatedElements); + } + + void TensorDescriptor::appendDim(size_t size, size_t stride) + { + m_sizes.push_back(size); + m_strides.push_back(stride); + + calculate(); + } + + int64_t TensorDescriptor::dimensionPadding(size_t dim) const + { + TENSILE_ASSERT_EXC(dim < dimensions()); + + if(dim == 0) + return m_strides[0] - 1; + + return m_strides[dim] - (m_strides[dim - 1] * m_sizes[dim - 1]); + } + + void TensorDescriptor::collapseDims(size_t begin, size_t end) + { + TENSILE_ASSERT_EXC(end >= begin); + TENSILE_ASSERT_EXC(begin < dimensions()); + TENSILE_ASSERT_EXC(end <= dimensions()); + + if(end <= (begin + 1)) + return; + + for(size_t i = begin + 1; i < end; i++) + TENSILE_ASSERT_EXC(dimensionPadding(i) == 0); + + size_t newDimensionSize = 1; + for(size_t i = begin; i < end; i++) + newDimensionSize *= m_sizes[i]; + + m_sizes.erase(m_sizes.begin() + (begin + 1), m_sizes.begin() + end); + m_sizes[begin] = newDimensionSize; + + m_strides.erase(m_strides.begin() + (begin + 1), m_strides.begin() + end); + + calculate(); + } + + std::string TensorDescriptor::ToString() const + { + std::ostringstream result; + + result << dimensions() << "-tensor<" << dataType() << ">" + << "( sizes("; + streamJoin(result, m_sizes, ", "); + + result << "), strides("; + streamJoin(result, m_strides, ", "); + + result << "), offset(" << m_offset << "))"; + + return result.str(); + } + + std::ostream& operator<<(std::ostream& stream, const TensorDescriptor& t) + { + return stream << t.ToString(); + } + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/TensorOps.cpp b/tensilelite/Tensile/Source/lib/source/TensorOps.cpp new file mode 100644 index 0000000000..390b93ef83 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/TensorOps.cpp @@ -0,0 +1,124 @@ +/** + * Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include + +namespace Tensile +{ + std::map TensorOp::typeNames; + + TensorOp::TensorOp() = default; + TensorOp::TensorOp(Type type) + : type(type) + { + } + + std::string ToString(TensorOp::Type t) + { + switch(t) + { + case TensorOp::Type::None: + return "None"; + case TensorOp::Type::ComplexConjugate: + return "ComplexConjugate"; + + case TensorOp::Type::Count:; + } + + return "Invalid"; + } + + std::string Suffix(TensorOp::Type t) + { + switch(t) + { + case TensorOp::Type::None: + return ""; + case TensorOp::Type::ComplexConjugate: + return "C"; + + case TensorOp::Type::Count:; + } + + return "Invalid"; + } + + TensorOp::Type TensorOp::GetType(std::string const& name) + { + static int call_once = (InitTypeNames(), 0); + + auto iter = typeNames.find(name); + if(iter == typeNames.end()) + throw std::runtime_error(concatenate("Invalid TensorOp type: ", name)); + + // Use variable to quiet the compiler. + if(call_once) + return iter->second; + return iter->second; + } + + void TensorOp::InitTypeNames() + { + for(int idx = 0; idx < static_cast(Type::Count); idx++) + { + Type type = static_cast(idx); + typeNames[ToString(type)] = type; + typeNames[Suffix(type)] = type; + } + } + + std::string TensorOp::name() const + { + return ToString(type); + } + + std::string TensorOp::suffix() const + { + return Suffix(type); + } + + std::ostream& operator<<(std::ostream& stream, TensorOp const& t) + { + return stream << t.type; + } + + std::istream& operator>>(std::istream& stream, TensorOp& t) + { + return stream >> t.type; + } + + std::ostream& operator<<(std::ostream& stream, TensorOp::Type const& t) + { + return stream << ToString(t); + } + + std::istream& operator>>(std::istream& stream, TensorOp::Type& t) + { + std::string typeName; + stream >> typeName; + + t = TensorOp::GetType(typeName); + + return stream; + } +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/Utils.cpp b/tensilelite/Tensile/Source/lib/source/Utils.cpp new file mode 100644 index 0000000000..900a4007ba --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/Utils.cpp @@ -0,0 +1,66 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include + +namespace Tensile +{ + StreamRead::StreamRead(std::string const& value, bool except) + : m_value(value) + , m_except(except) + , m_success(false) + { + } + + StreamRead::~StreamRead() = default; + + // bool StreamRead::operator bool() const { return m_success; } + + bool StreamRead::read(std::istream& stream) + { + m_success = false; + char ch; + + for(int i = 0; i < m_value.size(); i++) + { + if((ch = stream.get()) != m_value[i]) + { + for(int j = 0; j <= i; j++) + stream.unget(); + + if(m_except) + throw std::runtime_error( + concatenate("Expected '", m_value[i], "', found '", ch, "'.")); + + return false; + } + } + + m_success = true; + return true; + } + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/hip/CMakeLists.txt b/tensilelite/Tensile/Source/lib/source/hip/CMakeLists.txt new file mode 100644 index 0000000000..c751aa0ef2 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/hip/CMakeLists.txt @@ -0,0 +1,31 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +set(CMAKE_CXX_COMPILER ${HIP_HIPCC_EXECUTABLE}) + +add_library(TensileHip STATIC + HipSolutionAdapter.cpp + HipHardware.cpp) + +message( TensileHip SYSTEM ${HIP_INCLUDE_DIRS} ) + +include_directories(${HIP_INCLUDE_DIRS}) + diff --git a/tensilelite/Tensile/Source/lib/source/hip/HipHardware.cpp b/tensilelite/Tensile/Source/lib/source/hip/HipHardware.cpp new file mode 100644 index 0000000000..9fbfa0000a --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/hip/HipHardware.cpp @@ -0,0 +1,73 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include +#include + +namespace Tensile +{ + namespace hip + { + HipAMDGPU::HipAMDGPU(hipDeviceProp_t const& prop) + : AMDGPU(static_cast(prop.gcnArch), + prop.multiProcessorCount, + std::string(prop.name)) + , properties(prop) + { + } + + std::shared_ptr GetCurrentDevice() + { + int deviceId = 0; + HIP_CHECK_EXC(hipGetDevice(&deviceId)); + return GetDevice(deviceId); + } + + std::shared_ptr GetDevice(int deviceId) + { + hipDeviceProp_t prop; + HIP_CHECK_EXC(hipGetDeviceProperties(&prop, deviceId)); +#if HIP_VERSION >= 50220730 + int hip_version; + HIP_CHECK_EXC(hipRuntimeGetVersion(&hip_version)); + if(hip_version >= 50220730) + { + HIP_CHECK_EXC(hipDeviceGetAttribute(&prop.multiProcessorCount, + hipDeviceAttributePhysicalMultiProcessorCount, + deviceId)); + } +#endif + + return GetDevice(prop); + } + + std::shared_ptr GetDevice(hipDeviceProp_t const& prop) + { + return std::make_shared(prop); + } + } // namespace hip +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp b/tensilelite/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp new file mode 100644 index 0000000000..3311a72cc0 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp @@ -0,0 +1,339 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include + +#include + +#include +#include +#include +#include + +namespace Tensile +{ + namespace hip + { + SolutionAdapter::SolutionAdapter() + : m_debug(Debug::Instance().printKernelArguments()) + , m_debugSkipLaunch(Debug::Instance().skipKernelLaunch()) + { + } + + SolutionAdapter::SolutionAdapter(bool debug) + : m_debug(debug) + { + m_debug = debug || Debug::Instance().printKernelArguments(); + } + + SolutionAdapter::SolutionAdapter(bool debug, std::string const& name) + : m_debug(debug) + , m_name(name) + { + m_debug = debug || Debug::Instance().printKernelArguments(); + } + + SolutionAdapter::~SolutionAdapter() + { + for(auto module : m_modules) + hipModuleUnload(module); + } + + hipError_t SolutionAdapter::loadCodeObjectFile(std::string const& path) + { + hipModule_t module; + + HIP_CHECK_RETURN(hipModuleLoad(&module, path.c_str())); + + if(m_debug) + std::cout << "loaded code object" << path << std::endl; + + { + std::lock_guard guard(m_access); + m_modules.push_back(module); + m_loadedModuleNames.push_back(concatenate("File ", path)); + } + return hipSuccess; + } + + hipError_t SolutionAdapter::loadCodeObjectBytes(std::vector const& bytes) + { + return loadCodeObject(bytes.data()); + } + + hipError_t SolutionAdapter::loadCodeObject(const void* image) + { + hipModule_t module; + + HIP_CHECK_RETURN(hipModuleLoadData(&module, image)); + + if(m_debug) + std::cout << "loaded code object data." << std::endl; + + { + std::lock_guard guard(m_access); + m_modules.push_back(module); + m_loadedModuleNames.push_back("Module from bytes"); + } + return hipSuccess; + } + + void SolutionAdapter::loadEmbeddedCodeObjects() + { + loadEmbeddedCodeObjects(""); + } + + void SolutionAdapter::loadEmbeddedCodeObjects(std::string const& key) + { + auto const& embeddedData = EmbeddedData::Get(key); + + if(embeddedData.size() == 0) + { + if(m_debug || Debug::Instance().printCodeObjectInfo()) + { + std::cerr << "Found no embedded code objects"; + if(key != "") + std::cerr << " with the key " << key; + + std::cerr << "." << std::endl; + } + return; + } + + std::vector newModules; + newModules.reserve(embeddedData.size()); + + for(size_t i = 0; i < embeddedData.size(); i++) + { + hipModule_t nextModule; + try + { + auto error = hipModuleLoadData(&nextModule, embeddedData[i].data()); + + if(error == hipErrorUnknown || error == hipErrorSharedObjectInitFailed) + continue; + newModules.push_back(nextModule); + HIP_CHECK_EXC(error); + + if(m_debug) + std::cout << "Loaded code object for key " << key << std::endl; + } + catch(std::runtime_error const& exc) + { + std::cout << exc.what() << std::endl; + } + } + + { + std::lock_guard guard(m_access); + m_modules.insert(m_modules.end(), newModules.begin(), newModules.end()); + m_loadedModuleNames.push_back( + concatenate("Embedded code object ", key, " (", newModules.size(), ")")); + } + } + + hipError_t SolutionAdapter::initKernel(std::string const& name) + { + hipFunction_t function; + return getKernel(function, name); + } + + hipError_t SolutionAdapter::getKernel(hipFunction_t& rv, std::string const& name) + { + std::unique_lock guard(m_access); + hipError_t err = hipErrorNotFound; + + auto it = m_kernels.find(name); + if(it != m_kernels.end()) + { + rv = it->second; + return hipSuccess; + } + + for(auto module : m_modules) + { + err = hipModuleGetFunction(&rv, module, name.c_str()); + + if(err == hipSuccess) + { + m_kernels[name] = rv; + return err; + } + else if(err != hipErrorNotFound) + { + return err; + } + } + + return err; + } + + hipError_t SolutionAdapter::launchKernel(KernelInvocation const& kernel) + { + return launchKernel(kernel, nullptr, nullptr, nullptr); + } + + hipError_t SolutionAdapter::launchKernel(KernelInvocation const& kernel, + hipStream_t stream, + hipEvent_t startEvent, + hipEvent_t stopEvent) + { + if(m_debug) + { + std::cout << "Kernel " << kernel.kernelName << std::endl; + std::cout << " l" << kernel.workGroupSize << " x g" << kernel.numWorkGroups << " = " + << kernel.numWorkItems << std::endl; + std::cout << kernel.args; + } + if(m_debugSkipLaunch) + { + std::cout << "DEBUG: Skip kernel execution" << std::endl; + if(startEvent != nullptr) + HIP_CHECK_RETURN(hipEventRecord(startEvent, stream)); + if(stopEvent != nullptr) + HIP_CHECK_RETURN(hipEventRecord(stopEvent, stream)); + return hipSuccess; + } + + hipFunction_t function; + HIP_CHECK_RETURN(getKernel(function, kernel.kernelName)); + + void* kernelArgs = const_cast(kernel.args.data()); + size_t argsSize = kernel.args.size(); + + void* hipLaunchParams[] = {HIP_LAUNCH_PARAM_BUFFER_POINTER, + kernelArgs, + HIP_LAUNCH_PARAM_BUFFER_SIZE, + &argsSize, + HIP_LAUNCH_PARAM_END}; + + if(startEvent != nullptr) + HIP_CHECK_RETURN(hipEventRecord(startEvent, stream)); + HIP_CHECK_RETURN(hipExtModuleLaunchKernel(function, + kernel.numWorkItems.x, + kernel.numWorkItems.y, + kernel.numWorkItems.z, + kernel.workGroupSize.x, + kernel.workGroupSize.y, + kernel.workGroupSize.z, + kernel.sharedMemBytes, // sharedMem + stream, // stream + nullptr, + (void**)&hipLaunchParams, + nullptr, // event + nullptr // event + )); + if(stopEvent != nullptr) + HIP_CHECK_RETURN(hipEventRecord(stopEvent, stream)); + return hipSuccess; + } + + hipError_t SolutionAdapter::launchKernels(std::vector const& kernels) + { + for(auto const& k : kernels) + { + HIP_CHECK_RETURN(launchKernel(k)); + } + return hipSuccess; + } + + hipError_t SolutionAdapter::launchKernels(std::vector const& kernels, + hipStream_t stream, + hipEvent_t startEvent, + hipEvent_t stopEvent) + { + auto first = kernels.begin(); + auto last = kernels.end() - 1; + + for(auto iter = kernels.begin(); iter != kernels.end(); iter++) + { + hipEvent_t kStart = nullptr; + hipEvent_t kStop = nullptr; + + if(iter == first) + kStart = startEvent; + if(iter == last) + kStop = stopEvent; + + HIP_CHECK_RETURN(launchKernel(*iter, stream, kStart, kStop)); + } + return hipSuccess; + } + + hipError_t SolutionAdapter::launchKernels(std::vector const& kernels, + hipStream_t stream, + std::vector const& startEvents, + std::vector const& stopEvents) + { + if(kernels.size() != startEvents.size() || kernels.size() != stopEvents.size()) + throw std::runtime_error(concatenate("Must have an equal number of kernels (", + kernels.size(), + "), start events (", + startEvents.size(), + "), and stop events. (", + stopEvents.size(), + ")")); + + for(size_t i = 0; i < kernels.size(); i++) + { + HIP_CHECK_RETURN(launchKernel(kernels[i], stream, startEvents[i], stopEvents[i])); + } + return hipSuccess; + } + + std::ostream& operator<<(std::ostream& stream, SolutionAdapter const& adapter) + { + stream << "hip::SolutionAdapter"; + + if(adapter.m_debug) + { + stream << "[" << std::endl; + for(auto const& name : adapter.m_loadedModuleNames) + stream << name << std::endl; + + stream << "]"; + } + + stream << " (" << adapter.name() << ", " << adapter.m_modules.size() + << " total modules)" << std::endl; + + return stream; + } + + std::ostream& operator<<(std::ostream& stream, std::shared_ptr const& ptr) + { + if(ptr) + { + return stream << "*" << *ptr; + } + else + { + return stream << "(nullptr)"; + } + } + } // namespace hip +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/llvm/Loading.cpp b/tensilelite/Tensile/Source/lib/source/llvm/Loading.cpp new file mode 100644 index 0000000000..e0814b1db8 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/llvm/Loading.cpp @@ -0,0 +1,103 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include + +#include + +#include +#include +#include + +namespace Tensile +{ + + template + std::shared_ptr> + LLVMLoadLibraryFile(std::string const& filename) + { + std::shared_ptr> rv; + + try + { + auto inputFile = llvm::MemoryBuffer::getFile(filename); + llvm::yaml::Input yin((*inputFile)->getMemBufferRef()); + + yin >> rv; + + if(yin.error()) + { + return nullptr; + } + } + catch(std::runtime_error const& exc) + { + if(Debug::Instance().printDataInit()) + std::cout << "Error loading " << filename << " (YAML):" << std::endl + << exc.what() << std::endl; + + return nullptr; + } + + return rv; + } + + template + std::shared_ptr> + LLVMLoadLibraryData(std::vector const& data) + { + std::shared_ptr> rv; + + try + { + llvm::StringRef dataRef((const char*)data.data(), data.size()); + llvm::yaml::Input yin(dataRef); + + yin >> rv; + + if(yin.error()) + { + throw std::runtime_error(yin.error().message()); + } + } + catch(std::runtime_error const& exc) + { + if(Debug::Instance().printDataInit()) + std::cout << "Error loading YAML data:" << std::endl << exc.what() << std::endl; + + return nullptr; + } + + return rv; + } + + template std::shared_ptr> + LLVMLoadLibraryFile(std::string const& filename); + + template std::shared_ptr> + LLVMLoadLibraryData( + std::vector const& data); +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/llvm/YAML.cpp b/tensilelite/Tensile/Source/lib/source/llvm/YAML.cpp new file mode 100644 index 0000000000..ff76533fbe --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/llvm/YAML.cpp @@ -0,0 +1,9 @@ + +#include + +namespace Tensile +{ + namespace Serialization + { + } +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/msgpack/MessagePack.cpp b/tensilelite/Tensile/Source/lib/source/msgpack/MessagePack.cpp new file mode 100644 index 0000000000..8966da4407 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/msgpack/MessagePack.cpp @@ -0,0 +1,193 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2020-2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include + +#include + +#include + +namespace Tensile +{ + namespace Serialization + { + void objectToMap(msgpack::object& object, + std::unordered_map& result) + { + if(object.type != msgpack::type::object_type::MAP) + throw std::runtime_error(concatenate("Expected MAP, found ", object.type)); + + for(uint32_t i = 0; i < object.via.map.size; i++) + { + auto& element = object.via.map.ptr[i]; + + std::string key; + switch(element.key.type) + { + case msgpack::type::object_type::STR: + { + element.key.convert(key); + break; + } + case msgpack::type::object_type::POSITIVE_INTEGER: + { + auto iKey = element.key.as(); + key = std::to_string(iKey); + break; + } + default: + throw std::runtime_error("Unexpected map key type"); + } + + result[key] = std::move(element.val); + } + } + } + + template + std::shared_ptr> + MessagePackLoadLibraryFile(std::string const& filename) + { + // parse file into a msgpack::object_handle + msgpack::object_handle result; + try + { + std::ifstream in(filename, std::ios::in | std::ios::binary); + if(!in.is_open()) + { + if(Debug::Instance().printDataInit()) + std::cout << "Error loading " << filename << " (msgpack):\nFailed to open file" + << std::endl; + + return nullptr; + } + + msgpack::unpacker unp; + bool finished_parsing; + constexpr size_t buffer_size = 1 << 19; + do + { + unp.reserve_buffer(buffer_size); + in.read(unp.buffer(), buffer_size); + unp.buffer_consumed(in.gcount()); + finished_parsing = unp.next(result); // may throw msgpack::parse_error + } while(!finished_parsing && !in.fail()); + + if(!finished_parsing) + { + if(Debug::Instance().printDataInit()) + { + const char* const error_str + = in.eof() ? "Unexpected end of file" : "Read failure"; + std::cout << "Error loading " << filename << " (msgpack):\n" + << error_str << std::endl; + } + + return nullptr; + } + } + catch(std::runtime_error const& exc) + { + if(Debug::Instance().printDataInit()) + std::cout << "Error loading msgpack data:\n" << exc.what() << std::endl; + + return nullptr; + } + + // copy data from msgpack::object_handle into MasterSolutionLibrary + try + { + std::shared_ptr> rv; + + Serialization::MessagePackInput min(result.get()); + + Serialization::PointerMappingTraits::mapping(min, rv); + + if(!min.error.empty()) + { + std::ostringstream msg; + msg << "Error loading msgpack data:\n"; + for(auto const& err : min.error) + msg << err << std::endl; + + throw std::runtime_error(msg.str()); + } + + return rv; + } + catch(std::runtime_error const& exc) + { + if(Debug::Instance().printDataInit()) + std::cout << "Error loading msgpack data:\n" << exc.what() << std::endl; + + return nullptr; + } + } + + template + std::shared_ptr> + MessagePackLoadLibraryData(std::vector const& data) + { + try + { + std::shared_ptr> rv; + + auto result = msgpack::unpack((const char*)data.data(), data.size()); + Serialization::MessagePackInput min(result.get()); + + Serialization::PointerMappingTraits::mapping(min, rv); + + if(!min.error.empty()) + { + std::ostringstream msg; + msg << "Error loading msgpack data:" << std::endl; + for(auto const& err : min.error) + msg << err << std::endl; + + throw std::runtime_error(msg.str()); + } + + return rv; + } + catch(std::runtime_error const& exc) + { + if(Debug::Instance().printDataInit()) + std::cout << "Error loading msgpack data:" << std::endl << exc.what() << std::endl; + + return nullptr; + } + } + + template std::shared_ptr> + MessagePackLoadLibraryFile( + std::string const& filename); + + template std::shared_ptr> + MessagePackLoadLibraryData( + std::vector const& data); +} diff --git a/tensilelite/Tensile/Source/lib/source/ocl/CMakeLists.txt b/tensilelite/Tensile/Source/lib/source/ocl/CMakeLists.txt new file mode 100644 index 0000000000..556ca85837 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/ocl/CMakeLists.txt @@ -0,0 +1,12 @@ + +set(CMAKE_CXX_COMPILER ${HIP_HIPCC_EXECUTABLE}) + +add_library(TensileOcl STATIC + OclHardware.cpp + OclSolutionAdapter.cpp + OclUtils.cpp) + +message( TensileOcl SYSTEM ${HIP_INCLUDE_DIRS} ${HCC_INCLUDE_DIRS} ) + +include_directories(${HIP_INCLUDE_DIRS} ${HCC_INCLUDE_DIRS}) + diff --git a/tensilelite/Tensile/Source/lib/source/ocl/OclHardware.cpp b/tensilelite/Tensile/Source/lib/source/ocl/OclHardware.cpp new file mode 100644 index 0000000000..ac215292b5 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/ocl/OclHardware.cpp @@ -0,0 +1,40 @@ +#include + +#include + +#include +#include + +namespace Tensile +{ + namespace ocl + { + OclAMDGPU::OclAMDGPU(oclDeviceProp_t const& prop) + : AMDGPU(static_cast(prop.gcnArch), + prop.multiProcessorCount, + std::string(prop.name)) + , properties(prop) + { + } + + std::shared_ptr GetCurrentDevice() + { + return GetDevice(clGetDevicePropertiesAMD()); + } + + std::shared_ptr GetDevice(int deviceId) + { + return GetDevice(clGetDevicePropertiesAMD(deviceId)); + } + + std::shared_ptr GetDevice(int deviceId, cl::Context context) + { + return GetDevice(clGetDevicePropertiesAMD(deviceId, context)); + } + + std::shared_ptr GetDevice(oclDeviceProp_t const& prop) + { + return std::make_shared(prop); + } + } // namespace hip +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/ocl/OclSolutionAdapter.cpp b/tensilelite/Tensile/Source/lib/source/ocl/OclSolutionAdapter.cpp new file mode 100644 index 0000000000..a57f13691a --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/ocl/OclSolutionAdapter.cpp @@ -0,0 +1,379 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#ifndef CL_HPP_ENABLE_EXCEPTIONS +#error \ + "This implementation relies on CL exceptions to be enabled. Please define CL_HPP_ENABLE_EXCEPTIONS" +#endif + +#include +#include + +#include +#include + +#include +#include +#include + +namespace Tensile +{ + namespace ocl + { + SolutionAdapter::SolutionAdapter() + : m_debug(Debug::Instance().printKernelArguments()) + , m_debugSkipLaunch(Debug::Instance().skipKernelLaunch()) + , m_context(cl::Context::getDefault()) + , m_device(cl::Device::getDefault()) + { + } + + SolutionAdapter::SolutionAdapter(bool debug) + : m_debug(debug || Debug::Instance().printKernelArguments()) + , m_debugSkipLaunch(Debug::Instance().skipKernelLaunch()) + , m_context(cl::Context::getDefault()) + , m_device(cl::Device::getDefault()) + { + } + + SolutionAdapter::SolutionAdapter(bool debug, std::string const& name) + : m_debug(debug || Debug::Instance().printKernelArguments()) + , m_debugSkipLaunch(Debug::Instance().skipKernelLaunch()) + , m_context(cl::Context::getDefault()) + , m_device(cl::Device::getDefault()) + , m_name(name) + { + } + + SolutionAdapter::SolutionAdapter(bool debug, + std::string const& name, + cl::Context context, + cl::Device device) + : m_debug(debug || Debug::Instance().printKernelArguments()) + , m_debugSkipLaunch(Debug::Instance().skipKernelLaunch()) + , m_context(context) + , m_device(device) + , m_name(name) + { + } + + std::string SolutionAdapter::name() const + { + return m_name; + } + + void SolutionAdapter::loadCodeObjectFile(std::string const& path) + { + try + { + auto progModule = clModuleLoad(m_context, {m_device}, path); + + if(m_debug) + { + std::cout << "Loaded code object: " << path << std::endl; + } + + addModule(concatenate("File ", path), progModule); + } + // Binary might not be for this specific device, which is OK. + catch(cl::BuildError const& e) + { + if(m_debug) + { + std::cerr << "Error code: " << e.err() << " " << e.what() << std::endl; + for(auto const& log : e.getBuildLog()) + { + std::cerr << "Device: " << std::get<0>(log).getInfo() << " " + << std::get<1>(log) << std::endl; + } + } + } + } + + void SolutionAdapter::loadCodeObjectBytes(std::vector const& bytes) + { + try + { + auto progModule = clModuleLoadData(m_context, {m_device}, bytes); + + if(m_debug) + { + std::cout << "loaded code object data." << std::endl; + } + + addModule("Module from bytes", progModule); + } + // Binary might not be for this specific device, which is OK. + catch(cl::BuildError const& e) + { + if(m_debug) + { + std::cerr << "Error code: " << e.err() << " " << e.what() << std::endl; + for(auto const& log : e.getBuildLog()) + { + std::cerr << "Device: " << std::get<0>(log).getInfo() << " " + << std::get<1>(log) << std::endl; + } + } + } + } + + void SolutionAdapter::loadEmbeddedCodeObjects() + { + loadEmbeddedCodeObjects(""); + } + + void SolutionAdapter::loadEmbeddedCodeObjects(std::string const& key) + { + auto const& embeddedData = EmbeddedData::Get(key); + + if(embeddedData.size() == 0) + { + if(m_debug || Debug::Instance().printCodeObjectInfo()) + { + std::cout << "Found no embedded code objects"; + if(key != "") + { + std::cout << " with the key " << key; + } + + std::cout << "." << std::endl; + } + return; + } + + std::vector newModules; + newModules.reserve(embeddedData.size()); + for(size_t i = 0; i < embeddedData.size(); i++) + { + try + { + newModules.push_back(clModuleLoadData(m_context, {m_device}, embeddedData[i])); + + if(m_debug) + { + std::cout << "Loaded code object for key " << key << std::endl; + } + } + // Binary might not be for this specific device, which is OK. + catch(cl::BuildError const& e) + { + if(m_debug) + { + std::cerr << "Error code: " << e.err() << " " << e.what() << std::endl; + for(auto const& log : e.getBuildLog()) + { + std::cerr << "Device: " << std::get<0>(log).getInfo() + << " " << std::get<1>(log) << std::endl; + } + } + } + } + + if(m_debug || Debug::Instance().printCodeObjectInfo()) + { + std::cout << "Successfully loaded " << newModules.size() + << " embedded code object(s)\n"; + } + addModules(concatenate("Embedded code object ", key, " (", newModules.size(), ")"), + newModules); + } + + void SolutionAdapter::initKernel(std::string const& name) + { + getKernel(name); + } + + cl::Kernel SolutionAdapter::getKernel(std::string const& name) + { + std::unique_lock guard(m_access); + + auto it = m_kernels.find(name); + if(it != m_kernels.end()) + { + return it->second; + } + + for(auto module : m_modules) + { + try + { + return m_kernels[name] = cl::Kernel(module, name.c_str()); + } + // Kernel may not exist in all modules + catch(cl::Error& e) + { + if(e.err() != CL_INVALID_KERNEL_NAME) + { + if(m_debug) + { + std::cerr << "KernelError: " << e.err() << std::endl; + } + throw; + } + } + } + + throw cl::Error( + CL_INVALID_KERNEL_NAME, + concatenate("Kernel ", name, " not found in any loaded module.").c_str()); + } + + void SolutionAdapter::launchKernel(KernelInvocation const& kernel) + { + launchKernel(kernel, cl::CommandQueue::getDefault()); + } + + void SolutionAdapter::launchKernel(KernelInvocation const& kernel, + cl::CommandQueue stream, + cl::Event* timingEvent /* = nullptr */) + { + if(m_debug) + { + std::cout << "Kernel " << kernel.kernelName << std::endl; + std::cout << " l" << kernel.workGroupSize << " x g" << kernel.numWorkGroups << " = " + << kernel.numWorkItems << std::endl; + std::cout << kernel.args; + } + + if(m_debugSkipLaunch) + { + std::cout << "DEBUG: Skip kernel execution" << std::endl; + if(timingEvent != nullptr) + { + stream.enqueueMarkerWithWaitList(nullptr, timingEvent); + } + } + + auto kernelFunc = getKernel(kernel.kernelName); + { + unsigned int argIndex = 0; + for(auto arg : kernel.args) + { + kernelFunc.setArg(argIndex++, arg.second, arg.first); + } + } + + stream.enqueueNDRangeKernel( + kernelFunc, + cl::NDRange(), + {kernel.numWorkItems.x, kernel.numWorkItems.y, kernel.numWorkItems.z}, + {kernel.workGroupSize.x, kernel.workGroupSize.y, kernel.workGroupSize.z}, + nullptr, + timingEvent); + } + + void SolutionAdapter::launchKernels(std::vector const& kernels) + { + for(auto const& k : kernels) + { + launchKernel(k); + } + } + + void SolutionAdapter::launchKernels(std::vector const& kernels, + cl::CommandQueue stream, + cl::Event* timingEvent /* = nullptr */) + { + + static std::vector events(kernels.size()); + cl_uint eventIndex = 0; + for(auto const& k : kernels) + { + launchKernel(k, stream, &(events[eventIndex++])); + } + + stream.enqueueMarkerWithWaitList(&events, timingEvent); + } + + void SolutionAdapter::launchKernels(std::vector const& kernels, + cl::CommandQueue stream, + std::vector& timingEvents) + { + if(kernels.size() != timingEvents.size()) + { + throw std::runtime_error(concatenate("Must have an equal number of kernels (", + kernels.size(), + "), and timing events. (", + timingEvents.size(), + ")")); + } + + for(size_t i = 0; i < kernels.size(); i++) + { + launchKernel(kernels[i], stream, &timingEvents[i]); + } + } + + void SolutionAdapter::addModule(std::string const& name, cl::Program const& module) + { + std::lock_guard guard(m_access); + m_modules.push_back(module); + m_loadedModuleNames.push_back(name); + } + + void SolutionAdapter::addModules(std::string const& groupName, + std::vector const& modules) + { + std::lock_guard guard(m_access); + m_modules.insert(m_modules.end(), modules.begin(), modules.end()); + m_loadedModuleNames.push_back(groupName); + } + + std::ostream& operator<<(std::ostream& stream, SolutionAdapter const& adapter) + { + stream << "ocl::SolutionAdapter"; + + if(adapter.m_debug) + { + stream << "[" << std::endl; + for(auto const& name : adapter.m_loadedModuleNames) + { + stream << name << std::endl; + } + + stream << "]"; + } + + stream << " (" << adapter.name() << ", " << adapter.m_modules.size() + << " total modules)" << std::endl; + + return stream; + } + + std::ostream& operator<<(std::ostream& stream, std::shared_ptr const& ptr) + { + if(ptr) + { + return stream << "*" << *ptr; + } + else + { + return stream << "(nullptr)"; + } + } + } // namespace ocl +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/lib/source/ocl/OclUtils.cpp b/tensilelite/Tensile/Source/lib/source/ocl/OclUtils.cpp new file mode 100644 index 0000000000..05a3063116 --- /dev/null +++ b/tensilelite/Tensile/Source/lib/source/ocl/OclUtils.cpp @@ -0,0 +1,222 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright 2019-2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include + +#include +#include +#include +#include + +// std includes +#include +#include +#include + +namespace cl +{ + std::ostream& operator<<(std::ostream& stream, Buffer buffer) + { + return stream << static_cast(buffer()); + } + + // Add some definitions for extended device queries (cl_ext.h) + namespace detail + { + CL_HPP_DECLARE_PARAM_TRAITS_(cl_device_info, CL_DEVICE_BOARD_NAME_AMD, std::string); + CL_HPP_DECLARE_PARAM_TRAITS_(cl_device_info, CL_DEVICE_MAX_WORK_GROUP_SIZE_AMD, size_t); + CL_HPP_DECLARE_PARAM_TRAITS_(cl_device_info, + CL_DEVICE_TOPOLOGY_AMD, + cl_device_topology_amd); + } + +} // namespace cl + +namespace Tensile +{ + + namespace ocl + { + + cl::Program clModuleLoad(cl::Context context, + std::vector const& devices, + std::string const& path) + { + std::ifstream file(path, std::ios::binary | std::ios::in | std::ios::ate); + if(file.fail()) + { + throw std::runtime_error(concatenate("Failed to load binary file: ", path)); + } + + std::vector bytes(file.tellg()); + file.seekg(0, std::ios::beg); + file.read(reinterpret_cast(bytes.data()), bytes.size()); + file.close(); + + return clModuleLoadData(context, devices, bytes); + } + + cl::Program clModuleLoadData(cl::Context context, + std::vector const& devices, + std::vector const& bytes) + { + std::vector binErrs; + cl::Program program = cl::Program{context, devices, {bytes}, &binErrs, nullptr}; + + for(auto& binErr : binErrs) + { + if(binErr != CL_SUCCESS) + { + throw cl::Error(binErr, "Binary error code in Program creation"); + } + } + + program.build(devices); + + return program; + } + + oclDeviceProp_t clGetDevicePropertiesAMD() + { + auto device = cl::Device::getDefault(); + return clGetDevicePropertiesAMD(device); + } + + oclDeviceProp_t clGetDevicePropertiesAMD(int deviceId) + { + auto context = cl::Context::getDefault(); + return clGetDevicePropertiesAMD(deviceId, context); + } + + oclDeviceProp_t clGetDevicePropertiesAMD(int deviceId, cl::Context context) + { + auto devices = context.getInfo(); + + if(0 <= deviceId && deviceId < devices.size()) + { + return clGetDevicePropertiesAMD(devices[deviceId]); + } + else + { + throw cl::Error(CL_INVALID_DEVICE, "Device index out of range"); + } + } + + AMDGPU::Processor toProcessorId(std::string const& deviceString) + { + if(deviceString.find("gfx803") != std::string::npos) + { + return AMDGPU::Processor::gfx803; + } + else if(deviceString.find("gfx900") != std::string::npos) + { + return AMDGPU::Processor::gfx900; + } + else if(deviceString.find("gfx906") != std::string::npos) + { + return AMDGPU::Processor::gfx906; + } + else if(deviceString.find("gfx908") != std::string::npos) + { + return AMDGPU::Processor::gfx908; + } + else if(deviceString.find("gfx90a") != std::string::npos) + { + return AMDGPU::Processor::gfx90a; + } + else if(deviceString.find("gfx1010") != std::string::npos) + { + return AMDGPU::Processor::gfx1010; + } + else if(deviceString.find("gfx1011") != std::string::npos) + { + return AMDGPU::Processor::gfx1011; + } + else if(deviceString.find("gfx1012") != std::string::npos) + { + return AMDGPU::Processor::gfx1012; + } + else if(deviceString.find("gfx1030") != std::string::npos) + { + return AMDGPU::Processor::gfx1030; + } + else if(deviceString.find("gfx1100") != std::string::npos) + { + return AMDGPU::Processor::gfx1100; + } + else if(deviceString.find("gfx1101") != std::string::npos) + { + return AMDGPU::Processor::gfx1101; + } + else if(deviceString.find("gfx1102") != std::string::npos) + { + return AMDGPU::Processor::gfx1102; + } + else + { + return static_cast(0); + } + } + + oclDeviceProp_t clGetDevicePropertiesAMD(cl::Device device) + { + auto topology = device.getInfo(); + auto maxWorkItemSizes = device.getInfo(); + auto maxWorkGroupSize = device.getInfo(); + oclDeviceProp_t result = { + device.getInfo(), //std::string name; + device.getInfo(), // size_t totalGlobalMem; + device.getInfo< + CL_DEVICE_LOCAL_MEM_SIZE>(), // size_t sharedMemPerBlock; CL_DEVICE_LOCAL_MEM_SIZE + (int)device.getInfo< + CL_DEVICE_WAVEFRONT_WIDTH_AMD>(), //int warpSize; CL_WARP_SIZE_NV CL_DEVICE_WAVEFRONT_WIDTH_AMD + (int)device.getInfo(), //int maxThreadsPerBlock; + {0, 0, 0}, //int maxThreadsDim[3]; + {std::numeric_limits::max(), + std::numeric_limits::max(), + std::numeric_limits::max()}, //int maxGridSize[3]; + (int)device.getInfo() + * 1000, //int clockRate; CL_DEVICE_MAX_CLOCK_FREQUENCY + (int)device.getInfo< + CL_DEVICE_MAX_COMPUTE_UNITS>(), // int multiProcessorCount; CL_MAX_COMPUTE_UNITS + topology.pcie.bus, //int pciBusID; CL_DEVICE_TOPOLOGY_AMD / CL_DEVICE_PCI_BUS_ID_NV + topology.pcie.device, // int pciDeviceID; + device.getInfo< + CL_DEVICE_LOCAL_MEM_SIZE_PER_COMPUTE_UNIT_AMD>(), //size_t maxSharedMemoryPerMultiProcessor; + (int)toProcessorId(device.getInfo()) //int gcnArch; + }; + + result.maxThreadsDim[0] = maxWorkItemSizes[0]; + result.maxThreadsDim[1] = maxWorkItemSizes[1]; + result.maxThreadsDim[2] = maxWorkItemSizes[2]; + + return result; + } + + } // namespace ocl + +} // namespace Tensile diff --git a/tensilelite/Tensile/Source/multigpu.sh b/tensilelite/Tensile/Source/multigpu.sh new file mode 100755 index 0000000000..c21808dd00 --- /dev/null +++ b/tensilelite/Tensile/Source/multigpu.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +######################################## +# client parameters +M=5504 +N=5504 +K=5504 +NUM_PRINTOUTS=256 +NUM_ENQUEUES_PER_PRINTOUT=64 +DEVICES=( 0 1 2 3 ) +EXEC=./sgemm_gfx900 + +######################################## +# crtl-c will kill background process and quit +PIDS=() +function kill_procs() { + kill ${PIDS[@]} 2> /dev/null + echo "Aborted." +} +trap kill_procs INT + +######################################## +# launch clients +echo "" > out.cmd.txt +for DEVICE in ${DEVICES[@]} +do +CMD="${EXEC} --device-idx $DEVICE --sizes $M $N $K --num-enqueues-per-sync $NUM_ENQUEUES_PER_PRINTOUT --num-syncs-per-benchmark 1 --num-benchmarks $NUM_PRINTOUTS --use-gpu-timer 1 --function-idx 0 --init-c 3 --init-ab 3 --num-elements-to-validate 0 --print-max 0 | tee out.${DEVICE}.txt &" +echo $CMD >> out.cmd.txt +eval $CMD +PID=$! +PIDS+=($PID) +done + +######################################## +# wait for background processes +wait ${PIDS[@]} +echo "Done. Data logged to out.*.txt" diff --git a/tensilelite/Tensile/Source/tensile_bfloat16.h b/tensilelite/Tensile/Source/tensile_bfloat16.h new file mode 100644 index 0000000000..4ebfef6f01 --- /dev/null +++ b/tensilelite/Tensile/Source/tensile_bfloat16.h @@ -0,0 +1,249 @@ +/** + * MIT License + * + * Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/*!\file + * \brief tensile_bfloat16.h provides struct for tensile_bfloat16 typedef + */ + +#pragma once +#ifndef _TENSILE_BFLOAT16_H_ +#define _TENSILE_BFLOAT16_H_ + +// If this is a C compiler, C++ compiler below C++11, or a host-only compiler, +// we only include a minimal definition of tensile_bfloat16 +#if __cplusplus < 201103L || !defined(__HIPCC__) + +#include +typedef struct +{ + uint16_t data; +} tensile_bfloat16; + +#else // __cplusplus < 201103L || !defined(__HIPCC__) + +#include +#include +#include +#include +#include +#include + +struct tensile_bfloat16 +{ + uint16_t data; + + // Don't initialize `data` in purpose so that it could be used with + // `__shared__`, which forbid any initializer. + __host__ __device__ tensile_bfloat16() {} + + // round upper 16 bits of IEEE float to convert to bfloat16 + explicit __host__ __device__ tensile_bfloat16(float f) + : data(float_to_bfloat16(f)) + { + } + + // zero extend lower 16 bits of bfloat16 to convert to IEEE float + __host__ __device__ operator float() const + { + union + { + uint32_t int32; + float fp32; + } u = {uint32_t(data) << 16}; + return u.fp32; + } + +private: + static __host__ __device__ uint16_t float_to_bfloat16(float f) + { + union + { + float fp32; + uint32_t int32; + } u = {f}; + if(~u.int32 & 0x7f800000) + { + // When the exponent bits are not all 1s, then the value is zero, normal, + // or subnormal. We round the bfloat16 mantissa up by adding 0x7FFF, plus + // 1 if the least significant bit of the bfloat16 mantissa is 1 (odd). + // This causes the bfloat16's mantissa to be incremented by 1 if the 16 + // least significant bits of the float mantissa are greater than 0x8000, + // or if they are equal to 0x8000 and the least significant bit of the + // bfloat16 mantissa is 1 (odd). This causes it to be rounded to even when + // the lower 16 bits are exactly 0x8000. If the bfloat16 mantissa already + // has the value 0x7f, then incrementing it causes it to become 0x00 and + // the exponent is incremented by one, which is the next higher FP value + // to the unrounded bfloat16 value. When the bfloat16 value is subnormal + // with an exponent of 0x00 and a mantissa of 0x7F, it may be rounded up + // to a normal value with an exponent of 0x01 and a mantissa of 0x00. + // When the bfloat16 value has an exponent of 0xFE and a mantissa of 0x7F, + // incrementing it causes it to become an exponent of 0xFF and a mantissa + // of 0x00, which is Inf, the next higher value to the unrounded value. + u.int32 += 0x7fff + ((u.int32 >> 16) & 1); // Round to nearest, round to even + } + else if(u.int32 & 0xffff) + { + // When all of the exponent bits are 1, the value is Inf or NaN. + // Inf is indicated by a zero mantissa. NaN is indicated by any nonzero + // mantissa bit. Quiet NaN is indicated by the most significant mantissa + // bit being 1. Signaling NaN is indicated by the most significant + // mantissa bit being 0 but some other bit(s) being 1. If any of the + // lower 16 bits of the mantissa are 1, we set the least significant bit + // of the bfloat16 mantissa, in order to preserve signaling NaN in case + // the bloat16's mantissa bits are all 0. + u.int32 |= 0x10000; // Preserve signaling NaN + } + return uint16_t(u.int32 >> 16); + } +}; + +static_assert(std::is_standard_layout{}, + "tensile_bfloat16 is not a standard layout type, and thus is " + "incompatible with C."); + +static_assert(std::is_trivially_copyable{}, + "tensile_bfloat16 is not trivially copyable, and thus is " + "incompatible with C."); + +inline std::ostream& operator<<(std::ostream& os, const tensile_bfloat16& bf16) +{ + return os << float(bf16); +} +inline __host__ __device__ tensile_bfloat16 operator+(tensile_bfloat16 a) +{ + return a; +} +inline __host__ __device__ tensile_bfloat16 operator-(tensile_bfloat16 a) +{ + a.data ^= 0x8000; + return a; +} +inline __host__ __device__ tensile_bfloat16 operator+(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) + float(b)); +} +inline __host__ __device__ tensile_bfloat16 operator+(int a, tensile_bfloat16 b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline __host__ __device__ tensile_bfloat16 operator+(tensile_bfloat16 a, int b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline __host__ __device__ tensile_bfloat16 operator-(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) - float(b)); +} +inline __host__ __device__ tensile_bfloat16 operator*(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) * float(b)); +} +inline __host__ __device__ tensile_bfloat16 operator/(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) / float(b)); +} +inline __host__ __device__ bool operator<(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return float(a) < float(b); +} +inline __host__ __device__ bool operator==(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return float(a) == float(b); +} +inline __host__ __device__ bool operator>(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return b < a; +} +inline __host__ __device__ bool operator<=(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return !(a > b); +} +inline __host__ __device__ bool operator!=(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return !(a == b); +} +inline __host__ __device__ bool operator>=(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return !(a < b); +} +inline __host__ __device__ tensile_bfloat16& operator+=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a + b; +} +inline __host__ __device__ tensile_bfloat16& operator-=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a - b; +} +inline __host__ __device__ tensile_bfloat16& operator*=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a * b; +} +inline __host__ __device__ tensile_bfloat16& operator/=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a / b; +} +inline __host__ __device__ tensile_bfloat16& operator++(tensile_bfloat16& a) +{ + return a += tensile_bfloat16(1.0f); +} +inline __host__ __device__ tensile_bfloat16& operator--(tensile_bfloat16& a) +{ + return a -= tensile_bfloat16(1.0f); +} +inline __host__ __device__ tensile_bfloat16 operator++(tensile_bfloat16& a, int) +{ + tensile_bfloat16 orig = a; + ++a; + return orig; +} +inline __host__ __device__ tensile_bfloat16 operator--(tensile_bfloat16& a, int) +{ + tensile_bfloat16 orig = a; + --a; + return orig; +} +inline __host__ __device__ bool isinf(tensile_bfloat16 a) +{ + return !(~a.data & 0x7f80) && !(a.data & 0x7f); +} +inline __host__ __device__ bool isnan(tensile_bfloat16 a) +{ + return !(~a.data & 0x7f80) && +(a.data & 0x7f); +} +inline __host__ __device__ bool iszero(tensile_bfloat16 a) +{ + return !(a.data & 0x7fff); +} +inline tensile_bfloat16 sin(tensile_bfloat16 a) +{ + return tensile_bfloat16(sinf(float(a))); +} +inline tensile_bfloat16 cos(tensile_bfloat16 a) +{ + return tensile_bfloat16(cosf(float(a))); +} + +#endif // __cplusplus < 201103L || !defined(__HIPCC__) + +#endif // _TENSILE_BFLOAT16_H_ diff --git a/tensilelite/Tensile/Source/winners.awk b/tensilelite/Tensile/Source/winners.awk new file mode 100644 index 0000000000..25932bac94 --- /dev/null +++ b/tensilelite/Tensile/Source/winners.awk @@ -0,0 +1,42 @@ +################################################################################ +# Copyright 2017-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +######################################## +# Usage: awk -f winners.awk out.txt +######################################## + +BEGIN { + problemLine = " ---Problem---" + winnerLine = " ---Winner---" +} + +/Problem\[/ { + printf "%-40s - %s\n", problemLine, winnerLine + problemLine = $0 +} + +/\*,/ { + winnerLine = $0 +} + +END { + printf "%-40s - %s\n", problemLine, winnerLine +} diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py new file mode 100644 index 0000000000..a6b6d4d7f5 --- /dev/null +++ b/tensilelite/Tensile/Tensile.py @@ -0,0 +1,329 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +if __name__ == "__main__": + print("This file can no longer be run as a script. Run 'Tensile/bin/Tensile' instead.") + exit(1) + +import os +import sys +import argparse +from .Common import globalParameters, print1, printExit, ensurePath, \ + assignGlobalParameters, restoreDefaultGlobalParameters, HR +from . import BenchmarkProblems +from . import ClientWriter +from . import LibraryIO +from . import LibraryLogic +from . import __version__ +from datetime import datetime + + +############################################################################### +# Execute Steps in Config +# called from Tensile() below +# calls +# BenchmarkProblems.main() to run benchmark steps +# LibraryLogic.main() to analyse final benchmark data and produce logic/yaml +# ClientWriter.main() to create client which calls library based on above yaml +################################################################################ +def executeStepsInConfig(config): + + ############################################################################## + # Benchmark Problems + ############################################################################## + if "BenchmarkProblems" in config: + BenchmarkProblems.main(config["BenchmarkProblems"], config["UseCache"]) + print1("") + + ############################################################################## + # Library Logic + ############################################################################## + libraryLogicDataPath = os.path.join(globalParameters["WorkingPath"], \ + globalParameters["LibraryLogicPath"]) + if "LibraryLogic" in config: + if os.path.exists(libraryLogicDataPath): + libraryLogicFiles = os.listdir(libraryLogicDataPath) + else: + libraryLogicFiles = [] + if len(libraryLogicFiles) < 1 or globalParameters["ForceRedoLibraryLogic"]: + if config["LibraryLogic"] != None: + libraryLogicConfig = config["LibraryLogic"] + else: + libraryLogicConfig = {} + LibraryLogic.main(libraryLogicConfig) + print1("") + else: + print1("# LibraryLogic already done.") + print1("") + + ############################################################################## + # Write Client + ############################################################################## + if "LibraryClient" in config: + if config["LibraryClient"] != None: + libraryClientConfig = config["LibraryClient"] + else: + libraryClientConfig = {} + ClientWriter.main(libraryClientConfig) + print1("") + + +def addCommonArguments(argParser): + """ + Add a common set of arguments to `argParser`. + + Currently used by the main Tensile script and the unit tests but could also be used for TensileCreateLibrary. + """ + + def splitExtraParameters(par): + """ + Allows the --global-parameters option to specify any parameters from the command line. + """ + (key, value) = par.split("=") + value = eval(value) + return (key, value) + + argParser.add_argument("-d", "--device", dest="device", type=int, \ + help="override which device to benchmark") + argParser.add_argument("-p", "--platform", dest="platform", type=int, \ + help="override which OpenCL platform to benchmark") + argParser.add_argument("--runtime-language", dest="RuntimeLanguage", \ + choices=["HIP", "OCL"], help="override which runtime language to use") + argParser.add_argument("--code-object-version", dest="CodeObjectVersion", \ + choices=["V2", "V3"], default="V3", help="HSA code-object version") + argParser.add_argument("-v", "--verbose", action="store_true", \ + help="set PrintLevel=2") + argParser.add_argument("--debug", dest="debug", action="store_true", \ + help="set PrintLevel=2 and CMakeBuildType=Debug") + argParser.add_argument("--short-names", dest="shortNames", action="store_true", \ + help="use serial kernel and solution names") + argParser.add_argument("--no-merge-files", dest="noMergeFiles", action="store_true", \ + help="kernels and solutions written to individual files") + argParser.add_argument("--cxx-compiler", dest="CxxCompiler", choices=["hipcc"], \ + action="store", default="hipcc", help="select which compiler to use") + argParser.add_argument("--library-format", dest="LibraryFormat", choices=["yaml", "msgpack"], \ + action="store", default="yaml", help="select which library format to use") + argParser.add_argument("--client-build-path", default=None) + argParser.add_argument("--client-lock", default=None) + argParser.add_argument("--prebuilt-client", default=None) + + argParser.add_argument("--global-parameters", nargs="+", type=splitExtraParameters, default=[]) + + +def argUpdatedGlobalParameters(args): + """ + Returns a dictionary with `globalParameters` keys that should be updated based on `args`. + """ + rv = {} + # override config with command-line options + if args.device: + print1("# Command-line override: Device") + rv["Device"] = args.device + if args.platform: + print1("# Command-line override: Platform") + rv["Platform"] = args.platform + if args.RuntimeLanguage: + print1("# Command-line override: RuntimeLanguage") + rv["RuntimeLanguage"] = args.RuntimeLanguage + if args.CodeObjectVersion: + print1("# Command-line override: CodeObjectVersion") + rv["CodeObjectVersion"] = args.CodeObjectVersion + if args.verbose: + print1("# Command-line override: PrintLevel") + rv["PrintLevel"] = 2 + if args.debug: + print1("# Command-line override: Debug") + rv["PrintLevel"] = 2 + rv["CMakeBuildType"] = "Debug" + if args.shortNames: + rv["ShortNames"] = True + if args.noMergeFiles: + rv["MergeFiles"] = False + if args.CxxCompiler: + rv['CxxCompiler'] = args.CxxCompiler + if rv['CxxCompiler'] == "hipcc" and not args.CodeObjectVersion: + rv["CodeObjectVersion"] = "V3" + print1("") + if args.client_build_path: + rv["ClientBuildPath"] = args.client_build_path + if args.client_lock: + rv["ClientExecutionLockPath"] = args.client_lock + if args.prebuilt_client: + rv["PrebuiltClient"] = args.prebuilt_client + + for key, value in args.global_parameters: + rv[key] = value + + return rv + + +################################################################################ +# Tensile +# - below entry points call here +################################################################################ +def Tensile(userArgs): + global globalParameters + + # 1st half of splash + print1("") + print1(HR) + print1("#") + print1("# Tensile v%s" % (__version__)) + + # setup argument parser + # yapf: disable + argParser = argparse.ArgumentParser() + argParser.add_argument("config_file", type=os.path.realpath, nargs="+", + help="Benchmark config.yaml file") + argParser.add_argument("output_path", \ + help="Path to conduct benchmark and write output files") + argParser.add_argument("--version", action="version", \ + version="%(prog)s {version}".format(version=__version__)) + argParser.add_argument("--alternate-format", dest="AlternateFormat", action="store_true", + help="Alternate format for config_file(s): first file is alternate config " + "and optional second file is size list") + argParser.add_argument("--no-cache", dest="NoCache", action="store_true", + help="Ignore cache; redo parameter forking and solution generation") + # yapf: enable + + addCommonArguments(argParser) + args = argParser.parse_args(userArgs) + + configPaths = args.config_file + altFormat = args.AlternateFormat + useCache = not args.NoCache + + if altFormat and len(configPaths) > 2: + printExit("Only 1 or 2 config_files are accepted for the alternate config format: " + "the alternate config file and an optional size list") + elif not altFormat and len(configPaths) != 1: + printExit("Only 1 config_file is accepted for the default config format. " + "Did you mean to add '--alternate-formate'?") + + # 2nd half of splash + if len(configPaths) == 1: + print1("# Config: {}".format(configPaths[0])) + else: + print1("# Configs: {} and {}".format(configPaths[0], configPaths[1])) + print1("# Date & Time: %s" % (datetime.now().strftime("%d/%m/%Y %H:%M:%S"))) + print1("#") + print1(HR) + print1("") + + print1("# Restoring default globalParameters") + restoreDefaultGlobalParameters() + + # CxxCompiler and LibraryFormat needs to be updated before assignGlobalParameters. + if args.CxxCompiler: + globalParameters['CxxCompiler'] = args.CxxCompiler + if args.LibraryFormat: + globalParameters['LibraryFormat'] = args.LibraryFormat + + # default config format + if not altFormat: + config = LibraryIO.readYAML(configPaths[0]) + # convert alternate format into default format + else: + base = LibraryIO.readYAML(configPaths[0]) + sizes = [] + if len(configPaths) == 2: + sizes = LibraryIO.readYAML(configPaths[1]) + + config = {"GlobalParameters": base.get("GlobalParameters")} + if "LibraryLogic" in base and len(sizes) > 0: + config["LibraryLogic"] = base["LibraryLogic"] + if "LibraryClient" in base and len(sizes) > 0: + config["LibraryClient"] = None + + solParams = { + "BenchmarkCommonParameters": base.get("BenchmarkCommonParameters"), + "ForkParameters": base.get("ForkParameters"), + "GroupForkParameters": base.get("GroupForkParameters"), + "BenchmarkFinalParameters": [{ + "ProblemSizes": sizes + }] + } + config["BenchmarkProblems"] = [[base["ProblemType"], solParams]] + + config["UseCache"] = useCache + globalParameters["ConfigPath"] = configPaths + + # assign global parameters + if "GlobalParameters" in config: + assignGlobalParameters(config["GlobalParameters"]) + else: + assignGlobalParameters({}) + + globalParameters["OutputPath"] = ensurePath(os.path.abspath(args.output_path)) + globalParameters["WorkingPath"] = globalParameters["OutputPath"] + + overrideParameters = argUpdatedGlobalParameters(args) + + for key, value in overrideParameters.items(): + print("Overriding {0}={1}".format(key, value)) + globalParameters[key] = value + + # Execute Steps in the config script + executeStepsInConfig(config) + + +def TensileConfigPath(*args): + return os.path.join(os.path.dirname(os.path.realpath(__file__)), "Configs", *args) + + +def TensileTestPath(*args): + return os.path.join(os.path.dirname(os.path.realpath(__file__)), "Tests", *args) + + +################################################################################ +# Entry points +# the first several of these can be deprecated, only main() is used +################################################################################ + + +# installed "tensile_rocblas_sgemm" command +def TensileROCBLASSGEMM(): + Tensile([TensileConfigPath("rocblas_sgemm.yaml"), "."]) + + +# installed "tensile_rocblas_dgemm" command +def TensileROCBLASDGEMM(): + Tensile([TensileConfigPath("rocblas_dgemm.yaml"), "."]) + + +# installed "tensile_rocblas_cgemm" command +def TensileROCBLASCGEMM(): + Tensile([TensileConfigPath("rocblas_cgemm.yaml"), "."]) + + +# installed "tensile_rocblas_zgemm" command +def TensileROCBLASZGEMM(): + Tensile([TensileConfigPath("rocblas_zgemm.yaml"), "."]) + + +# installed "tensile_sgemm" command +def TensileSGEMM5760(): + Tensile([TensileConfigPath("sgemm_5760.yaml"), "."]) + + +# installed "tensile" command +def main(): + Tensile(sys.argv[1:]) diff --git a/tensilelite/Tensile/TensileBenchmarkCluster.py b/tensilelite/Tensile/TensileBenchmarkCluster.py new file mode 100644 index 0000000000..3ba7a9bf04 --- /dev/null +++ b/tensilelite/Tensile/TensileBenchmarkCluster.py @@ -0,0 +1,483 @@ +################################################################################ +# Copyright 2016-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +import shlex, subprocess +import sys +import os +import argparse + +from .BenchmarkSplitter import BenchmarkSplitter +from .Configuration import ProjectConfig +from .TensileBenchmarkClusterScripts import ScriptWriter +from Tensile.Utilities.merge import mergePartialLogics + +try: + import mgzip as gzip +except ImportError: + print("Package mgzip not found: docker zipping will be slow. Install pip3 install mgzip to improve performance.") + + # Fallback package import + import gzip + +class BenchmarkImplSLURM(object): + + # baseImage: remote URL to rocm build + # dockerFilePath: path to docker file for slurm tensile build + # tag: name:TAG of new image + # outDir: where to export the new image + # tensile: tensile code to use on github + @staticmethod + def __createTensileBenchmarkContainer(baseImage, dockerFilePath, tag, outDir, logDir, tensileFork, tensileBranch, tensileCommit): + """ + Build a docker container with a specific + ROCm image, Tensile branch and tag. Docker + will pre-build the Tensile Client and + configure the container to run the benchmark. + """ + # Save stdout and stderr to file + with open(os.path.join(logDir, "dockerBuildLog.log"), 'w') as logFile: + + # Docker build command + buildCmd = str("\ + docker build \ + -t {0} \ + --pull -f {1} \ + --build-arg user_uid=$UID \ + --build-arg base_image={2} \ + --build-arg tensile_fork={3} \ + --build-arg tensile_branch={4} \ + --build-arg tensile_commit={5} \ + . ").format(tag, dockerFilePath, baseImage, tensileFork, tensileBranch, tensileCommit) + + # Build container and save output streams + print("Building docker image: {0} ...".format(tag)) + print(buildCmd) + subprocess.check_call(shlex.split(buildCmd), stdout=logFile, stderr=logFile) + print("Done building docker image!") + + # Docker save command + imageBaseName = tag.split(':')[0] + archivePath = os.path.join(outDir, imageBaseName + str(".tar.gz")) + saveCmd = str("docker save {0}").format(tag) + + # Docker will save .tar binary to stdout as long as it's not attached to console. + # Pipe stdout into gzip to get smaller .tar.gz archive + print("Saving docker image: {0} to {1} ...".format(tag, archivePath)) + + with gzip.open(archivePath, 'wb') as zipFile: + with subprocess.Popen(shlex.split(saveCmd), stdout=subprocess.PIPE, stderr=logFile) as proc: + zipFile.write(proc.stdout.read()) + + print("Done saving docker image!") + + @staticmethod + def __createClusterBenchmarkScripts(baseDir, tasksDir, taskScriptName, jobScriptName): + """ + Build SLURM specific scripts. + Entrypoint is the 'sbatch' command, which will request resources and + enqueue several taskScripts. + TaskScripts each enqueue a benchmark run to + the cluster with 'srun'. + NodeScripts are the actual host code on each node to run the benchmark. + In this case, they will load the docker image and mount directories + that contain the .yaml file, and the results. + """ + # Entrypoint - SLURM specific + ScriptWriter.writeBenchmarkJobScript(jobScriptName, baseDir) + + # Job submission script - SLURM specific + ScriptWriter.writeBenchmarkTaskScript(taskScriptName, baseDir) + + # Move each individual config into it's own subfolder and create the node host script + # These scripts are NOT SLURM specific but rely on the implementation of the executable. + configFiles = [f for f in os.listdir(tasksDir) if os.path.isfile(os.path.join(tasksDir, f))] + for f in configFiles: + (baseFileName, _) = os.path.splitext(f) + configSubdir = os.path.join(tasksDir, baseFileName) + try: + os.makedirs(configSubdir) + except: + pass + os.rename(os.path.join(tasksDir, f), os.path.join(configSubdir, f)) + ScriptWriter.writeBenchmarkNodeScript(baseFileName + ".sh", configSubdir) + + @classmethod + def initializeConfig(cls, config): + """ + Store SLURM backend-specific configurations. + These can all be overridden via commandline + """ + # Dirs + rootTensileDir = config["RootTensileDir"] + + # SLURM config + baseSection = config.createSection("SLURM") + + # Script names + section = baseSection.createSection("SCRIPTS") + section.createValue("JobScriptName", "runBenchmark.sh") + section.createValue("TaskScriptName", "enqueueTask.sh") + + # Docker build + section = baseSection.createSection("DOCKER") + section.createValue("DockerBaseImage", "compute-artifactory.amd.com:5000/rocm-plus-docker/compute-rocm-dkms-amd-feature-targetid:3004-STG1") + section.createValue("DockerBuildFile", os.path.join(rootTensileDir, "docker", "dockerfile-tensile-tuning-slurm")) + section.createValue("DockerImageName", "tensile-tuning-cluster-executable") + section.createValue("DockerImageTag", "TEST") + section.createValue("TensileFork", "ROCmSoftwarePlatform") + section.createValue("TensileBranch", "develop") + section.createValue("TensileCommit", "HEAD") + + @classmethod + def generateBenchmark(cls, config): + """ + Build executables used in SLURM backend, + based on configuration parameters + """ + # Dirs + (baseDir, tasksDir, imageDir, logsDir) = \ + (config["BenchmarkBaseDir"], \ + config["BenchmarkTasksDir"], \ + config["BenchmarkImageDir"], \ + config["BenchmarkLogsDir"]) + + # Create the base image for task executable in image dir + sConfig = config["SLURM"]["DOCKER"] + cls.__createTensileBenchmarkContainer( \ + sConfig["DockerBaseImage"], \ + sConfig["DockerBuildFile"], \ + "{0}:{1}".format(sConfig["DockerImageName"], sConfig["DockerImageTag"]), \ + imageDir, \ + logsDir, \ + sConfig["TensileFork"], \ + sConfig["TensileBranch"], \ + sConfig["TensileCommit"]) + + # Create the scripts that will be used to invoke the benchmark in base dir + # Create the scripts that node hosts will run in the tasks dir + sConfig = config["SLURM"]["SCRIPTS"] + cls.__createClusterBenchmarkScripts( \ + baseDir, \ + tasksDir, \ + sConfig["TaskScriptName"], \ + sConfig["JobScriptName"]) + + @classmethod + def preInvokeBenchmark(cls, config): + pass + + @classmethod + def invokeBenchmark(cls, config): + + # Dirs + (baseDir, tasksDir, imageDir, resultsDir, logsDir) = \ + (config["BenchmarkBaseDir"], \ + config["BenchmarkTasksDir"], \ + config["BenchmarkImageDir"], \ + config["BenchmarkResultsDir"], \ + config["BenchmarkLogsDir"]) + + # Entry point script + scriptName = config["SLURM"]["SCRIPTS"]["JobScriptName"] + runScriptPath = os.path.join(baseDir, scriptName) + + # Enqueue script + enqueueScriptName = config["SLURM"]["SCRIPTS"]["TaskScriptName"] + enqueueScriptPath = os.path.join(baseDir, enqueueScriptName) + + # Log file + (name, ext) = os.path.splitext(scriptName) + logFilePath = os.path.join(logsDir, name + ".log") + + invokeCmd = str("\ + {0} \ + -i {1} \ + -l {2} \ + -r {3} \ + -s {4} \ + -t {5}").format(runScriptPath, imageDir, logsDir, resultsDir, enqueueScriptPath, tasksDir) + + with open(logFilePath, "wt") as logFile: + subprocess.check_call(shlex.split(invokeCmd), stdout=logFile, stderr=logFile) + + @classmethod + def postInvokeBenchmark(cls, benchmarkObj): + pass + + +class TensileBenchmarkCluster(object): + """ + This is the main driver class for building and running + benchmarks with a cluster-configured backend. + + The general workflow (main): + - Deploy benchmark + - Invoke benchmark + - Merge results + + Expected backend interface: + - initializeConfig + - generateBenchmark + - preInvokeBenchmark + - invokeBenchmark + - postInvokeBenchmark + + Other notes: + - The command line provides full control over + workflow and configuration. + - You can choose different backends as they + become available + - Workflow steps may be selected individually + - Any part of configuration may be overidden + by command line args. + """ + + def __init__(self, cmdlineArgs): + self._config = ProjectConfig() + self.__initializeConfig(cmdlineArgs) + + def __initializeConfig(self, cmdlineArgs): + + """ + Initialize configuration parameters needed + to build and run the benchmark. + """ + + # 1. Parse command line + args = self.__parseArgs(cmdlineArgs) + + # 2. Setup default configurations that depend on inputs + # Directories + self._config.createValue("RootTensileDir", os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + self._config.createValue("BenchmarkLogicPath", args.BenchmarkLogicPath) + self._config.createValue("BenchmarkBaseDir", args.DeploymentPath) + self._config.createValue("BenchmarkTasksDir", os.path.join(args.DeploymentPath, "Tasks")) + self._config.createValue("BenchmarkImageDir", os.path.join(args.DeploymentPath, "Image")) + self._config.createValue("BenchmarkResultsDir", os.path.join(args.DeploymentPath, "Results")) + self._config.createValue("BenchmarkFinalLogicDir", os.path.join(args.DeploymentPath, "Results", "Final")) + self._config.createValue("BenchmarkLogsDir", os.path.join(args.DeploymentPath, "Logs")) + + # Benchmarking + self._config.createValue("BenchmarkTaskSize", 10) # Sizes per benchmark file + self._config.addConstraint("BenchmarkTaskSize > 0") + + self._config.createValue("RunDeployStep", True and (not args.RunOnly and not args.ResultsOnly and not args.RunAndResultsOnly)) + self._config.createValue("RunBenchmarkStep", True and (not args.DeployOnly and not args.ResultsOnly)) + self._config.createValue("RunResultsStep", True and (not args.DeployOnly and not args.RunOnly)) + self._config.addConstraint("RunDeployStep or RunBenchmarkStep or RunResultsStep") + + # Results merging settings + self._config.createValue("FinalLogicForceMerge", False) + self._config.createValue("FinalLogicTrim", True) + + # Initialize the backend implementation + if args.ClusterBackend.lower() == "slurm": + self._backendImpl = BenchmarkImplSLURM + else: + raise NotImplementedError("Cluster backend not recognized") + + # 3. Setup backend configuration + self._backendImpl.initializeConfig(self._config) + + # 4. Override requested config parameters + overrideArgs = {} + for key, value in args.benchmark_parameters: + overrideArgs[key] = value + self.__overrideConfig(overrideArgs) + + self._config.checkConstraints() + + @staticmethod + def __parseArgs(cmdlineArgs): + """ + Parse input arguments to call script + """ + + def splitExtraParameters(par): + """ + Allows the --benchmark-parameters option to specify any parameters from the command line. + """ + (key, value) = par.split("=") + value = eval(value) + return (key, value) + + # Parse incoming args + argParser = argparse.ArgumentParser() + argParser.add_argument("BenchmarkLogicPath", help="Path to benchmark config .yaml files.") + argParser.add_argument("DeploymentPath", help="Where to deploy benchmarking files. Should target a directory on shared nfs mount of cluster.") + argParser.add_argument("--cluster-backend", dest="ClusterBackend", type=str, default="slurm", help="Choose backend plugin to run benchmark") + argParser.add_argument("--deploy-only", dest="DeployOnly", action="store_true", default=False, help="Deploy benchmarking files only without running or reducing results") + argParser.add_argument("--run-only", dest="RunOnly", action="store_true", default=False, help="Run benchmark without deploying or reducing results") + argParser.add_argument("--results-only", dest="ResultsOnly", action="store_true", default=False, help="Reduce results without deploying or running") + argParser.add_argument("--run-and-results-only", dest="RunAndResultsOnly", action="store_true", default=False, help="Run benchmark and reduce results without deploying") + argParser.add_argument("--benchmark-parameters", nargs="+", type=splitExtraParameters, default=[], help="Pairs of X=Y assignments. Note: if Y is a string, then it must have escaped quotes \\\"Y\\\"") + return argParser.parse_args() + + def __overrideConfig(self, args): + """ + Assign Benchmark Parameters + Each parameter has a default parameter, and the user + can override them, those overridings happen here + """ + for key in args: + value = args[key] + if key not in self._config: + print("Warning: Benchmark parameter {0} = {1} unrecognised.".format( key, value )) + else: + self._config[key] = value + + def __generateClusterBenchmark(self): + """ + Generate everything for the benchmark to run + """ + # Ensure that we have all directories + self.ensurePath(self.baseDir()) + self.ensurePath(self.tasksDir()) + self.ensurePath(self.imageDir()) + self.ensurePath(self.resultsDir()) + self.ensurePath(self.finalLogicDir()) + self.ensurePath(self.logsDir()) + + # Split master config into smaller task-sized configs + # These are stored under the tasks dir + BenchmarkSplitter.splitBenchmarkBySizes( \ + self._config["BenchmarkLogicPath"], \ + self.tasksDir(), \ + self._config["BenchmarkTaskSize"], \ + suffixFormat="{:04d}") # Support lots of jobs up to 9999 + + # Delegate to the backend implementation to generate everything it needs for the benchmark run + self._backendImpl.generateBenchmark(self._config) + + def __runClusterBenchmark(self): + """ + Invoke backend benchmark + """ + # Delegate to the backend implementation to invoke + self._backendImpl.preInvokeBenchmark(self._config) + self._backendImpl.invokeBenchmark(self._config) + self._backendImpl.postInvokeBenchmark(self._config) + + def __combineClusterBenchmarkResults(self): + """ + Combine all partial results into the final + kernel logic. + Each partial result is expected to be under + the base/ResultsDir//3_LibraryLogic + directory. + """ + (resultsDir, finalLogicDir) = (self.resultsDir(), self.finalLogicDir()) + + # Find the partial logic .yaml files + # These are in each result directory under 3_LibraryLogic + resultsDirs = [os.path.join(resultsDir, d, "3_LibraryLogic") for d in os.listdir(resultsDir) if os.path.isdir(os.path.join(resultsDir, d, "3_LibraryLogic"))] + + resultsFiles = [] + for d in resultsDirs: + resultsFiles += [os.path.join(d, f) for f in os.listdir(d) if os.path.isfile(os.path.join(d, f)) ] + + if len(resultsDirs) != len(resultsFiles): + print("Warning: inconsistent number of expected results. Check that results are complete.") + + mergePartialLogics( + resultsFiles, \ + finalLogicDir, \ + self._config["FinalLogicForceMerge"], \ + self._config["FinalLogicTrim"]) + + def workflowSteps(self): + """ + Helper function to easily receive a tuple of important + workflow step indicators + """ + return( \ + self._config["RunDeployStep"], \ + self._config["RunBenchmarkStep"], \ + self._config["RunResultsStep"]) + + def config(self): + return self._config + + def rootTensileDir(self): + return self.config()["RootTensileDir"] + + def baseDir(self): + return self.config()["BenchmarkBaseDir"] + + def tasksDir(self): + return self.config()["BenchmarkTasksDir"] + + def imageDir(self): + return self.config()["BenchmarkImageDir"] + + def resultsDir(self): + return self.config()["BenchmarkResultsDir"] + + def finalLogicDir(self): + return self.config()["BenchmarkFinalLogicDir"] + + def logsDir(self): + return self.config()["BenchmarkLogsDir"] + + @staticmethod + def ensurePath(path): + """ + Helper function to create path if necessary + """ + try: + os.makedirs(path) + except OSError: + pass + return path + + def main(self): + + """ + Main driver for benchmark: + - Deploy + - Run + - Results + """ + (doDeploy, doBenchmark, doResults) = self.workflowSteps() + + # Deploy + if doDeploy is True: + print("Preparing benchmarking files...") + self.__generateClusterBenchmark() + print("Finished preparing benchmarking files") + + # Benchmark invoke + if doBenchmark is True: + print("Running benchmark tasks (this might take a while)...") + self.__runClusterBenchmark() + print("Finished benchmark tasks") + + # Combining results + if doResults is True: + print("Combining benchmark results...") + self.__combineClusterBenchmarkResults() + print("Final logic file saved to: {0}".format(self.finalLogicDir())) + + print("Finished") + +def main(): + entryPoint = TensileBenchmarkCluster(sys.argv[1:]) + entryPoint.main() diff --git a/tensilelite/Tensile/TensileBenchmarkClusterScripts.py b/tensilelite/Tensile/TensileBenchmarkClusterScripts.py new file mode 100644 index 0000000000..8c0351051b --- /dev/null +++ b/tensilelite/Tensile/TensileBenchmarkClusterScripts.py @@ -0,0 +1,372 @@ +import os + +class ScriptHelper(object): + """ + Helper class to facilitate formatting when + writing bash scripts. + """ + + @classmethod + def genLine(cls, text): + return str("{0}\n").format(text) + + @classmethod + def genComment(cls, text): + return cls.genLine(str("#") + str(text)) + + @classmethod + def genEcho(cls, text): + return cls.genLine(str("echo \"{0}\";").format(str(text))) + + @classmethod + def makeExecutable(cls, path): + mode = os.stat(path).st_mode + mode |= (mode & 0o444) >> 2 # copy R bits to X + os.chmod(path, mode) + +class ScriptWriter(object): + """ + A script writing utility class that holds a string + buffer and wraps writing to the buffer. + Has factory functions to build scripts with regular + workflow interface. + """ + def __init__(self): + self._buffer = "" + + def writeLine(self, text): + self._buffer += ScriptHelper.genLine(text) + + def writeComment(self, text): + self._buffer += ScriptHelper.genComment(text) + + def writeEcho(self, text): + self._buffer += ScriptHelper.genEcho(text) + + def dumpToFile(self, fileName, outputDir = "."): + scriptPath = os.path.join(outputDir, fileName) + with open(scriptPath, "w") as f: + f.write(self._buffer) + ScriptHelper.makeExecutable(scriptPath) + + def __str__(self): + return self._buffer + + @classmethod + def writeScript(bufferClass, writerClass, fileName, outputDir = "."): + buffer = bufferClass() + writerClass.writeHeader(buffer) + writerClass.writeBody(buffer) + writerClass.writeEpilogue(buffer) + buffer.dumpToFile(fileName, outputDir) + + @classmethod + def previewScript(bufferClass, writerClass): + buffer = bufferClass() + writerClass.writeHeader(buffer) + writerClass.writeBody(buffer) + writerClass.writeEpilogue(buffer) + print(buffer) + + @classmethod + def writeBenchmarkNodeScript(cls, fileName, outputDir="."): + cls.writeScript(BenchmarkNodeWriter, fileName, outputDir) + + @classmethod + def writeBenchmarkTaskScript(cls, fileName, outputDir="."): + cls.writeScript(BenchmarkTaskWriter, fileName, outputDir) + + @classmethod + def writeBenchmarkJobScript(cls, fileName, outputDir="."): + cls.writeScript(BenchmarkJobWriter, fileName, outputDir) + + +class BenchmarkNodeWriter(object): + """ + Node host run script. + Creates an instance of the docker image and + mounts the logs, results and tasks directories + and finally invokes the benchmark on the + supplied yaml. + """ + @staticmethod + def __initHeader(sBuffer): + sBuffer.writeComment("!/bin/bash") + sBuffer.writeComment("Benchmark node script") + sBuffer.writeComment("Actual host task to be run by a SLURM worker node") + sBuffer.writeLine("") + + @staticmethod + def __initializeFuncs(sBuffer): + sBuffer.writeComment("Funcs") + sBuffer.writeLine("usage() { echo \"Usage: $0 [-i ] [-l ] [-r ] [-t ]\" 1>&2; exit 1; }") + sBuffer.writeLine("failAndExit() { echo \"FAILED: $1\" 1>&2; exit 1; }") + sBuffer.writeLine("") + + @staticmethod + def __parseArgs(sBuffer): + sBuffer.writeComment("Parse arguments") + sBuffer.writeLine("while getopts i:l:r:t: flag") + sBuffer.writeLine("do") + sBuffer.writeLine(" case \"${flag}\" in") + sBuffer.writeLine(" i) dockerImagePath=${OPTARG};;") + sBuffer.writeLine(" l) logDir=${OPTARG};;") + sBuffer.writeLine(" r) resultDir=${OPTARG};;") + sBuffer.writeLine(" t) taskDir=${OPTARG};;") + sBuffer.writeLine(" *) usage;;") + sBuffer.writeLine(" esac") + sBuffer.writeLine("done") + sBuffer.writeLine("[ -z \"${dockerImagePath}\" ] || [ -z \"${logDir}\" ] || [ -z \"${resultDir}\" ] || [ -z \"${taskDir}\" ] && usage;") + sBuffer.writeLine("[ -d \"${logDir}\" ] && [ -d \"${resultDir}\" ] && [ -d \"${taskDir}\" ] || failAndExit \"Directory not found\";") + sBuffer.writeLine("[ -f \"${dockerImagePath}\" ] || failAndExit \"Docker image not found: $dockerImagePath\";") + sBuffer.writeEcho("Docker image path: $dockerImagePath") + sBuffer.writeEcho("Path to log: $logDir") + sBuffer.writeEcho("Path to task: $taskDir") + sBuffer.writeEcho("Path to result: $resultDir") + sBuffer.writeLine("") + + @staticmethod + def __loadDockerImage(sBuffer): + sBuffer.writeComment("Import docker image") + sBuffer.writeComment("Successful output is \"Loaded image: imageName:TAG\"") + sBuffer.writeEcho("Loading docker image...") + sBuffer.writeLine("dockerLoadOutput=`docker load < $dockerImagePath || failAndExit \"Importing docker image\"`") + sBuffer.writeLine("") + sBuffer.writeEcho("Docker Load Result: $dockerLoadOutput") + sBuffer.writeLine("[ -z \"${dockerLoadOutput}\" ] && failAndExit \"Loading docker image $dockerImagePath\"") + sBuffer.writeLine("") + sBuffer.writeComment("Split the output on ':' or spaces") + sBuffer.writeComment("Capture the docker image name and tag") + sBuffer.writeLine("outputSplit=(${dockerLoadOutput//:/ })") + sBuffer.writeLine("dockerName=${outputSplit[2]}") + sBuffer.writeLine("dockerTag=${outputSplit[3]}") + sBuffer.writeLine("") + sBuffer.writeComment("Find docker image record with correct name and tag") + sBuffer.writeLine("dockerImageQuery=`docker images | grep -i \"$dockerName\" | grep -i \"$dockerTag\"`") + sBuffer.writeLine("[ -z \"${dockerImageQuery}\" ] && failAndExit \"Finding docker image record for $dockerName:$dockerTag\"") + sBuffer.writeLine("") + sBuffer.writeComment("Get docker image ID") + sBuffer.writeLine("outputSplit=(${dockerImageQuery// / })") + sBuffer.writeLine("dockerImageId=${outputSplit[2]}") + sBuffer.writeLine("[ -z \"${dockerImageId}\" ] && failAndExit \"Finding docker image id for $dockerName:$dockerTag\"") + sBuffer.writeEcho("Success: Loaded Image ID: ${dockerImageId}") + sBuffer.writeLine("") + + @staticmethod + def __runTask(sBuffer): + sBuffer.writeComment("Check the task dir exists") + sBuffer.writeLine("") + sBuffer.writeComment("Run container with mounted taskDir") + sBuffer.writeEcho("Running container... (this might take some time):") + sBuffer.writeLine("runCmd=\"docker run --rm --network=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -v \"$taskDir\":\"/TaskDir\" -v \"$resultDir\":\"/ResultDir\" -v \"$logDir\":\"/LogDir\" $dockerImageId\"") + sBuffer.writeEcho("$runCmd") + sBuffer.writeLine("$runCmd") + sBuffer.writeLine("returnCode=$?") + + @staticmethod + def __exit(sBuffer): + sBuffer.writeEcho("Done!") + sBuffer.writeLine("exit $returnCode") + + @classmethod + def writeHeader(cls, sBuffer): + cls.__initHeader(sBuffer) + + @classmethod + def writeBody(cls, sBuffer): + cls.__initializeFuncs(sBuffer) + cls.__parseArgs(sBuffer) + cls.__loadDockerImage(sBuffer) + cls.__runTask(sBuffer) + + @classmethod + def writeEpilogue(cls, sBuffer): + cls.__exit(sBuffer) + +class BenchmarkTaskWriter(object): + """ + SLURM specific script. + Invokes the 'srun' command to queue a task. + SLURM environment variables tell us which + indexed task to run as part of the array. + The final task submitted is the node script + which will run on one of the hosts. + """ + @staticmethod + def __initHeader(sBuffer): + sBuffer.writeComment("!/bin/bash") + sBuffer.writeComment("Benchmark task enqueue script") + sBuffer.writeComment("This script will configure and commit a single task to the cluster.") + sBuffer.writeComment("Intended to be the run target of SLURM SBATCH multiplexer") + sBuffer.writeLine("") + + @staticmethod + def __initializeFuncs(sBuffer): + sBuffer.writeComment("Funcs") + sBuffer.writeLine("usage() { echo \"Usage: $0 [-i ] [-l ] [-r ] [-t ] \" 1>&2; exit 1; }") + sBuffer.writeLine("failAndExit() { echo \"FAILED: $1\" 1>&2; exit 1; }") + sBuffer.writeLine("") + + @staticmethod + def __parseArgs(sBuffer): + sBuffer.writeComment("Parse arguments") + sBuffer.writeLine("while getopts i:l:r:t: flag") + sBuffer.writeLine("do") + sBuffer.writeLine(" case \"${flag}\" in") + sBuffer.writeLine(" i) imageDir=${OPTARG};; ") + sBuffer.writeLine(" l) logsDir=${OPTARG};;") + sBuffer.writeLine(" r) resultsDir=${OPTARG};;") + sBuffer.writeLine(" t) tasksDir=${OPTARG};;") + sBuffer.writeLine(" *) usage;;") + sBuffer.writeLine(" esac") + sBuffer.writeLine("done") + sBuffer.writeLine("[ -z \"${imageDir}\" ] || [ -z \"${logsDir}\" ] || [ -z \"${resultsDir}\" ] || [ -z \"${tasksDir}\" ] && usage;") + sBuffer.writeLine("[ -d \"${imageDir}\" ] && [ -d \"${logsDir}\" ] && [ -d \"${resultsDir}\" ] && [ -d \"${tasksDir}\" ] || failAndExit \"Directory not found\";") + sBuffer.writeEcho("Image dir: $imageDir") + sBuffer.writeEcho("Logs dir: $logsDir") + sBuffer.writeEcho("Tasks dir: $tasksDir") + sBuffer.writeEcho("Results dir: $resultsDir") + sBuffer.writeLine("") + + @staticmethod + def __configureTask(sBuffer): + sBuffer.writeComment("Setup task parameters") + sBuffer.writeLine("pushd $tasksDir") + sBuffer.writeLine("subDirs=(*)") + sBuffer.writeLine("taskName=${subDirs[$SLURM_ARRAY_TASK_ID]}") + sBuffer.writeLine("taskDir=$tasksDir/$taskName") + sBuffer.writeLine("taskResultDir=$resultsDir/$taskName") + sBuffer.writeLine("mkdir -p $taskResultDir") + sBuffer.writeLine("taskLogDir=$logsDir/$taskName") + sBuffer.writeLine("mkdir -p $taskLogDir") + sBuffer.writeLine("task=($taskDir/*.sh)") + sBuffer.writeLine("image=($imageDir/*.tar.gz)") + sBuffer.writeEcho("Task ID: $SLURM_ARRAY_TASK_ID / $SLURM_TASK_MAX") + sBuffer.writeEcho("Task Name: $taskName") + sBuffer.writeEcho("Host Name: `hostname`") + sBuffer.writeLine("") + + @staticmethod + def __submitTask(sBuffer): + sBuffer.writeComment("Enqueue node task") + sBuffer.writeLine("srun -N 1 \"$task\" -i \"$image\" -l \"$taskLogDir\" -r \"$taskResultDir\" -t \"$taskDir\"") + sBuffer.writeLine("returnCode=$?") + + @staticmethod + def __exit(sBuffer): + sBuffer.writeEcho("Done!") + sBuffer.writeLine("exit $returnCode") + + @classmethod + def writeHeader(cls, sBuffer): + cls.__initHeader(sBuffer) + + @classmethod + def writeBody(cls, sBuffer): + cls.__initializeFuncs(sBuffer) + cls.__parseArgs(sBuffer) + cls.__configureTask(sBuffer) + cls.__submitTask(sBuffer) + + @classmethod + def writeEpilogue(cls, sBuffer): + cls.__exit(sBuffer) + +class BenchmarkJobWriter(object): + """ + SLURM specific script. + Invokes the 'sbatch' and allocates resources. + Will create an array of N tasks that will each + run one instance of the task script. + N = number of sub-folders inside task folder + Assumes one node per task. + Will wait until the entire batch is complete + before exiting. + NOTE: the wait feature requires that all task + scripts have a return code, or will hang forever! + """ + @staticmethod + def __initHeader(sBuffer): + sBuffer.writeComment("!/bin/bash") + sBuffer.writeComment("Benchmark batch job script") + sBuffer.writeComment("This script will configure and submit a batch job to the cluster") + sBuffer.writeComment("This is the SLURM cluster benchmark entry-point") + sBuffer.writeLine("") + + @staticmethod + def __initializeFuncs(sBuffer): + sBuffer.writeComment("Funcs") + sBuffer.writeLine("usage() { echo \"Usage: $0 [-i ] [-l ] [-r ] [-s ] [-t ] \" 1>&2; exit 1; }") + sBuffer.writeLine("failAndExit() { echo \"FAILED: $1\" 1>&2; exit 1; }") + sBuffer.writeLine("") + + @staticmethod + def __parseArgs(sBuffer): + sBuffer.writeComment("Parse arguments") + sBuffer.writeLine("while getopts i:l:r:s:t: flag") + sBuffer.writeLine("do") + sBuffer.writeLine(" case \"${flag}\" in") + sBuffer.writeLine(" i) imageDir=${OPTARG};;") + sBuffer.writeLine(" l) logsDir=${OPTARG};;") + sBuffer.writeLine(" r) resultsDir=${OPTARG};;") + sBuffer.writeLine(" s) taskScript=${OPTARG};;") + sBuffer.writeLine(" t) tasksDir=${OPTARG};;") + sBuffer.writeLine(" *) usage;;") + sBuffer.writeLine(" esac") + sBuffer.writeLine("done") + sBuffer.writeLine("[ -z \"${imageDir}\" ] || [ -z \"${logsDir}\" ] || [ -z \"${resultsDir}\" ] || [ -z \"${tasksDir}\" ] && usage;") + sBuffer.writeLine("[ -d \"${imageDir}\" ] && [ -d \"${logsDir}\" ] && [ -d \"${resultsDir}\" ] && [ -d \"${tasksDir}\" ] || failAndExit \"Directory not found\";") + sBuffer.writeLine("[ -f \"${taskScript}\" ] || failAndExit \"Task script not found\";") + sBuffer.writeEcho("Image dir: $imageDir") + sBuffer.writeEcho("Logs dir: $logsDir") + sBuffer.writeEcho("Tasks dir: $tasksDir") + sBuffer.writeEcho("Results dir: $resultsDir") + sBuffer.writeEcho("Enqueue script: $taskScript") + sBuffer.writeLine("") + + @staticmethod + def __configureJob(sBuffer): + sBuffer.writeComment("Setup local environment") + sBuffer.writeComment("Get in the directory where this script lives") + sBuffer.writeLine("pushd $( dirname `greadlink -f ${BASH_SOURCE[0]} || readlink -f ${BASH_SOURCE[0]}` )") + sBuffer.writeComment("Enumerate the tasks and set up batch params based on task count") + sBuffer.writeLine("subDirs=($tasksDir/*)") + sBuffer.writeLine("let arraySize=${#subDirs[@]}") + sBuffer.writeLine("[ -z \"${arraySize}\" ] && failAndExit \"No task count\";") + sBuffer.writeLine("let arrayStart=0") + sBuffer.writeLine("let arrayEnd=$arraySize-1") + sBuffer.writeComment("Create logs directory for SLURM") + sBuffer.writeLine("slurmLogsDir=$logsDir/SLURM") + sBuffer.writeLine("mkdir -p $slurmLogsDir") + sBuffer.writeLine("") + + @staticmethod + def __submitJob(sBuffer): + sBuffer.writeComment("Invoke batch request to SLURM") + sBuffer.writeComment("Will wait until the task completes before exiting") + sBuffer.writeLine("runCmd=\"sbatch --nodes=1 --array=$arrayStart-$arrayEnd -o $slurmLogsDir/slurm-%A_%a.out --wait $taskScript -i $imageDir -l $logsDir -r $resultsDir -t $tasksDir\"") + sBuffer.writeEcho("Submitting batch job to SLURM. This may take a while...") + sBuffer.writeEcho("$runCmd") + sBuffer.writeLine("$runCmd") + sBuffer.writeLine("returnCode=$?") + sBuffer.writeLine("popd") + + @staticmethod + def __exit(sBuffer): + sBuffer.writeEcho("Done!") + sBuffer.writeLine("exit $returnCode") + + @classmethod + def writeHeader(cls, sBuffer): + cls.__initHeader(sBuffer) + + @classmethod + def writeBody(cls, sBuffer): + cls.__initializeFuncs(sBuffer) + cls.__parseArgs(sBuffer) + cls.__configureJob(sBuffer) + cls.__submitJob(sBuffer) + + @classmethod + def writeEpilogue(cls, sBuffer): + cls.__exit(sBuffer) diff --git a/tensilelite/Tensile/TensileBenchmarkLibraryClient.py b/tensilelite/Tensile/TensileBenchmarkLibraryClient.py new file mode 100644 index 0000000000..ee08a17968 --- /dev/null +++ b/tensilelite/Tensile/TensileBenchmarkLibraryClient.py @@ -0,0 +1,181 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +import sys +import csv +import os +from subprocess import Popen, PIPE + +################################################################################ +# Benchmark Problem Size +################################################################################ +def BenchmarkProblemSize(cmdPrefix, row): + cmd = cmdPrefix + cmd += " --sizes" + for size in row: + cmd += " " + cmd += size.lstrip() + sys.stderr.write(cmd) + sys.stderr.write("\n") + process = Popen(cmd, stdout=PIPE, shell=True) + stdout = process.communicate()[0] + sys.stderr.write(stdout) + + # find beginning of data + initializingIdx = stdout.find("Initializing") + newLineIdx = stdout.find("\n", initializingIdx) + stdout = stdout[newLineIdx+1:] + + gflopList = [] + msList = [] + + # skip first b/c warmup/lookup + newLineIdx = stdout.find("\n") + stdout = stdout[newLineIdx+1:] + + # parse every line of data + while "\n" in stdout: + newLineIdx = stdout.find("\n") + lineString = stdout[:newLineIdx] + splits = lineString.split(",") + if len(splits) < 3: + break + gflopsString = splits[1].lstrip() + msString = splits[3].lstrip() + gflops = float(gflopsString) + ms = float(msString) + gflopList.append(gflops) + msList.append(ms) + + # next line + stdout = stdout[newLineIdx+1:] + return (gflopList, msList) + + +################################################################################ +# Print Stats +################################################################################ +def PrintStats(header, row, gflopList, msList): + meanGFlops = mean(gflopList) + meanMs = mean(msList) + medianGFlops = median(gflopList) + medianMs = median(msList) + stddevGFlops = stddev(gflopList) / meanGFlops + stddevMs = stddev(msList) / meanMs + + # format output + line = "" + for size in row: + line += "%6u, " % int(size) + line += "%9.2f, %9.4f, " % (medianGFlops, medianMs) + line += "%9.2f, %9.4f, " % (meanGFlops, meanMs) + line += "%9.5f, %9.5f" % (stddevGFlops, stddevMs) + sys.stdout.write(line) + sys.stdout.write("\n") + sys.stdout.flush() + sys.stderr.write("[STDOUT] %s\n" % header) + sys.stderr.write("[STDOUT] %s\n" % line) + sys.stderr.write("[END]\n\n\n") + sys.stderr.flush() + + +################################################################################ +# Benchmark Problem Sizes +################################################################################ +def TensileBenchmarkLibraryClient(userArgs): + if len(userArgs) < 2: + line = "USAGE: python TensileBenchmarkLibraryClient.py sizes.csv library_client_command > name.txt 2> name.raw.txt \n" + sys.stderr.write(line) + line = "Example: python TensileBenchmarkLibraryClient.py sizes.csv ./4_LibraryClient/build/client --function-idx 1 --num-benchmarks 100 --use-gpu-timer 0 > nn.txt 2> nn.raw.txt\n" + sys.stderr.write(line) + line = "[BE SURE TO PIN CLOCKS]\n" + sys.stderr.write(line) + exit(-1) + + # parse problem sizes path + problemSizesPath = os.path.realpath( userArgs[0] ) + line = "ProblemSizesPath: %s\n" % problemSizesPath + sys.stdout.write(line) + sys.stderr.write(line) + + # parse library client command + libraryClientCommandList = userArgs[1:] + libraryClientCommand = " ".join(libraryClientCommandList) + line = "LibraryClientCommand: %s\n" % libraryClientCommand + sys.stdout.write(line) + sys.stdout.write("\n") + sys.stderr.write(line) + sys.stderr.write("\n\n") + + # read problem sizes file + csvFileRaw = open(problemSizesPath, "r") + csvFile = csv.reader(csvFileRaw) + + # column headers + numIndices = -1 + for row in csvFile: + numIndices = len(row) + firstRow = row + break + header = "" + for i in range(0, numIndices): + sizeStr = "size%u" % i + header += "%6s, " % sizeStr + header += "%9s, %9s, " % ( "medianGF", "medianMs" ) + header += "%9s, %9s, " % ( "meanGF", "meanMs" ) + header += "%9s, %9s" % ( "rstdGF", "rstdMs" ) + sys.stdout.write(header) + sys.stdout.write("\n") + + # benchmark each problem size + for row in [firstRow]: + (gflopList, msList) = BenchmarkProblemSize(libraryClientCommand, row) + PrintStats(header, row, gflopList, msList) + + # benchmark each problem size + for row in csvFile: + (gflopList, msList) = BenchmarkProblemSize(libraryClientCommand, row) + PrintStats(header, row, gflopList, msList) + +def median(lst): + sortedList = sorted(lst) + return sortedList[len(sortedList)/2] + +def mean(lst): + total = 0 + for i in range(len(lst)): + total += lst[i] + return (total / len(lst)) + +def stddev(lst): + total = 0 + mn = mean(lst) + for i in range(len(lst)): + total += pow((lst[i]-mn),2) + return (total/(len(lst)-1))**0.5 + +# installed "tensileBenchmarkLibraryClient" command +def main(): + TensileBenchmarkLibraryClient(sys.argv[1:]) + +if __name__ == "__main__": + main() + diff --git a/tensilelite/Tensile/TensileClientConfig.py b/tensilelite/Tensile/TensileClientConfig.py new file mode 100644 index 0000000000..cbfbd90596 --- /dev/null +++ b/tensilelite/Tensile/TensileClientConfig.py @@ -0,0 +1,187 @@ +############################################################################### +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from . import Common +from . import ClientWriter +from . import LibraryIO +from .Contractions import ProblemType as ContractionsProblemType +from .SolutionStructs import ProblemSizes, ProblemType +from .Common import print1, printExit, printWarning, assignGlobalParameters, \ + restoreDefaultGlobalParameters, HR +from .Tensile import addCommonArguments, argUpdatedGlobalParameters +from . import __version__ + +import argparse +import os +import sys + + +def getGlobalParams(config): + """Try to parse out global parameters from the places it could be""" + globalParams = None + try: + globalParams = config["GlobalParameters"] + except (TypeError, LookupError): + pass + + return globalParams + + +def getProblemDict(config): + """Try to parse out a problem type dict from the places it could be""" + problemDict = None + try: # Tensile Config file (first entry) + problemDict = config["BenchmarkProblems"][0][0] + if len(config["BenchmarkProblems"]) > 1: + printWarning("More than one BenchmarkProblem in config file: only using first") + except (TypeError, LookupError): + pass + else: + return problemDict + + try: # Solution Selection "base" file + problemDict = config["ProblemType"] + except (TypeError, LookupError): + pass + + return problemDict + + +def getSizeList(config): + """ + Try to parse out a size list from the places it could be + (including the whole config being the list) + """ + sizeList = None + try: # Tensile config file (first entry) + sizeList = config["BenchmarkProblems"][0][1]["BenchmarkFinalParameters"][0]["ProblemSizes"] + except (TypeError, LookupError): + pass + else: + return sizeList + + # check if whole config looks like a size list + keep = True + if type(config) is list: + for i in config: + if not (type(i) is dict and ("Exact" in i or "Range" in i)): + keep = False + break + else: + keep = False + + if keep: + sizeList = config + return sizeList + + +def parseConfig(config): + """Parse out all data we can get from the config""" + globalParams = getGlobalParams(config) + problemDict = getProblemDict(config) + sizeList = getSizeList(config) + + return (globalParams, problemDict, sizeList) + + +def TensileClientConfig(userArgs): + print1("") + print1(HR) + print1("#") + print1("# Tensile Client Config v{}".format(__version__)) + print1("#") + print1(HR) + print1("") + + # argument parsing + # yapf: disable + argParser = argparse.ArgumentParser() + argParser.add_argument("ConfigYaml", type=os.path.realpath, nargs="+", + help="Config yaml(s) containing parameters and problem and size information") + argParser.add_argument("--output-config", "-o", dest="OutputConfig", type=os.path.realpath, + required=True, help="Path to output resulting client config file") + argParser.add_argument("--merge-sizes", dest="MergeSizes", action="store_true", + help="Allow sizes from multiple config files") + # yapf: enable + + addCommonArguments(argParser) + args = argParser.parse_args(userArgs) + + # loop through all configs and parse out all the data we can + configs = [LibraryIO.readYAML(x) for x in args.ConfigYaml] + + globalParams = {} + problemDict = None + sizeList = None + + for config in configs: + (myGlobalParams, myProblemDict, mySizeList) = parseConfig(config) + + # if we got data from the config, keep it + # if we already had that data, warn and exit + if myGlobalParams is not None: + if globalParams == {} or globalParams == myGlobalParams: + globalParams = myGlobalParams + else: + printExit("Multiple definitions for GlobalParameters found:\n{}\nand\n{}".format( + globalParams, myGlobalParams)) + + if myProblemDict is not None: + if problemDict is None or problemDict == myProblemDict: + problemDict = myProblemDict + else: + printExit("Multiple definitions for ProblemType found:\n{}\nand\n{}".format( + problemDict, myProblemDict)) + + if mySizeList is not None: + if sizeList is None or sizeList == mySizeList: + sizeList = mySizeList + elif args.MergeSizes: + sizeList += mySizeList + else: + printExit("Multiple size lists found:\n{}\nand\n{}\n" + "Run with --merge-sizes to keep all size lists found".format( + sizeList, mySizeList)) + + if problemDict is None: + printExit("No ProblemType found; cannot produce output") + if sizeList is None: + printExit("No SizeList found; cannot produce output") + + ssProblemType = ProblemType(problemDict) + conProblemType = ContractionsProblemType.FromOriginalState(ssProblemType) + sizes = ProblemSizes(ssProblemType, sizeList) # TODO doesn't seem to work for range sizes + + # update globals + restoreDefaultGlobalParameters() + assignGlobalParameters(globalParams) + + overrideParameters = argUpdatedGlobalParameters(args) + for key, value in overrideParameters.items(): + print1("Overriding {0}={1}".format(key, value)) + Common.globalParameters[key] = value + + # write output + ClientWriter.writeClientConfigIni(sizes, conProblemType, "", [], "", args.OutputConfig, None) + + +def main(): + TensileClientConfig(sys.argv[1:]) diff --git a/tensilelite/Tensile/TensileCreateLibrary.py b/tensilelite/Tensile/TensileCreateLibrary.py new file mode 100644 index 0000000000..601231d7f9 --- /dev/null +++ b/tensilelite/Tensile/TensileCreateLibrary.py @@ -0,0 +1,1596 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ +# This script only gets called by CMake + +if __name__ == "__main__": + print("This file can no longer be run as a script. Run 'Tensile/bin/TensileCreateLibrary' instead.") + exit(1) + +from . import Common +from . import ClientExecutable +from . import EmbeddedData +from . import LibraryIO +from . import Utils +from .TensileInstructions import getGfxName +from .Common import globalParameters, HR, print1, print2, printExit, ensurePath, \ + CHeader, CMakeHeader, assignGlobalParameters, \ + listToInitializer, architectureMap +from .KernelWriterAssembly import KernelWriterAssembly +from .SolutionLibrary import MasterSolutionLibrary +from .SolutionStructs import Solution + +import argparse +import collections +import itertools +import os +import re +import shlex +import shutil +import subprocess +import sys +import time +from copy import deepcopy + +################################################################################ +def processKernelSource(kernel, kernelWriterAssembly): + """ + Generate source for a single kernel. + Returns (error, source, header, kernelName). + """ + try: + kernelWriter = kernelWriterAssembly + # get kernel name + kernelName = kernelWriter.getKernelFileBase(kernel) + (err, src) = kernelWriter.getSourceFileString(kernel) + header = kernelWriter.getHeaderFileString(kernel) + except RuntimeError: + return (1, "", "", kernelName) + + return (err, src, header, kernelName) + +def getAssemblyCodeObjectFiles(kernels, kernelWriterAssembly, outputPath): + destDir = ensurePath(os.path.join(outputPath, 'library')) + asmDir = kernelWriterAssembly.getAssemblyDirectory() + assemblyKernels = list([k for k in kernels if k['KernelLanguage'] == 'Assembly']) + if len(assemblyKernels) == 0: + return [] + + archs = collections.defaultdict(list) + for k in assemblyKernels: + archs[tuple(k['ISA'])].append(k) + coFiles = [] + for arch, archKernels in archs.items(): + archName = getGfxName(arch) + objectFiles = list([kernelWriterAssembly.getKernelFileBase(k) + '.o' \ + for k in archKernels \ + if k['KernelLanguage'] == 'Assembly']) + if len(objectFiles) == 0: + continue + if globalParameters["MergeFiles"] or globalParameters["NumMergedFiles"] > 1: + coFile = os.path.join(destDir, 'TensileLibrary_{}.co'.format(archName)) + if "PackageLibrary" in globalParameters and globalParameters["PackageLibrary"]: + destArchDir = ensurePath(os.path.join(destDir, archName)) + coFile = os.path.join(os.path.normcase(destArchDir), 'TensileLibrary_{}.co'.format(archName)) + + if os.name == "nt": + # On Windows, the objectFiles list command line (including spaces) + # exceeds the limit of 8191 characters, so using response file + + responseArgs = objectFiles + responseFile = os.path.join(asmDir, 'clangArgs.txt') + with open(responseFile, 'wt') as file: + file.write( " ".join(responseArgs) ) + file.flush() + + args = [globalParameters['AssemblerPath'], '-target', 'amdgcn-amd-amdhsa', '-o', coFile, '@clangArgs.txt'] + subprocess.check_call(args, cwd=asmDir) + else: + args = kernelWriterAssembly.getLinkCodeObjectArgs(objectFiles, coFile) + subprocess.check_call(args, cwd=asmDir) + + coFiles.append(coFile) + else: + # no mergefiles + + assemblyKernelNames = [kernelWriterAssembly.getKernelFileBase(k) for k in archKernels] + origCOFiles = [os.path.join(asmDir, k + '.co') for k in assemblyKernelNames] + newCOFiles = [] + if globalParameters["PackageLibrary"]: + newCOFiles = [os.path.join(destDir, archName, k + '.co') for k in assemblyKernelNames] + else: + newCOFiles = [os.path.join(destDir, k + '_' + archName + '.co') for k in assemblyKernelNames] + + for src, dst in Utils.tqdm(zip(origCOFiles, newCOFiles), "Copying code objects"): + shutil.copyfile(src, dst) + coFiles += newCOFiles + + return coFiles + +def which(p): + exes = [p+x for x in ['.bat', '', '.exe']] # bat may be front end for file with no extension + system_path = os.environ['PATH'].split(os.pathsep) + if p == 'hipcc' and 'CMAKE_CXX_COMPILER' in os.environ and os.path.isfile(os.environ['CMAKE_CXX_COMPILER']): + return os.environ['CMAKE_CXX_COMPILER'] + for dirname in system_path+[globalParameters["ROCmBinPath"]]: + for exe in exes: + candidate = os.path.join(os.path.expanduser(dirname), exe) + if os.path.isfile(candidate): + return candidate + return None + +def splitArchs(): + # Helper for architecture + def isSupported(arch): + return globalParameters["AsmCaps"][arch]["SupportedISA"] and \ + globalParameters["AsmCaps"][arch]["SupportedSource"] + + if ";" in globalParameters["Architecture"]: + wantedArchs = globalParameters["Architecture"].split(";") + else: + wantedArchs = globalParameters["Architecture"].split("_") + archs = [] + cmdlineArchs = [] + if "all" in wantedArchs: + for arch in globalParameters['SupportedISA']: + if isSupported(arch): + if (arch == (9,0,6) or arch == (9,0,8) or arch == (9,0,10)): + if (arch == (9,0,10)): + archs += [getGfxName(arch) + '-xnack+'] + cmdlineArchs += [getGfxName(arch) + ':xnack+'] + archs += [getGfxName(arch) + '-xnack-'] + cmdlineArchs += [getGfxName(arch) + ':xnack-'] + else: + archs += [getGfxName(arch)] + cmdlineArchs += [getGfxName(arch)] + else: + for arch in wantedArchs: + archs += [re.sub(":", "-", arch)] + cmdlineArchs += [arch] + return archs, cmdlineArchs + +def buildSourceCodeObjectFile(CxxCompiler, outputPath, kernelFile): + buildPath = ensurePath(os.path.join(globalParameters['WorkingPath'], 'code_object_tmp')) + destDir = ensurePath(os.path.join(outputPath, 'library')) + (_, filename) = os.path.split(kernelFile) + (base, _) = os.path.splitext(filename) + + if "CmakeCxxCompiler" in globalParameters and globalParameters["CmakeCxxCompiler"] is not None: + os.environ["CMAKE_CXX_COMPILER"] = globalParameters["CmakeCxxCompiler"] + + objectFilename = base + '.o' + soFilename = base + '.so' + + if (CxxCompiler == "hipcc"): + archs, cmdlineArchs = splitArchs() + + archFlags = ['--offload-arch=' + arch for arch in cmdlineArchs] + + hipFlags = ["--genco", "-D__HIP_HCC_COMPAT_MODE__=1"] #needs to be fixed when Maneesh's change is made available + + hipFlags += ['-I', outputPath] + + launcher = shlex.split(os.environ.get('Tensile_CXX_COMPILER_LAUNCHER', '')) + + if os.name == "nt": + hipFlags += ['-std=c++14', '-fms-extensions', '-fms-compatibility', '-fPIC', '-Wno-deprecated-declarations'] + compileArgs = launcher + [which('hipcc')] + hipFlags + archFlags + [kernelFile, '-c', '-o', os.path.join(buildPath, objectFilename)] + else: + compileArgs = launcher + [which('hipcc')] + hipFlags + archFlags + [kernelFile, '-c', '-o', os.path.join(buildPath, objectFilename)] + + if globalParameters["PrintCodeCommands"]: + print('hipcc:', ' '.join(compileArgs)) + subprocess.check_call(compileArgs) + + # get hipcc version due to compatiblity reasons + hipccver = globalParameters['HipClangVersion'].split(".") + hipccMaj = int(hipccver[0]) + hipccMin = int(hipccver[1]) + # for hipclang 5.2 and above, clang offload bundler changes the way input/output files are specified + inflag = "-inputs" + outflag = "-outputs" + if (hipccMaj == 5 and hipccMin >= 2) or hipccMaj >= 6: + inflag = "-input" + outflag = "-output" + + infile = os.path.join(buildPath, objectFilename) + try: + bundlerArgs = [globalParameters["ClangOffloadBundlerPath"], "-type=o", "%s=%s" % (inflag, infile), "-list"] + listing = subprocess.check_output(bundlerArgs, stderr=subprocess.STDOUT).decode().split("\n") + for target in listing: + matched = re.search("gfx.*$", target) + if matched: + arch = re.sub(":", "-", matched.group()) + outfile = os.path.join(buildPath, "{0}-000-{1}.hsaco".format(soFilename, arch)) + bundlerArgs = [globalParameters["ClangOffloadBundlerPath"], "-type=o", "-targets=%s" % target, + "%s=%s" % (inflag, infile), "%s=%s" % (outflag, outfile), "-unbundle"] + if globalParameters["PrintCodeCommands"]: + print(' '.join(bundlerArgs)) + subprocess.check_call(bundlerArgs) + + except subprocess.CalledProcessError: + for i in range(len(archs)): + outfile = os.path.join(buildPath, "{0}-000-{1}.hsaco".format(soFilename, archs[i])) + bundlerArgs = [globalParameters["ClangOffloadBundlerPath"], "-type=o", "-targets=hip-amdgcn-amd-amdhsa--%s" % cmdlineArchs[i], + "%s=%s" % (inflag, infile), "%s=%s" % (outflag, outfile), "-unbundle"] + if globalParameters["PrintCodeCommands"]: + print(' '.join(bundlerArgs)) + subprocess.check_call(bundlerArgs) + + coFilenames = ["{0}-000-{1}.hsaco".format(soFilename, arch) for arch in archs] + else: + raise RuntimeError("Unknown compiler {}".format(CxxCompiler)) + + destCosList = [] + if "PackageLibrary" in globalParameters and globalParameters["PackageLibrary"]: + for arch in archs: + ensurePath(os.path.join(destDir, arch)) + archCoFilenames = [name for name in coFilenames if arch in name] + extractedCOs = [os.path.join(buildPath, name) for name in archCoFilenames] + destCOs = [os.path.join(destDir, arch, name) for name in archCoFilenames] + destCosList += destCOs + if globalParameters["PrintCodeCommands"]: + print ("# copy source code objects : ", extractedCOs) + print ("# to dest source code objects : ", destCOs) + for (src, dst) in zip(extractedCOs, destCOs): + shutil.copyfile(src, dst) + else: + coFilenames = [name for name in coFilenames] + extractedCOs = [os.path.join(buildPath, name) for name in coFilenames] + destCOs = [os.path.join(destDir, name) for name in coFilenames] + destCosList += destCOs + for (src, dst) in zip(extractedCOs, destCOs): + shutil.copyfile(src, dst) + + return destCosList + +def buildSourceCodeObjectFiles(CxxCompiler, kernelFiles, outputPath): + args = zip(itertools.repeat(CxxCompiler), itertools.repeat(outputPath), kernelFiles) + + coFiles = Common.ParallelMap(buildSourceCodeObjectFile, args, "Compiling source kernels", + method=lambda x: x.starmap) + + return itertools.chain.from_iterable(coFiles) + +################################################################################ +def prepAsm(kernelWriterAssembly): + """ + Create and prepare the assembly directory - called ONCE per output dir: + """ + asmPath = ensurePath(os.path.join(globalParameters["WorkingPath"], "assembly") ) + assemblerFileName = os.path.join(asmPath, \ + "asm-new.%s"%("bat" if os.name=="nt" else "sh")) + assemblerFile = open(assemblerFileName, "w") + if os.name == "nt": + assemblerFile.write("echo Windows: Copying instead of Assembling\n") + assemblerFile.write("copy %1.s %1.o\n") + assemblerFile.write("copy %1.o %1.co\n") + else: + assemblerFile.write("#!/bin/sh {log}\n".format(log = "-x" if globalParameters["PrintLevel"] >=2 else "")) + assemblerFile.write("# usage: asm-new.sh kernelName(no extension) [--wave32]\n") + + assemblerFile.write("f=$1\n") + assemblerFile.write("shift\n") + assemblerFile.write('if [ ! -z "$1" ] && [ "$1" = "--wave32" ]; then\n') + assemblerFile.write(" wave=32\n") + assemblerFile.write(" shift\n") + assemblerFile.write("else\n") + assemblerFile.write(" wave=64\n") + assemblerFile.write("fi\n") + + + isa = globalParameters["CurrentISA"] + assemblerFile.write("h={gfxName}\n".format(gfxName = getGfxName(isa))) + + cArgs32 = kernelWriterAssembly.getCompileArgs("$f.s", "$f.o", isa=isa, wavefrontSize=32) + cArgs64 = kernelWriterAssembly.getCompileArgs("$f.s", "$f.o", isa=isa, wavefrontSize=64) + lArgs = kernelWriterAssembly.getLinkCodeObjectArgs(["$f.o"], "$f.co") + + assemblerFile.write("if [ $wave -eq 32 ]; then\n") + assemblerFile.write(" ".join(cArgs32) + "\n") + assemblerFile.write("else\n") + assemblerFile.write(" ".join(cArgs64) + "\n") + assemblerFile.write("fi\n") + + + assemblerFile.write(" ".join(lArgs) + "\n") + + assemblerFile.write("ERR=$?\n") + assemblerFile.write("if [[ $ERR -ne 0 ]]\n") + assemblerFile.write("then\n") + assemblerFile.write(" echo one\n") + assemblerFile.write(" exit $ERR\n") + assemblerFile.write("fi\n") + + assemblerFile.write("cp $f.co ../../../library/${f}_$h.co\n") + assemblerFile.write("mkdir -p ../../../asm_backup && ") + assemblerFile.write("cp $f.s ../../../asm_backup/$f.s\n") + + assemblerFile.close() + os.chmod(assemblerFileName, 0o777) + +################################################################################ +def buildKernelSourceAndHeaderFiles(results, outputPath, kernelsWithBuildErrs, \ + kernelSourceFile, kernelHeaderFile): + """ + Logs errors and writes appropriate info to kernelSourceFile and kernelHeaderFile. + + Arguments: + results: list of (err, src, header, kernelName) + outputPath: path to source directory + kernelsWithBuildErrs: Dictionary to be updated with kernels that have errors + kernelSourceFile: File to write source data to + kernelHeaderFile: File to write header data to + + Returns: + sourceFilenames: Array containing source kernel filenames + """ + sourceFilenames = [] + + # Find kernels to write + kernelsToWrite = [] + for (err,src,header,kernelName) in results: + + # Keep track of kernels with errors + if err: + kernelsWithBuildErrs[kernelName] = err + + # Don't create a file for empty kernels + if len(src.strip()) == 0: + continue + kernelsToWrite.append((err, src, header, kernelName)) + + # Write kernel data to files + if globalParameters["MergeFiles"]: + + # Merge all kernels into one file + if globalParameters["NumMergedFiles"] == 1: + sourceFilenames.append(os.path.join(os.path.normcase(outputPath), "Kernels.cpp")) + for (err,src,header,kernelName) in kernelsToWrite: + kernelSourceFile.write(src) + kernelHeaderFile.write(header) + + # Merge kernels into n seperate files + else: + + # Calculate num of kernels per file + numValidKernels = len(kernelsToWrite) + numMergedFiles = globalParameters["NumMergedFiles"] + if numMergedFiles > numValidKernels: + numMergedFiles = numValidKernels + + # Number of kernels per file (except the last file which gets this plus any remainder) + kernelsPerFile = numValidKernels // numMergedFiles + + # Group kernel sources and headers into a list of lists for writing + kernelSourceList = [] + kernelHeaderList = [] + for kernelIndex, (_,src,header,_) in enumerate(kernelsToWrite): + makeNewFilegroup = kernelIndex % kernelsPerFile == 0 + isNotLastFile = len(kernelSourceList) < numMergedFiles + if makeNewFilegroup and isNotLastFile: + kernelHeaderList.append([]) + kernelSourceList.append([]) + kernelHeaderList[-1].append(src) + kernelSourceList[-1].append(header) + + # Write source files + for fileIndex, kernelSources in enumerate(kernelSourceList): + kernelName = "Kernels" + str(fileIndex) + kernelFilePath = os.path.join(outputPath, "Kernels", kernelName) + + with open(kernelFilePath + ".cpp", "w", encoding="utf-8") as kernelSourceFile: + kernelSourceFile.write(CHeader) + kernelSourceFile.write('#include "{}.h"\n'.format(kernelName)) + sourceFilenames.append(kernelFilePath + ".cpp") + for src in kernelSources: + kernelSourceFile.write(src) + + # Write header files + for fileIndex, kernelHeaders in enumerate(kernelHeaderList): + kernelName = "Kernels" + str(fileIndex) + kernelFilePath = os.path.join(outputPath, "Kernels", kernelName) + + with open(kernelFilePath + ".h", "w", encoding="utf-8") as kernelHeaderFile: + kernelHeaderFile.write(CHeader) + kernelHeaderFile.write("#pragma once\n") + if globalParameters["RuntimeLanguage"] == "HIP": + kernelHeaderFile.write("#include \n") + kernelHeaderFile.write("#include \n\n") + kernelHeaderFile.write("#include \n\n") + for header in kernelHeaders: + kernelHeaderFile.write(header) + + # Write kernels into seperate files + else: + for (err,src,header,kernelName) in kernelsToWrite: + + # write kernel.cpp + filepath = os.path.join(outputPath, "Kernels", kernelName) + sourceFilename = filepath + ".cpp" + sourceFilenames.append(sourceFilename) + kernelSourceFile = open(sourceFilename, "w", encoding="utf-8") + kernelSourceFile.write(CHeader) + kernelSourceFile.write(src) + kernelSourceFile.close() + + # write kernel.h + headerFilename = filepath + ".h" + kernelHeaderFile = open(headerFilename, "w", encoding="utf-8") + kernelHeaderFile.write(CHeader) + kernelHeaderFile.write(header) + kernelHeaderFile.close() + + return sourceFilenames + +################################################################################ +# Write Solutions and Kernels for BenchmarkClient or LibraryClient +################################################################################ +def writeSolutionsAndKernels(outputPath, CxxCompiler, problemTypes, solutions, kernels, kernelHelperObjs, \ + kernelWriterAssembly, errorTolerant=False): + start = time.time() + + codeObjectFiles = [] + + # Push working path into build_tmp folder because there may be more than + # one process running this script. This is to avoid build directory clashing. + # NOTE: file paths must not contain the lower case word 'kernel' or the + # /opt/rocm/bin/extractkernel will fail. + # See buildSourceCodeObjectFile:167 for the call to this binary. + Common.pushWorkingPath('build_tmp') + Common.pushWorkingPath(os.path.basename(outputPath).upper()) + + print1("# Writing Kernels...") + kernelFiles = [] + kernelSourceFile = None + kernelHeaderFile = None + + if not globalParameters["MergeFiles"] or globalParameters["NumMergedFiles"] > 1: + ensurePath(os.path.join(outputPath, "Kernels")) + + ############################################################################## + # Write Kernels + ############################################################################## + if globalParameters["MergeFiles"] and globalParameters["NumMergedFiles"] == 1: + kernelSourceFilename = os.path.join(os.path.normcase(outputPath), "Kernels.cpp") + kernelHeaderFilename = os.path.join(os.path.normcase(outputPath), "Kernels.h") + + kernelSourceFile = open(kernelSourceFilename, "w") + kernelHeaderFile = open(kernelHeaderFilename, "w") + kernelSourceFile.write(CHeader) + kernelHeaderFile.write(CHeader) + kernelSourceFile.write("#include \"Kernels.h\"\n") + kernelHeaderFile.write("#pragma once\n") + if globalParameters["RuntimeLanguage"] == "HIP": + kernelHeaderFile.write("#include \n") + kernelHeaderFile.write("#include \n\n") + kernelHeaderFile.write("#include \"KernelHeader.h\"\n\n") + + kernelsWithBuildErrs = {} + + prepAsm(kernelWriterAssembly) + + kIter = zip(kernels, itertools.repeat(kernelWriterAssembly)) + results = Common.ParallelMap(processKernelSource, kIter, "Generating kernels", method=lambda x: x.starmap, maxTasksPerChild=1) + + removeKernels = [] + removeSolutions = [] + removeResults = [] + for kernIdx, res in Utils.tqdm(enumerate(results)): + (err,src,header,kernelName) = res + if(err == -2): + removeKernels.append(kernels[kernIdx]) + removeSolutions.append(solutions[kernIdx]) + removeResults.append(results[kernIdx]) + for kern in removeKernels: + kernels.remove(kern) + for solut in removeSolutions: + solutions.remove(solut) + for rel in removeResults: + results.remove(rel) + + kernelFiles += buildKernelSourceAndHeaderFiles(results, outputPath, kernelsWithBuildErrs, kernelSourceFile, kernelHeaderFile) + + kernelsToBuild = list(kernels) + if errorTolerant: + def success(kernel): + writer = kernelWriterAssembly + kernelName = writer.getKernelName(kernel) + return kernelName not in kernelsWithBuildErrs + kernelsToBuild = list(filter(success, kernelsToBuild)) + + if False:#len(kernelsWithBuildErrs) > 0: + print("\nKernel compilation failed in one or more subprocesses. May want to set CpuThreads=0 and re-run to make debug easier") + printExit("** kernel compilation failure **") + + # Put all kernel helper objects into the first merged kernel file + if globalParameters["NumMergedFiles"] > 1 and len(kernelFiles) > 0: + kernelFilename = kernelFiles[0].replace(".cpp", "") + kernelSourceFile = open(kernelFilename + ".cpp", 'a', encoding="utf-8") + kernelHeaderFile = open(kernelFilename + ".h", 'a', encoding="utf-8") + + # handle helper kernel function + for ko in kernelHelperObjs: + kernelName = ko.getKernelName() + + # write kernel.cpp + if not globalParameters["MergeFiles"]: + kernelSourceFilename = os.path.join(outputPath, "Kernels", kernelName+".cpp") + kernelSourceFile = open(kernelSourceFilename, "w") + kernelSourceFile.write(CHeader) + kernelFiles.append(kernelSourceFilename) + + (err, src) = ko.getSourceFileString() + kernelSourceFile.write(src) + if err: + print("*** warning: invalid kernel#%u"%kernelName) + + if not globalParameters["MergeFiles"]: + kernelSourceFile.close() + + # write kernel.h + if not globalParameters["MergeFiles"]: + kernelHeaderFile = open(os.path.join(os.path.normcase(outputPath), "Kernels", kernelName + ".h"), "w") + kernelHeaderFile.write(CHeader) + + kernelHeaderFile.write(ko.getHeaderFileString()) + + if not globalParameters["MergeFiles"]: + kernelHeaderFile.close() + + # close merged + if globalParameters["MergeFiles"]: + if kernelSourceFile: + kernelSourceFile.close() + if kernelHeaderFile: + kernelHeaderFile.close() + + if not globalParameters["GenerateSourcesAndExit"]: + codeObjectFiles += buildSourceCodeObjectFiles(CxxCompiler, kernelFiles, outputPath) + codeObjectFiles += getAssemblyCodeObjectFiles(kernelsToBuild, kernelWriterAssembly, outputPath) + + stop = time.time() + print("# Kernel Building elapsed time = %.1f secs" % (stop-start)) + + Common.popWorkingPath() # build_tmp + Common.popWorkingPath() # workingDir + + return codeObjectFiles + +################################################################################ +# Write Logic +################################################################################ +def writeLogic(outputPath, logicData, solutionWriter ): + print1("# Writing Library Logic") + + if not globalParameters["MergeFiles"] or globalParameters["NumMergedFiles"] > 1: + ensurePath(os.path.join(outputPath, "Logic")) + + # Tensile.h + h = "" + h += "#pragma once\n" + h += "#include \"TensileTypes.h\"\n" + h += "#include \"SolutionHelper.h\"\n" + h += "#include \"SolutionMapper.h\"\n" + + # TensileInternal.h + ih = "" + ih += "#include \"Tensile.h\"\n" + + # Tensile.cpp + sourceIncludes = "" + sourceIncludes += "#include \"Solutions.h\"\n" + sourceIncludes += "#include \"Tensile.h\"\n" + sourceIncludes += "#include \"TensileInternal.h\"\n" + sourceIncludes += "#include \"SolutionMapper.h\"\n" + + s = sourceIncludes + + ######################################## + # problemType + for problemType in Utils.tqdm(logicData): + + # function argument list + argListSizes = solutionWriter.getArgList(problemType, False, False, False, False, True) + argListData = solutionWriter.getArgList(problemType, False, True, True, True, True) + argListAll = solutionWriter.getArgList(problemType, True, True, True, True, True) + + # tensile initializer + h += "\nvoid tensileInitialize();\n\n" + + # declare tensile_ProblemType + h += "\n// enqueue solution\n" + h += "TensileStatus tensile_%s(\n" % problemType + for i in range(0, len(argListData)): + h += " %s %s = %s%s" \ + % (argListData[i][0], argListData[i][1], argListData[i][2], \ + ",\n" if i < len(argListData)-1 else ");\n\n") + + + numSizes = problemType["TotalIndices"] + assert(not problemType["UseInitialStridesCD"]) + firstStride = 0 if problemType["UseInitialStridesAB"] else 1 + lastStrideA = len(problemType["IndexAssignmentsA"]) + lastStrideB = len(problemType["IndexAssignmentsB"]) + lastStrideC = problemType["NumIndicesC"] + lastStrideD = problemType["NumIndicesC"] + h += "typedef ProblemKey<%u> ProblemKey_%s;\n" % (numSizes,problemType) + h += "typedef ProblemDims<%u,%u,%u,%u,%u,%u> ProblemDims_%s;\n" \ + % (firstStride, lastStrideD, lastStrideC, lastStrideA, lastStrideB, numSizes, problemType) + h += "typedef SolutionMapper SolutionMapper_%s;\n" \ + % (problemType, problemType, problemType) + + # declare tensileGetSolutionPointer_ProblemType + h += "\n// get solution pointer\n" + h += "SolutionMapper_%s::SolutionRuntime *\n" % (problemType) + h += "tensileGetSolutionPointer_%s(\n" % (problemType) + for i in range(0, len(argListSizes)): + h += " %s %s%s" \ + % (argListSizes[i][0], argListSizes[i][1], \ + ",\n" if i < len(argListSizes)-1 else ");\n\n") + + # declare tensileName_ + h += "// get solution name\n" + h += "const char * tensileGetSolutionName_%s(\n" \ + % (problemType) + for i in range(0, len(argListSizes)): + h += " %s %s%s" \ + % (argListSizes[i][0], argListSizes[i][1], \ + ",\n" if i < len(argListSizes)-1 else ");\n\n") + + + # get solution naming for problem type + solutionsForProblemType = [] + for scheduleTuple in logicData[problemType]: + solutionsForSchedule = scheduleTuple[2] + for solution in solutionsForSchedule: + if solution not in solutionsForProblemType: + solutionsForProblemType.append(solution) + + # solution names for problem type + solutionNamesForProblemType = [] + for solution in solutionsForProblemType: + solutionName = solutionWriter.getSolutionName(solution) + solutionNamesForProblemType.append(solutionName) + + # reset problemType source + if not globalParameters["MergeFiles"] or globalParameters["NumMergedFiles"] > 1: + filePrefix = "Tensile_%s" % (problemType) + s = sourceIncludes + + for solutionName in solutionNamesForProblemType: + s += "#include \"%s.h\"\n" % solutionName + + ######################################## + # Per-problem constants here: + # These are common for all schedules and thus do not include schedule name (vega,hip,etc) + s += "\n" + s += "/*******************************************************************************\n" + s += "* Per-Problem Functions for %s\n" % problemType + s += "*******************************************************************************/\n" + + s += "// Problem type include the index assignments for free, summation, batch:\n" + s += "static const ProblemType problemType_%s( " % problemType + s += listToInitializer(problemType["IndicesFree"]) + ", " + s += listToInitializer(problemType["IndicesSummation"]) + ", " + s += listToInitializer(problemType["IndicesBatch"]) + ", " + s += listToInitializer(problemType["IndexAssignmentsA"]) + ", " + s += listToInitializer(problemType["IndexAssignmentsB"]) + s += ");\n" + + s += "\n" + s += "// Master solution mapper is the entry point for problem->solution mapping\n" + s += "// There is one master solution mapper per problem type\n" + s += "// The master solution mapper contains pointers to the solution mappers for each device\n" + s += "static MasterSolutionMapper masterSolutionMapper_%s;\n " % (problemType,problemType) + + + ######################################## + # implement per-Schedule functions in source + s += "\n" + s += "/*******************************************************************************\n * Per-Schedule Functions\n *******************************************************************************/" + for scheduleTuple in logicData[problemType]: + + # get logic parameters for problem type + scheduleName = scheduleTuple[0] + deviceNames = scheduleTuple[1] + solutionsForSchedule = scheduleTuple[2] + indexOrder = scheduleTuple[3] + exactLogic = scheduleTuple[4] + rangeLogic = scheduleTuple[5] + + # solution names for schedule + solutionNamesForSchedule = [] + for solution in solutionsForSchedule: + solutionName = solutionWriter.getSolutionName(solution) + solutionNamesForSchedule.append(solutionName) + + s += "\n\n" + schedProbName = "%s_%s" % (scheduleName, problemType) + s += writeSolutionAndExactTable(scheduleName, deviceNames, schedProbName, problemType, \ + solutionsForSchedule, solutionNamesForSchedule, exactLogic) + + + # Per-problem function here: + # function tensileGetSolutionPointer_ProblemType + del schedProbName + del scheduleName + s += "\n// problem dims -> solution logic\n" + s += "SolutionMapper_%s::SolutionRuntime *\n" % (problemType) + s += "tensileGetSolutionPointer_%s(\n" % (problemType) + for i in range(0, len(argListSizes)): + s += " %s %s%s" \ + % (argListSizes[i][0], argListSizes[i][1], \ + ",\n" if i < len(argListSizes)-1 else ") {\n\n") + + exactLogicStr = writeExactLogic(problemType, indexOrder, \ + solutionsForSchedule, exactLogic, \ + solutionNamesForSchedule, True) + if rangeLogic != None: + print("** warning: ignored ranges in logic file, these should have been expanded with ExpandRanges=1 during Tensile phase 3") + s += " /* exact mappings */\n" + s += exactLogicStr + s += "\n return nullptr;\n" + s += "\n}\n" + + # function tensileGetSolutionName_Schedule_ProblemType + s += "\n// get solution name for problem dims\n" + s += "const char * tensileGetSolutionName_%s(\n" \ + % (problemType) + for i in range(0, len(argListSizes)): + s += " %s %s%s" \ + % (argListSizes[i][0], argListSizes[i][1], \ + ",\n" if i < len(argListSizes)-1 else ") {\n\n") + + exactLogicStr = writeExactLogic(problemType, indexOrder, \ + solutionsForSchedule, exactLogic, \ + solutionNamesForSchedule, False) + s += " /* exact mappings */\n" + s += exactLogicStr + s += "\n}\n" + + ######################################## + # implement problem-type functions in source + s += "/*******************************************************************************\n * Per-ProblemType Functions\n *******************************************************************************/" + + + # declare tensile_ProblemType + s += "\n// main call to solution; enqueues a kernel\n" + s += "TensileStatus tensile_%s(\n" % problemType + for i in range(0, len(argListData)): + s += " %s %s%s" \ + % (argListData[i][0], argListData[i][1], \ + ",\n" if i < len(argListData)-1 else ") {\n") + s += " auto solution = tensileGetSolutionPointer_%s(\n" % (problemType) + for i in range(0, len(argListSizes)): + s += " %s%s" \ + % (argListSizes[i][1], ", " if i < len(argListSizes)-1 else ");") + s += "\n" + s += " if (solution) {\n" + s += " TensileSolutionPointer_%s f = reinterpret_cast (solution->_info->_functionPtr);\n" \ + % (problemType, problemType) + s += " auto solutionLock = &solution->_lock;\n" + s += " return f(" + for i in range(0, len(argListAll)): + s += "%s%s" \ + % (argListAll[i][1], ", " if i < len(argListAll)-1 else ");\n") + s += " } else {\n" + #s += " printf(\"solution not valid, returning fail\\n\");" + s += " return tensileStatusFailure; // no solution found\n" + s += " }\n" + s += "}\n" + + # open and close problemType files + if not globalParameters["MergeFiles"] or globalParameters["NumMergedFiles"] > 1: + logicSourceFile = open(os.path.join(os.path.normcase(outputPath), "Logic", \ + "%s.cpp" % filePrefix), "w") + logicSourceFile.write(s) + logicSourceFile.close() + + s += "\n" + s += writeTensileInitialize(logicData) + + # close merged files + if globalParameters["MergeFiles"] and globalParameters["NumMergedFiles"] == 1: + logicSourceFile = open(os.path.join(os.path.normcase(outputPath), \ + "Tensile.cpp"), "w") + logicSourceFile.write(s) + logicSourceFile.close() + + logicHeaderFile = open(os.path.join(os.path.normcase(outputPath), \ + "Tensile.h"), "w") + logicHeaderFile.write(h) + logicHeaderFile.close() + + internalHeaderFile = open(os.path.join(os.path.normcase(outputPath), \ + "TensileInternal.h"), "w") + internalHeaderFile.write(ih) + internalHeaderFile.close() + + +def writeTensileInitialize(logicData): + + s = "/*******************************************************************************\n" + s += "* Tensilze initializer\n" + s += "*******************************************************************************/\n" + s += "void tensileInitialize() {\n" + + for problemType in logicData: + s += " masterSolutionMapper_%s.initialize();\n" % problemType + + for scheduleTuple in logicData[problemType]: + scheduleName = scheduleTuple[0] + deviceNames = scheduleTuple[1] + + + schedProbName = "%s_%s" % (scheduleName, problemType) + s += " solutionMapper_%s.initializeMappers(" % (schedProbName) + s += "{%s}," % (', '.join('"{0}"'.format(w) for w in deviceNames)) + s += "&masterSolutionMapper_%s);\n" % (problemType) + + s += "}" + + return s + +def writeSolutionAndExactTable(scheduleName, deviceNames, schedProbName, problemType, \ + solutionsForSchedule, solutionNames, exactLogic): + s = "" + s += "namespace { // Start schedule '%s'\n" % scheduleName + + s += "// solution table - function, name, assertion requirements\n" + s += "static const SolutionInfo solutionTable_%s[] = {\n" % (schedProbName) + for i in range(0, len(solutionsForSchedule)): + solution = solutionsForSchedule[i] + solutionName = solutionNames[i] + s += " {(void*)%s, \"%s\", {%d, %d, %d, %d, %d} }%s // %d" % \ + (solutionName, solutionName, \ + solution["AssertSummationElementMultiple"], \ + solution["AssertFree0ElementMultiple"], \ + solution["AssertFree1ElementMultiple"], \ + False, \ + "," if i < len(solutionsForSchedule)-1 else "", \ + i) + s += "\n" + + s += "};\n\n" + + # Write the exact problems here + s += "// table of exact problem dims and selected solutionIdx\n" + s += "static const std::pair embeddedExactTable_%s[] = {\n" % (problemType,schedProbName) + numSizes = problemType["TotalIndices"] + for ruleIdx in range(0, len(exactLogic)): + rule = exactLogic[ruleIdx] + problemSize = rule[0][:numSizes] + solutionIdx = rule[1][0] + solutionGFlops = rule[1][1] + s += " { {" + for i in range(0, len(problemSize)): + if i == 0: + s += "%u" % problemSize[i] + else: + s += ", %u" % problemSize[i] + s += "}, %u}" % (solutionIdx) + s += "," if ruleIdx != len(exactLogic)-1 else " " + s += " // %.0f GFlop/s" % (solutionGFlops) + s += "\n" + s += "};\n\n" + + # Create a solution mapper and init with the table above: + s += "// The solution master constructor here adds device to the master solution mapper\n" + s += "// The entrypoint to find a solution for this problem is through the master solution master\n" + s += "static SolutionMapper_%s solutionMapper_%s(\n" % (problemType, schedProbName) + s += " \"%s\", // schedule+problem name\n" % (schedProbName) + s += " solutionTable_%s, %u,\n" % (schedProbName, len(solutionsForSchedule)) + s += " embeddedExactTable_%s, %u,\n" % (schedProbName, len(exactLogic)) + s += " &problemType_%s);\n" % (problemType) + + s += "} // end anonymous namespace\n" + return s + + +################################################################################ +# Write Range Logic Recursive +# ptr : +# True : write logic to return the function pointer +# False : write logic to return the function name +################################################################################ +def writeExactLogic(problemType, indexOrder, + solutionsForSchedule, exactLogic, \ + solutionNames, ptr): + s = "" + s += " ProblemDims_%s pdims(" % problemType + indexChars = globalParameters["IndexChars"] + firstStrideAB = 0 if problemType["UseInitialStridesAB"] else 1 + firstStrideCD = 0 if problemType["UseInitialStridesCD"] else 1 + lastStrideD = problemType["NumIndicesC"] + lastStrideC = problemType["NumIndicesC"] + lastStrideA = len(problemType["IndexAssignmentsA"]) + lastStrideB = len(problemType["IndexAssignmentsB"]) + for i in range(firstStrideCD,lastStrideD): + if i != firstStrideCD: s += ", " + s += "strideD%u%s" % (i, indexChars[i]) + for i in range(firstStrideCD,lastStrideC): + s += ", strideC%u%s" % (i, indexChars[i]) + for i in range(firstStrideAB,lastStrideA): + s += ", strideA%u%s" % (i, \ + indexChars[problemType["IndexAssignmentsA"][i]]) + for i in range(firstStrideAB,lastStrideB): + s += ", strideB%u%s" % (i, \ + indexChars[problemType["IndexAssignmentsB"][i]]) + for i in range(0,len(indexOrder)): + s += ", size%s" % indexChars[i] + s += ");\n" + + s += " auto solutionMapper = reinterpret_cast (masterSolutionMapper_%s.mapper());\n" \ + % (problemType, problemType) + if ptr: + s += " return solutionMapper->getSolutionWithFallback(pdims,&masterSolutionMapper_%s);\n" % problemType + else: + s += " return solutionMapper->getSolutionWithFallback(pdims,&masterSolutionMapper_%s)->_info->_name;\n" % problemType + + return s + + +################################################################################ +# Write Solution Call +################################################################################ +def writeSolutionCall(solutionName, problemType): + indexChars = globalParameters["IndexChars"] + s = "" + s += "%s(" % solutionName + # solution parameters + s += " dataD, dataC, dataA, dataB, alpha" + if problemType["UseBeta"]: + s += ", beta" + s += ", offsetC, offsetA, offsetB" + firstStrideAB = firstStrideCD = 1 + if problemType["UseInitialStridesAB"]: + firstStrideAB = 0 + if problemType["UseInitialStridesCD"]: + firstStrideCD = 0 + lastStrideD = problemType["NumIndicesC"] + lastStrideC = problemType["NumIndicesC"] + lastStrideA = len(problemType["IndexAssignmentsA"]) + lastStrideB = len(problemType["IndexAssignmentsB"]) + for i in range(firstStrideCD,lastStrideD): + s += ", strideD%u%s" % (i, indexChars[i]) + for i in range(firstStrideCD,lastStrideC): + s += ", strideC%u%s" % (i, indexChars[i]) + for i in range(firstStrideAB,lastStrideA): + s += ", strideA%u%s" % (i, \ + indexChars[problemType["IndexAssignmentsA"][i]]) + for i in range(firstStrideAB,lastStrideB): + s += ", strideB%u%s" % (i, \ + indexChars[problemType["IndexAssignmentsB"][i]]) + for i in range(0, problemType["TotalIndices"]): + s += ", size%s" % indexChars[i] + s += ", stream, numInputEvents, inputEvents, outputEvent )" + return s + +############################################################################## +# Min Naming / Solution and Kernel Writers +############################################################################## +def getSolutionAndKernelWriters(solutions, kernels): + + # if any kernels are assembly, append every ISA supported + kernelSerialNaming = Solution.getSerialNaming(kernels) + + solutionMinNaming = Solution.getMinNaming(solutions) + kernelMinNaming = Solution.getMinNaming(kernels) + kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming) + + return (kernelWriterAssembly, kernelMinNaming, solutionMinNaming) + +################################################################################ +# copy static cpp files and headers +################################################################################ +def copyStaticFiles(outputPath=None): + if outputPath is None: + outputPath = globalParameters["WorkingPath"] + libraryStaticFiles = [ + "TensileTypes.h", + "tensile_bfloat16.h", + "KernelHeader.h" ] + + for fileName in libraryStaticFiles: + # copy file + shutil.copy( os.path.join(globalParameters["SourcePath"], fileName), \ + outputPath ) + + return libraryStaticFiles + +def buildObjectFileNames(kernelWriterAssembly, kernels, kernelHelperObjs): + + # Build lists of output object names + sourceKernelNames = [] + asmKernelNames = [] + kernelHelperObjNames = [] + + solutionFiles = [] + sourceKernelFiles = [] + asmKernelFiles = [] + sourceLibFiles = [] + asmLibFiles = [] + + asmKernels = list([k for k in kernels if k['KernelLanguage'] == 'Assembly']) + + # Build a list of kernel object names. + for kernel in asmKernels: + asmKernelNames += [kernelWriterAssembly.getKernelFileBase(kernel)] + + kernelHelperObjNames = [ko.getKernelName() for ko in kernelHelperObjs] + + cxxCompiler = globalParameters["CxxCompiler"] + + # Source based kernels are built for all supported architectures + if (cxxCompiler == 'hipcc'): + sourceArchs, _ = splitArchs() + else: + raise RuntimeError("Unknown compiler %s" % cxxCompiler) + + # Asm based kernels target the configured ISA + asmArchs = collections.defaultdict(list) + for (kernelName, kernel) in zip(asmKernelNames, asmKernels): + asmArchs[kernelName].append(getGfxName(kernel['ISA'])) + + # Build a list of source files + if not globalParameters["MergeFiles"]: + for kernelName in (sourceKernelNames + asmKernelNames + kernelHelperObjNames): + sourceKernelFiles += [ + "%s.h" % (kernelName), + "%s.cpp" % (kernelName)] + elif globalParameters["NumMergedFiles"] > 1: + for kernelIndex in range(0, globalParameters["NumMergedFiles"]): + sourceKernelFiles += [ + "Kernels%s.h" % str(kernelIndex), + "Kernels%s.cpp" % str(kernelIndex)] + for kernelName in (kernelHelperObjNames): + sourceKernelFiles += [ + "%s.h" % (kernelName), + "%s.cpp" % (kernelName)] + else: + sourceKernelFiles += ["Kernels.h", "Kernels.cpp"] + + # Build a list of assembly files + for asmKernelName in asmKernelNames: + asmKernelFiles += [ + "%s.s" % (asmKernelName), + "%s.o" % (asmKernelName), + "%s.co" % (asmKernelName)] + + # Build a list of lib names from source + if not globalParameters["MergeFiles"]: + + allSources = sourceKernelNames + kernelHelperObjNames + + for kernelName in (allSources): + if (cxxCompiler == 'hipcc'): + sourceLibFiles += ["%s.so-000-%s.hsaco" % (kernelName, arch) for arch in sourceArchs] + else: + raise RuntimeError("Unknown compiler {}".format(cxxCompiler)) + elif globalParameters["NumMergedFiles"] > 1: + if (cxxCompiler == 'hipcc'): + for kernelIndex in range(0, globalParameters["NumMergedFiles"]): + sourceLibFiles += ["Kernels%d.so-000-%s.hsaco" % (kernelIndex, arch) for arch in sourceArchs] + else: + raise RuntimeError("Unknown compiler {}".format(cxxCompiler)) + else: # Merge + if (cxxCompiler == 'hipcc'): + sourceLibFiles += ["Kernels.so-000-%s.hsaco" % (arch) for arch in sourceArchs] + else: + raise RuntimeError("Unknown compiler {}".format(cxxCompiler)) + + # Build a list of asm lib names + if globalParameters["MergeFiles"]: + # Find all unique arch values for current asm kernels + uniqueArchs = set(itertools.chain(*asmArchs.values())) + asmLibFiles += ["TensileLibrary_%s.co" % (arch) for arch in uniqueArchs] + else: + for asmKernelName, archs in asmArchs.items(): + asmLibFiles += ["%s_%s.co" % (asmKernelName, str(arch)) for arch in archs] + + return (solutionFiles, sourceKernelFiles, asmKernelFiles, sourceLibFiles, asmLibFiles) + +def buildObjectFilePaths(prefixDir, solutionFiles, sourceKernelFiles, asmKernelFiles, sourceLibFiles, asmLibFiles): + + solutionPaths = [] + sourceKernelPaths = [] + asmKernelPaths = [] + sourceLibPaths = [] + asmLibPaths = [] + libMetadataPaths = [] + + + # Build full paths for source kernel files + sourceKernelDir = "" + if not globalParameters["MergeFiles"] or globalParameters["NumMergedFiles"] > 1: + sourceKernelDir = os.path.join(prefixDir, "Kernels") + else: + sourceKernelDir = prefixDir + + for sourceKernelFile in sourceKernelFiles: + sourceKernelPaths += [ os.path.join(sourceKernelDir, sourceKernelFile) ] + + # Build full paths for asm kernel files + asmKernelDir = os.path.join(prefixDir, "assembly") + + for asmKernelFile in asmKernelFiles: + asmKernelPaths += [ os.path.join(asmKernelDir, asmKernelFile) ] + + # Build full paths for source and asm library files + libDir = os.path.join(prefixDir, "library") + + libraryExt = ".yaml" if globalParameters["LibraryFormat"] == "yaml" else ".dat" + if not globalParameters["SeparateArchitectures"]: + libMetadataPaths = [ os.path.join(libDir, "TensileLibrary"+libraryExt) ] + + for sourceLibFile in sourceLibFiles: + sourceLibPaths += [ os.path.join(libDir, sourceLibFile) ] + + for asmLibFile in asmLibFiles: + # Asm lib files are enumerated in the form of + # KernelName_gfxXXXXX.co + # Strip the gfxXXXX portion and use that as a subdirectory + asmLibFileNoExt = str(os.path.splitext(asmLibFile)[0]) + asmArch = asmLibFileNoExt[asmLibFileNoExt.find("_gfx"):] + if globalParameters["PackageLibrary"]: + + # asmArch contains _gfxXXXX. Don't use the underscore in new path + asmLibPaths += [ os.path.join( + libDir, asmArch[1:], asmLibFile.replace(asmArch, ''))] + else: + if globalParameters["SeparateArchitectures"]: + libMetadataPaths += [ os.path.join(libDir, "TensileLibrary"+asmArch+libraryExt) ] + asmLibPaths += [ os.path.join(libDir, asmLibFile) ] + + return (solutionPaths, sourceKernelPaths, asmKernelPaths, sourceLibPaths, asmLibPaths, libMetadataPaths) + +################################################################################ +# Write CMake +################################################################################ +def writeCMake(outputPath, solutionFiles, kernelFiles, libraryStaticFiles): + print1("# Writing Custom CMake") + + # Build output file paths, using relative CMake symbol + cmakeSrcDir = "${CMAKE_SOURCE_DIR}" + (solutionPaths, sourceKernelPaths, asmKernelPaths, sourceLibPaths, asmLibPaths, _) = \ + buildObjectFilePaths(cmakeSrcDir, solutionFiles, kernelFiles, [], [], []) + + # Build full paths the static library files + staticFilePaths = [] + for staticFile in libraryStaticFiles: + staticFilePaths += [ os.path.join(cmakeSrcDir, staticFile) ] + + # Proceed to generate cmake file + generatedFile = open(os.path.join(os.path.normcase(outputPath), "Generated.cmake"), "w") + generatedFile.write(CMakeHeader) + + # write TensileClient_KERNELS symbol + generatedFile.write("set( TensileClient_KERNELS\n") + for kernelFile in sourceKernelPaths: + generatedFile.write(" %s\n" % (kernelFile)) + generatedFile.write(" )\n") + + # write TensileClient_SOURCE symbol + generatedFile.write("set( TensileClient_SOURCE\n") + for fileName in libraryStaticFiles: + generatedFile.write(" ${CMAKE_SOURCE_DIR}/%s\n" % fileName) + generatedFile.write(" )\n\n") + + generatedFile.close() + +################################################################################ +# Generate Kernel Objects From Solutions +################################################################################ +def generateKernelObjectsFromSolutions(solutions): + # create solution writer and kernel writer + kernels = [] + kernelHelperObjs = [] + kernelHelperNames = set() + + for solution in solutions: + kernels += solution.getKernels() + solutionHelperKernels = solution.getHelperKernelObjects() + kernelHelperObjs += solutionHelperKernels + for ko in solutionHelperKernels: + kernelHelperNames.add(ko.getKernelName()) + + # remove duplicates while preserving order + kernels = list(dict.fromkeys(kernels)) + kernelHelperObjs = list(dict.fromkeys(kernelHelperObjs)) + return (kernels, kernelHelperObjs, kernelHelperNames) + +################################################################################ +# Generate Logic Data and Solutions +################################################################################ +def generateLogicDataAndSolutions(logicFiles, args): + libraries = Common.ParallelMap(LibraryIO.parseLibraryLogicFile, logicFiles, "Reading logic files") + + solutions = [] + logicData = {} # keys are problemTypes, values are schedules + + masterLibraries = {} + fullMasterLibrary = None + + nextSolIndex = 0 + + for logic in Utils.tqdm(libraries, "Processing logic data"): + (scheduleName, deviceNames, problemType, solutionsForSchedule, \ + indexOrder, exactLogic, rangeLogic, newLibrary, architectureName) = logic + + if globalParameters["PackageLibrary"]: + if architectureName in masterLibraries: + masterLibraries[architectureName].merge(deepcopy(newLibrary)) + else: + masterLibraries[architectureName] = deepcopy(newLibrary) + masterLibraries[architectureName].version = args.version + elif globalParameters["SeparateArchitectures"]: + if architectureName in masterLibraries: + nextSolIndex = masterLibraries[architectureName].merge(deepcopy(newLibrary), nextSolIndex) + else: + masterLibraries[architectureName] = deepcopy(newLibrary) + masterLibraries[architectureName].version = args.version + else: + if fullMasterLibrary is None: + fullMasterLibrary = deepcopy(newLibrary) + fullMasterLibrary.version = args.version + else: + fullMasterLibrary.merge(deepcopy(newLibrary)) + + if problemType not in logicData: + logicData[problemType] = [] + logicData[problemType].append((scheduleName, deviceNames, \ + solutionsForSchedule, indexOrder, exactLogic, rangeLogic )) + + for solution in solutionsForSchedule: + solutions.append(solution) + + if globalParameters["SeparateArchitectures"] and "fallback" in masterLibraries.keys(): + for key, value in masterLibraries.items(): + if key != "fallback": + value.merge(deepcopy(masterLibraries["fallback"])) + + masterLibraries.pop("fallback") + + # remove duplicates while preserving order + solutions = list(dict.fromkeys(solutions)) + return logicData, solutions, masterLibraries, fullMasterLibrary + +################################################################################ +# Write Benchmark Client Files +################################################################################ +def writeBenchmarkClientFiles(libraryWorkingPath, tensileSourcePath, solutions, cxxCompiler): + + if not globalParameters["GenerateSourcesAndExit"]: + copyStaticFiles(libraryWorkingPath) + + kernels, kernelsBetaOnly, _ = generateKernelObjectsFromSolutions(solutions) + kernelWriterAssembly, \ + kernelMinNaming, _ = getSolutionAndKernelWriters(solutions, kernels) + + # write solution, kernels and CMake + problemType = solutions[0]["ProblemType"] + codeObjectFiles = writeSolutionsAndKernels( \ + libraryWorkingPath, cxxCompiler, [problemType], solutions, kernels, kernelsBetaOnly, \ + kernelWriterAssembly, errorTolerant=True ) + + newLibraryDir = ensurePath(os.path.join(libraryWorkingPath, 'library')) + newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary.yaml") + newLibrary = MasterSolutionLibrary.BenchmarkingLibrary(solutions) + newLibrary.applyNaming(kernelMinNaming) + + LibraryIO.writeYAML(newLibraryFile, Utils.state(newLibrary)) + + return (codeObjectFiles, newLibrary) + +def WriteClientLibraryFromSolutions(solutionList, libraryWorkingPath, tensileSourcePath = None): + + if tensileSourcePath == None: + tensileSourcePath = os.path.dirname(os.path.realpath(__file__)) + firstSolution = deepcopy(solutionList[0]) + problemType = firstSolution["ProblemType"].state + problemType["DataType"] = problemType["DataType"].value + problemType["DestDataType"] = problemType["DestDataType"].value + problemType["ComputeDataType"] = problemType["ComputeDataType"].value + cxxCompiler = globalParameters["CxxCompiler"] + + effectiveWorkingPath = os.path.join(libraryWorkingPath, "library") + ensurePath(effectiveWorkingPath) + mataDataFilePath = os.path.join(effectiveWorkingPath, 'metadata.yaml') + + metaData = {"ProblemType":problemType} + LibraryIO.writeYAML(mataDataFilePath, metaData) + + codeObjectFiles, newLibrary = writeBenchmarkClientFiles(libraryWorkingPath, tensileSourcePath, solutionList, cxxCompiler ) + + return (codeObjectFiles, newLibrary) + +################################################################################ +# Tensile Create Library +################################################################################ +def TensileCreateLibrary(): + print1("") + print1(HR) + print1("# Tensile Create Library") + print2(HR) + print2("") + + ############################################################################## + # Parse Command Line Arguments + ############################################################################## + def splitExtraParameters(par): + """ + Allows the --global-parameters option to specify any parameters from the command line. + """ + + (key, value) = par.split("=") + value = eval(value) + return (key, value) + + print2("Arguments: %s" % sys.argv) + argParser = argparse.ArgumentParser() + argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") + argParser.add_argument("OutputPath", help="Where to write library files?") + argParser.add_argument("RuntimeLanguage", help="Which runtime language?", choices=["OCL", "HIP", "HSA"]) + argParser.add_argument("--cxx-compiler", dest="CxxCompiler", choices=["hipcc"], action="store", default="hipcc") + argParser.add_argument("--cmake-cxx-compiler", dest="CmakeCxxCompiler", action="store") + argParser.add_argument("--code-object-version", dest="CodeObjectVersion", choices=["V2", "V3"], action="store", default="V3") + argParser.add_argument("--architecture", dest="Architecture", type=str, action="store", default="all", help="Supported archs: " + " ".join(architectureMap.keys())) + argParser.add_argument("--merge-files", dest="MergeFiles", action="store_true") + argParser.add_argument("--no-merge-files", dest="MergeFiles", action="store_false") + argParser.add_argument("--num-merged-files", dest="NumMergedFiles", type=int, default=1, help="Number of files the kernels should be written into.") + argParser.add_argument("--short-file-names", dest="ShortNames", action="store_true") + argParser.add_argument("--no-short-file-names", dest="ShortNames", action="store_false") + argParser.add_argument("--library-print-debug", dest="LibraryPrintDebug", action="store_true") + argParser.add_argument("--no-library-print-debug", dest="LibraryPrintDebug", action="store_false") + argParser.add_argument("--no-enumerate", action="store_true", help="Do not run rocm_agent_enumerator.") + argParser.add_argument("--package-library", dest="PackageLibrary", action="store_true", default=False) + argParser.add_argument("--embed-library", dest="EmbedLibrary", + help="Embed (new) library files into static variables. Specify the name of the library.") + + argParser.add_argument("--embed-library-key", dest="EmbedLibraryKey", default=None, + help="Access key for embedding library files.") + argParser.add_argument("--version", help="Version string to embed into library file.") + argParser.add_argument("--generate-manifest-and-exit", dest="GenerateManifestAndExit", action="store_true", + default=False, help="Output manifest file with list of expected library objects and exit.") + argParser.add_argument("--library-format", dest="LibraryFormat", choices=["yaml", "msgpack"], + action="store", default="msgpack", help="select which library format to use") + argParser.add_argument("--generate-sources-and-exit", dest="GenerateSourcesAndExit", action="store_true", + default=False, help="Output source files only and exit.") + argParser.add_argument("--jobs", "-j", dest="CpuThreads", type=int, + default=-1, help="Number of parallel jobs to launch.") + argParser.add_argument("--verbose", "-v", dest="PrintLevel", type=int, + default=1, help="Set printout verbosity level.") + argParser.add_argument("--separate-architectures", dest="SeparateArchitectures", action="store_true", + default=False, help="Separates TensileLibrary file by architecture") + argParser.add_argument("--build-client", dest="BuildClient", action="store_true", + help="Build Tensile client") + argParser.add_argument("--client-config", dest="ClientConfig", action="store_true", + help="Create client config for setting the library and code object files") + argParser.add_argument("--global-parameters", nargs="+", type=splitExtraParameters, default=[]) + + args = argParser.parse_args() + + logicPath = args.LogicPath + outputPath = args.OutputPath + CxxCompiler = args.CxxCompiler + libraryFormat = args.LibraryFormat + print2("OutputPath: %s" % outputPath) + ensurePath(outputPath) + outputPath = os.path.abspath(outputPath) + arguments = {} + arguments["RuntimeLanguage"] = args.RuntimeLanguage + arguments["CodeObjectVersion"] = args.CodeObjectVersion + arguments["Architecture"] = args.Architecture + arguments["SeparateArchitectures"] = args.SeparateArchitectures + arguments["CxxCompiler"] = args.CxxCompiler + if args.CmakeCxxCompiler: + os.environ["CMAKE_CXX_COMPILER"] = args.CmakeCxxCompiler + arguments["MergeFiles"] = args.MergeFiles + arguments["NumMergedFiles"] = args.NumMergedFiles + arguments["ShortNames"] = args.ShortNames + arguments["LibraryPrintDebug"] = args.LibraryPrintDebug + arguments["CodeFromFiles"] = False + arguments["EmbedLibrary"] = args.EmbedLibrary + arguments["LibraryFormat"] = args.LibraryFormat + if args.no_enumerate: + arguments["ROCmAgentEnumeratorPath"] = False + arguments["PackageLibrary"] = args.PackageLibrary + + arguments["GenerateManifestAndExit"] = args.GenerateManifestAndExit + + arguments["GenerateSourcesAndExit"] = args.GenerateSourcesAndExit + if arguments["GenerateSourcesAndExit"]: + # Generated sources are preserved and go into output dir + arguments["WorkingPath"] = outputPath + + arguments["CpuThreads"] = args.CpuThreads + arguments["PrintLevel"] = args.PrintLevel + + for key, value in args.global_parameters: + arguments[key] = value + + assignGlobalParameters(arguments) + + print1("# CodeObjectVersion from TensileCreateLibrary: %s" % arguments["CodeObjectVersion"]) + print1("# CxxCompiler from TensileCreateLibrary: %s" % CxxCompiler) + print1("# Architecture from TensileCreateLibrary: %s" % arguments["Architecture"]) + print1("# LibraryFormat from TensileCreateLibrary: %s" % libraryFormat) + + if not os.path.exists(logicPath): + printExit("LogicPath %s doesn't exist" % logicPath) + + if ";" in arguments["Architecture"]: + archs = arguments["Architecture"].split(";") # user arg list format + else: + archs = arguments["Architecture"].split("_") # workaround for cmake list in list issue + logicArchs = set() + for arch in archs: + if arch in architectureMap: + logicArchs.add(architectureMap[arch]) + else: + printExit("Architecture %s not supported" % arch) + + # Recursive directory search + logicFiles = [] + for root, dirs, files in os.walk(logicPath): + logicFiles += [os.path.join(root, f) for f in files + if os.path.splitext(f)[1]==".yaml" \ + and (any(logicArch in os.path.splitext(f)[0] for logicArch in logicArchs) \ + or "hip" in os.path.splitext(f)[0]) ] + + print1("# LibraryLogicFiles:" % logicFiles) + for logicFile in logicFiles: + print1("# %s" % logicFile) + + ############################################################################## + # Parse config files + ############################################################################## + + # Parse logicData, solutions, and masterLibraries from logic files + logicData, solutions, masterLibraries, fullMasterLibrary = generateLogicDataAndSolutions(logicFiles, args) + + kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) + + # if any kernels are assembly, append every ISA supported + kernelWriterAssembly, kernelMinNaming, _ = getSolutionAndKernelWriters(solutions, kernels) + + staticFiles = copyStaticFiles(outputPath) + + # Build a list of files to be expected + (solutionFiles, + sourceKernelFiles, + asmKernelFiles, + sourceLibFiles, + asmLibFiles) = buildObjectFileNames(kernelWriterAssembly, kernels, kernelHelperObjs) + + (_, + _, + _, + sourceLibPaths, + asmLibPaths, + libMetadataPaths) = buildObjectFilePaths(outputPath, solutionFiles, sourceKernelFiles, \ + asmKernelFiles, sourceLibFiles, asmLibFiles) + + # Generate manifest file + libraryPath = os.path.join(outputPath, "library") + ensurePath(libraryPath) + generatedFile = open(os.path.join(libraryPath, "TensileManifest.txt"), "w") + + # Manifest file contains YAML file, output library paths and cpp source for embedding. + for filePath in libMetadataPaths + sourceLibPaths + asmLibPaths: + generatedFile.write("%s\n" %(filePath) ) + generatedFile.close() + + if globalParameters["GenerateManifestAndExit"] == True: + return + + # generate cmake for the source kernels, + if not arguments["GenerateSourcesAndExit"]: + writeCMake(outputPath, solutionFiles, sourceKernelFiles, staticFiles) + + # Make sure to copy the library static files. + for fileName in staticFiles: + shutil.copy( os.path.join(globalParameters["SourcePath"], fileName), \ + outputPath ) + + # write solutions and kernels + problemTypes = list(logicData.keys()) + + codeObjectFiles = writeSolutionsAndKernels(outputPath, CxxCompiler, problemTypes, solutions, + kernels, kernelHelperObjs, kernelWriterAssembly) + + bothLibSet = set(sourceLibPaths + asmLibPaths) + setA = set( map( os.path.normcase, set(codeObjectFiles) ) ) + setB = set( map( os.path.normcase, bothLibSet ) ) + + sanityCheck0 = setA - setB + sanityCheck1 = setB - setA + + if globalParameters["PrintCodeCommands"]: + print("codeObjectFiles:", codeObjectFiles) + print("sourceLibPaths + asmLibPaths:", sourceLibPaths + asmLibPaths) + + assert len(sanityCheck0) == 0, "Unexpected code object files: {}".format(sanityCheck0) + if not globalParameters["GenerateSourcesAndExit"]: + assert len(sanityCheck1) == 0, "Missing expected code object files: {}".format(sanityCheck1) + + archs = [getGfxName(arch) for arch in globalParameters['SupportedISA'] \ + if globalParameters["AsmCaps"][arch]["SupportedISA"]] + newLibraryDir = ensurePath(os.path.join(outputPath, 'library')) + + if globalParameters["PackageLibrary"]: + for archName, newMasterLibrary in masterLibraries.items(): + if (archName in archs): + archPath = ensurePath(os.path.join(newLibraryDir, archName)) + masterFile = os.path.join(archPath, "TensileLibrary") + newMasterLibrary.applyNaming(kernelMinNaming) + LibraryIO.write(masterFile, Utils.state(newMasterLibrary), args.LibraryFormat) + elif globalParameters["SeparateArchitectures"]: + for archName, newMasterLibrary in masterLibraries.items(): + if archName in archs: + masterFile = os.path.join(newLibraryDir, "TensileLibrary_"+archName) + newMasterLibrary.applyNaming(kernelMinNaming) + LibraryIO.write(masterFile, Utils.state(newMasterLibrary), args.LibraryFormat) + else: + masterFile = os.path.join(newLibraryDir, "TensileLibrary") + fullMasterLibrary.applyNaming(kernelMinNaming) + LibraryIO.write(masterFile, Utils.state(fullMasterLibrary), args.LibraryFormat) + + theMasterLibrary = fullMasterLibrary + if globalParameters["PackageLibrary"] or globalParameters["SeparateArchitectures"]: + theMasterLibrary = list(masterLibraries.values())[0] + + if args.EmbedLibrary is not None: + embedFileName = os.path.join(outputPath, "library/{}.cpp".format(args.EmbedLibrary)) + with EmbeddedData.EmbeddedDataFile(embedFileName) as embedFile: + + ext = ".yaml" if globalParameters["LibraryFormat"] == "yaml" else ".dat" + embedFile.embed_file(theMasterLibrary.cpp_base_class, masterFile + ext, nullTerminated=True, + key=args.EmbedLibraryKey) + + for co in Utils.tqdm(codeObjectFiles): + embedFile.embed_file("SolutionAdapter", co, nullTerminated=False, + key=args.EmbedLibraryKey) + + if args.BuildClient: + print1("# Building Tensile Client") + ClientExecutable.getClientExecutable(outputPath) + + if args.ClientConfig: + # write simple ini for best solution mode linked to library we just made + iniFile = os.path.join(outputPath, "best-solution.ini") + with open(iniFile, "w") as f: + def param(key, value): + f.write("{}={}\n".format(key, value)) + + libraryFile = masterFile + ".yaml" \ + if globalParameters["LibraryFormat"] == "yaml" else masterFile + ".dat" + + param("library-file", libraryFile) + for coFile in codeObjectFiles: + param("code-object", os.path.join(outputPath,coFile)) + + param("best-solution", True) + + + print1("# Tensile Library Writer DONE") + print1(HR) + print1("") diff --git a/tensilelite/Tensile/TensileInstructions/Base.py b/tensilelite/Tensile/TensileInstructions/Base.py new file mode 100644 index 0000000000..3a564abf31 --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/Base.py @@ -0,0 +1,274 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Formatting import __TI_DEBUG_LEVEL__, printExit + +from dataclasses import dataclass +from typing import Tuple + +import subprocess +import threading + +class TensileInstructions: + + _instance = None + _lock = threading.Lock() + + def __new__(cls, *args, **kwargs): + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._isaInfo = {} # type: ignore + cls._instance._kernelInfo = {} + return cls._instance + + @dataclass + class IsaInfo: + assemblerPath: str + asmCaps: dict + archCaps: dict + asmBugs: dict + + @dataclass + class kernelInfo: + isa: Tuple[int, int, int] + wavefrontSize: int = 64 + + def init(self, isaVersion: Tuple[int, int, int], assemblerPath: str, debug: bool=False) -> None: + with self._lock: + if len(self._kernelInfo) > 1000: + self._kernelInfo = _removeIdent(self._kernelInfo) + self._kernelInfo[threading.get_ident()] = TensileInstructions.kernelInfo(isa=isaVersion) + if isaVersion not in self._isaInfo: # type: ignore + asmCaps = _initAsmCaps(isaVersion, assemblerPath, debug) + archCaps = _initArchCaps(isaVersion) + asmBugs = _initAsmBugs(asmCaps) + self._isaInfo[isaVersion] = TensileInstructions.IsaInfo(assemblerPath, # type: ignore + asmCaps, archCaps, asmBugs) + + def setDebugLevel(self, level: int) -> None: + __TI_DEBUG_LEVEL__ = level + + def setKernelInfo(self, isaVersion: Tuple[int, int, int], wavefrontSize: int) -> None: + if isaVersion not in self._isaInfo: # type: ignore + printExit("Current isa %s not initialized."%str(isaVersion)) + with self._lock: + if len(self._kernelInfo) > 1000: + self._kernelInfo = _removeIdent(self._kernelInfo) + tid = threading.get_ident() + if tid not in self._kernelInfo: + self._kernelInfo[threading.get_ident()] = \ + TensileInstructions.kernelInfo(isa=isaVersion, wavefrontSize=wavefrontSize) + else: + self._kernelInfo[threading.get_ident()].isa = isaVersion + self._kernelInfo[threading.get_ident()].wavefrontSize = wavefrontSize + + def getCurrentIsa(self) -> Tuple[int]: + return self._kernelInfo[threading.get_ident()].isa + + def getAsmCaps(self) -> dict: + return self._isaInfo[self._kernelInfo[threading.get_ident()].isa].asmCaps # type: ignore + + def getArchCaps(self) -> dict: + return self._isaInfo[self._kernelInfo[threading.get_ident()].isa].archCaps # type: ignore + + def getAsmBugs(self) -> dict: + return self._isaInfo[self._kernelInfo[threading.get_ident()].isa].asmBugs # type: ignore + + def getKernel(self) -> kernelInfo: + return self._kernelInfo[threading.get_ident()] + +def printItemList(listOfItems, tag="__unnamed__") -> None: + header = "="*40 + print("%s\nbegin list %s\n%s"%(header, tag, header)) + for i, item in enumerate(listOfItems): + item = list(item) if isinstance(item, tuple) else [item] + print("list[%s] %s"%(i, "-"*30)) + for j, t in enumerate(item): + ostream = t.prettyPrint() + ostream = ostream[:-1] if len(ostream)>0 and ostream[-1:] == '\n' else ostream + print(ostream) + print("%s\nend list %s\n%s"%(header, tag, header)) + +# Global +_global_ti = TensileInstructions() + +class Item: + """ + Base class for Modules, Instructions, etc + Item is a atomic collection of or more instructions and commentsA + """ + + def __init__(self, name: str="") -> None: + self.parent = "" + self.name = name + + @property + def asmCaps(self) -> dict: + return _global_ti.getAsmCaps() + + @property + def archCaps(self) -> dict: + return _global_ti.getArchCaps() + + @property + def asmBugs(self) -> dict: + return _global_ti.getAsmBugs() + + @property + def kernel(self) -> TensileInstructions.kernelInfo: + return _global_ti.getKernel() + + def countType(self, ttype) -> int: + return int(isinstance(self, ttype)) + + def prettyPrint(self, indent="") -> str: + ostream = "" + ostream += "%s%s "%(indent, type(self).__name__) + ostream += str(self) + return ostream + +def getGfxName(arch: Tuple[int, int, int]) -> str: + # convert last digit to hex because reasons + name = str(arch[0]) + str(arch[1]) + ('%x' % arch[2]) + return 'gfx' + ''.join(map(str,name)) + +def _removeIdent(isaDict) -> list: + ids = [th.ident for th in threading.enumerate()] + isaDict = [id for id in isaDict if id in ids] + return isaDict + +def _tryAssembler(isaVersion: Tuple[int, int, int], assemblerPath: str, asmString: str, \ + debug: bool=False, *options) -> bool: + """ + Try to assemble the asmString for the specified target processor + Success is defined as assembler returning no error code or stderr/stdout + """ + options = list(options) + + if isaVersion[0] >= 10: + options += ['-mwavefrontsize64'] + + args = [assemblerPath, '-x', 'assembler', + '-target', 'amdgcn-amdhsa', + '-mcpu='+ getGfxName(isaVersion), + '-mcode-object-version=3', + *options, + '-'] + + result = subprocess.run(args, input=asmString.encode(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = result.stdout.decode() + + if debug: + print("isaVersion: ", isaVersion) + print("asm_cmd:", ' '.join(args)) + print("asmString: ", asmString) + print("output: ", output) + print("return code: ", result.returncode) + + if output != "" or result.returncode != 0: + return False + return True + +######################################## +# Get Caps +######################################## + +def _initAsmCaps(isaVersion, assemblerPath, isDebug) -> dict: + """ Determine assembler capabilities by testing short instructions sequences """ + rv = {} + rv["SupportedISA"] = _tryAssembler(isaVersion, assemblerPath, "", isDebug) + rv["HasExplicitCO"] = _tryAssembler(isaVersion, assemblerPath, "v_add_co_u32 v0,vcc,v0,1", isDebug) + rv["HasExplicitNC"] = _tryAssembler(isaVersion, assemblerPath, "v_add_nc_u32 v0,v0,1", isDebug) + + rv["HasDirectToLds"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dword v40, v36, s[24:27], s28 offen offset:0 lds", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "buffer_load_b32 v40, v36, s[24:27], s28 offen offset:0 lds", isDebug) + rv["HasAddLshl"] = _tryAssembler(isaVersion, assemblerPath, "v_add_lshl_u32 v47, v36, v34, 0x2", isDebug) + rv["HasLshlOr"] = _tryAssembler(isaVersion, assemblerPath, "v_lshl_or_b32 v47, v36, 0x2, v34", isDebug) + rv["HasSMulHi"] = _tryAssembler(isaVersion, assemblerPath, "s_mul_hi_u32 s47, s36, s34", isDebug) + rv["HasCodeObjectV3"] = _tryAssembler(isaVersion, assemblerPath, "", isDebug, "-mcode-object-version=2") + + rv["HasMFMA"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x2bf16 a[0:31], v32, v33, a[0:31]", isDebug) + rv["HasMFMA_f64"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f64_16x16x4f64 v[0:7], v[32:33], v[36:37], v[0:7]", isDebug) + rv["HasMFMA_bf16_1k"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x4bf16_1k a[0:31], v[32:33], v[36:37], a[0:31]", isDebug) + + rv["v_mac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f16 v47, v36, v34", isDebug) + + rv["v_fma_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0,0]", isDebug) + rv["v_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34", isDebug) + + rv["v_pk_fma_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0]", isDebug) + rv["v_pk_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34", isDebug) + + rv["v_pk_add_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_add_f32 v[48:49], v[36:37], v[0:1]", isDebug) + + rv["v_mad_mix_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mad_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", isDebug) + rv["v_fma_mix_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", isDebug) + + rv["v_dot2_f32_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_dot2_f32_f16 v20, v36, v34, v20", isDebug) + rv["v_dot2c_f32_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_dot2c_f32_f16 v47, v36, v34", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "v_dot2acc_f32_f16 v47, v36, v34", isDebug) + + rv["v_dot4_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34", isDebug) + rv["v_dot4c_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4c_i32_i8 v47, v36, v34", isDebug) + rv["VOP3v_dot4_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34, v47", isDebug) + + rv["v_mac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f32 v20, v21, v22", isDebug) + rv["v_fma_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f32 v20, v21, v22, v23", isDebug) + rv["v_fmac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fmac_f32 v20, v21, v22", isDebug) + + rv["v_fma_f64"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f64 v[20:21], v[22:23], v[24:25], v[20:21]", isDebug) + + rv["HasAtomicAdd"] = _tryAssembler(isaVersion, assemblerPath, "buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:0", isDebug) + + + if _tryAssembler(isaVersion, assemblerPath, "s_waitcnt vmcnt(63)", isDebug): + rv["MaxVmcnt"] = 63 + elif _tryAssembler(isaVersion, assemblerPath, "s_waitcnt vmcnt(15)", isDebug): + rv["MaxVmcnt"] = 15 + else: + rv["MaxVmcnt"] = 0 + + # TODO- Need to query the max cap, just like vmcnt as well? + rv["MaxLgkmcnt"] = 15 + + rv["SupportedSource"] = True + + return rv + +def _initArchCaps(isaVersion) -> dict: + rv = {} + rv["HasEccHalf"] = (isaVersion==(9,0,6) or isaVersion==(9,0,8) or isaVersion==(9,0,10)) + rv["Waitcnt0Disabled"] = (isaVersion == (9,0,8) or isaVersion==(9,0,10)) + rv["SeparateVscnt"] = isaVersion[0] in (10, 11) + rv["CMPXWritesSGPR"] = isaVersion[0] not in (10, 11) + rv["HasWave32"] = isaVersion[0] in (10, 11) + rv["HasAccCD"] = (isaVersion==(9,0,10)) + rv["ArchAccUnifiedRegs"] = (isaVersion==(9,0,10)) + + return rv + +def _initAsmBugs(asmCaps) -> dict: + rv = {} + rv["ExplicitCO"] = asmCaps["HasExplicitCO"] + rv["ExplicitNC"] = asmCaps["HasExplicitNC"] + + return rv diff --git a/tensilelite/Tensile/TensileInstructions/Code.py b/tensilelite/Tensile/TensileInstructions/Code.py new file mode 100644 index 0000000000..90ade533db --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/Code.py @@ -0,0 +1,859 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Base import Item, getGfxName +from .Enums import SignatureValueKind +from .Formatting import slash, slash50, block, block3Line, blockNewLine, \ + formatStr, printExit +from .Instructions import Instruction, MacroInstruction + +from math import ceil +from typing import Optional +import ctypes + +# Global to print module names around strings +printModuleNames = 0 + +class Label (Item): + """ + Label that can be the target of a jump. + """ + def __init__(self, label, comment): + super().__init__("") + assert(isinstance(label, int) or isinstance(label, str)) + self.label = label + self.comment = comment + + @staticmethod + def getFormatting(label): + if isinstance(label, int): + return "label_%04u" % (label) + else: + return "label_%s" % (label) + + def getLabelName(self): + return Label.getFormatting(self.label) + + def __str__(self): + t = self.getLabelName() + ":" + if self.comment: + t += " /// %s" % self.comment + t += "\n" + return t + +class Macro(Item): + def __init__(self, *args): + super().__init__("") + self.itemList = [] + self.macro = "" + self.addTitle(*args) + + def addTitle(self, *args): + self.name = args[0] + self.macro = MacroInstruction(name=args[0], args=list(args[1:])) + + def add(self, item): + # This is a workaround + if isinstance(item, (Instruction, Module, TextBlock)): + item.parent = self # type: ignore + self.itemList.append(item) + else: + assert 0, "unknown item type (%s) for Code.add. item=%s"%(type(item), item) + return item + + def addComment0(self, comment): + """ + Convenience function to format arg as a comment and add TextBlock item + This comment is a single line /* MYCOMMENT */ + """ + self.add(TextBlock("/* %s */\n"%comment)) + + def prettyPrint(self,indent=""): + ostream = "" + ostream += '%s%s "%s"\n'%(indent, type(self).__name__, self.name) + for i in self.itemList: + ostream += i.prettyPrint(indent.replace("|--", "| ") + "|--") + return ostream + + def setItems(self, itemList): + self.itemList = itemList + + def items(self): + """ + Return list of items in the Macro + Items may be other Inst + """ + return self.itemList + + def __str__(self): + assert(self.macro) + s = "" + if printModuleNames: + s += "// %s { \n" % self.name + s += ".macro " + str(self.macro).replace(",", "") + s += "".join([(" " + str(x).replace(",", "")) for x in self.itemList]) + s += ".endm\n" + if printModuleNames: + s += "// } %s\n" % self.name + return s + +class Module(Item): + """ + Modules contain lists of text instructions, Inst objects, or additional modules + They can be easily converted to string that represents all items in the list + and can be mixed with standard text. + The intent is to allow the kernel writer to express the structure of the + code (ie which instructions are a related module) so the scheduler can later + make intelligent and legal transformations. + """ + def __init__(self, name="") -> None: + super().__init__(name) + self.itemList = [] + self.tempVgpr = None + self._isNoOpt = False + + def setNoOpt(self, noOpt: bool) -> None: + self._isNoOpt = noOpt + + def isNoOpt(self) -> bool: + return self._isNoOpt + + def findNamedItem(self, targetName): + return next((item for item in self.itemList if item.name==targetName), None) + + def setInlineAsmPrintMode(self, mode): + for item in self.itemList: + if isinstance(item, Module): + item.setInlineAsmPrintMode(mode) + elif isinstance(item, Instruction): + item.setInlineAsm(mode) + + def __str__(self): + s = "" + if printModuleNames: + s += "// %s { \n" % self.name + s += "".join([str(x) for x in self.itemList]) + if printModuleNames: + s += "// } %s\n" % self.name + return s + + def addSpaceLine(self): + self.itemList.append(TextBlock("\n")) + + def add(self, item, pos=-1): + """ + Add specified item to the list of items in the module. + Item MUST be a Item (not a string) - can use + addText(...)) to add a string. + All additions to itemList should use this function. + + Returns item to facilitate one-line create/add patterns + """ + if isinstance(item, Item): + item.parent = self # type: ignore + if pos == -1: + self.itemList.append(item) + else: + self.itemList.insert(pos, item) + else: + assert 0, "unknown item type (%s) for Module.add. item=%s"%(type(item), item) + return item + + def appendModule(self, module): + """ + Append items to module. + """ + assert(isinstance(module, Module)) + for i in module.items(): + self.add(i) + return module + + def addModuleAsFlatItems(self, module): + """ + Add items to module. + + Returns items to facilitate one-line create/add patterns + """ + assert(isinstance(module, Module)) + for i in module.flatitems(): + self.add(i) + return module + + def findIndex(self, targetItem): + if isinstance(targetItem, Item): + return self.itemList.index(targetItem) + return -1 + + def addComment(self, comment): + """ + Convenience function to format arg as a comment and add TextBlock item + This comment is a single line // MYCOMMENT + """ + self.add(TextBlock(slash(comment))) + + def addCommentAlign(self, comment): + """ + Convenience function to format arg as a comment and add TextBlock item + This comment is a single line // MYCOMMENT with the same format as + Instruction. + """ + self.add(TextBlock(slash50(comment))) + + def addComment0(self, comment): + """ + Convenience function to format arg as a comment and add TextBlock item + This comment is a single line /* MYCOMMENT */ + """ + self.add(TextBlock(block(comment))) + + def addComment1(self, comment): + """ + Convenience function to format arg as a comment and add TextBlock item + This comment is a blank line followed by /* MYCOMMENT */ + """ + self.add(TextBlock(blockNewLine(comment))) + + def addComment2(self, comment): + self.add(TextBlock(block3Line(comment))) + + def prettyPrint(self,indent=""): + ostream = "" + ostream += '%s%s "%s"\n'%(indent, type(self).__name__, self.name) + for i in self.itemList: + ostream += i.prettyPrint(indent.replace("|--", "| ") + "|--") + return ostream + """ + Test code: + mod1 = Code.Module("TopModule") + mod2 = Code.Module("Module-lvl2") + mod2.add(Code.Inst("bogusInst", "comments")) + mod3 = Code.Module("Module-lvl3") + mod3.add(Code.TextBlock("bogusTextBlock\nbogusTextBlock2\nbogusTextBlock3")) + mod3.add(Code.GlobalReadInst("bogusGlobalReadInst", "comments")) + mod2.add(mod3) + mod1.add(mod2) + mod1.add(Code.Inst("bogusInst", "comments")) + mod1.add(mod2) + + print(mod1.prettyPrint()) + Output: + Module "TopModule" + |--Module "Module-lvl2" + | |--Inst bogusInst // comments + | |--Module "Module-lvl3" + | | |--TextBlock + | | | |--bogusTextBlock + | | | |--bogusTextBlock2 + | | | |--bogusTextBlock3 + | | |--GlobalReadInst bogusGlobalReadInst // comments + |--Inst bogusInst // comments + |--Module "Module-lvl2" + | |--Inst bogusInst // comments + | |--Module "Module-lvl3" + | | |--TextBlock + | | | |--bogusTextBlock + | | | |--bogusTextBlock2 + | | | |--bogusTextBlock3 + | | |--GlobalReadInst bogusGlobalReadInst // comments + """ + + def countTypeList(self, ttypeList): + count = 0 + # add "Module" type to type list filter, where we want to count recursively + # the types under "Module" + if Module not in ttypeList: + ttypeList.append(Module) + for ttype in ttypeList: + count += self.countType(ttype) + return count + + def countType(self,ttype): + """ + Count number of items with specified type in this Module + Will recursively count occurrences in submodules + (Overrides Item.countType) + """ + count=0 + for i in self.itemList: + if isinstance(i, Module): + count += i.countType(ttype) + else: + count += int(isinstance(i, ttype)) + return count + + def count(self): + count=0 + for i in self.itemList: + if isinstance(i, Module): + count += i.count() + else: + count += 1 + return count + + def setItems(self, itemList): + self.itemList = itemList + + def items(self): + """ + Return list of items in the Module + Items may be other Modules, TexBlock, or Inst + """ + return self.itemList + + def replaceItem(self, srcItem, dstItem): + """ + Replace item from itemList. + Items may be other Modules, TexBlock, or Inst + """ + for index, s in enumerate(self.itemList): + if s is srcItem: + dstItem.parent = self + self.itemList[index] = dstItem + break + + def replaceItemByIndex(self, index, item): + """ + Replace item from itemList, do nothing if + exceed length of the itemList + Items may be other Modules, TexBlock, or Inst + """ + if index >= len(self.itemList): + return + item.parent = self + self.itemList[index] = item + + def removeItemByIndex(self, index): + """ + Remove item from itemList, remove the last element if + exceed length of the itemList + Items may be other Modules, TexBlock, or Inst + """ + if index >= len(self.itemList): + index = -1 + del self.itemList[index] + + def removeItem(self, item): + self.itemList = [ x for x in self.itemList if x is not item ] + + def removeItemsByName(self, name): + """ + Remove items from itemList + Items may be other Modules, TexBlock, or Inst + """ + self.itemList = [ x for x in self.itemList if x.name != name ] + + def flatitems(self): + """ + Return flattened list of items in the Module + Items in sub-modules will be flattened into single list + Items may be TexBlock or Inst + """ + flatitems = [] + for i in self.itemList: + if isinstance(i, Module): + flatitems += i.flatitems() + else: + flatitems.append(i) + return flatitems + + def addTempVgpr(self, vgpr): + self.tempVgpr = vgpr + +class StructuredModule(Module): + def __init__(self, name=""): + Module.__init__(self, name) + self.header = Module("header") + self.middle = Module("middle") + self.footer = Module("footer") + + self.add(self.header) + self.add(self.middle) + self.add(self.footer) + +class TextBlock(Item): + """ + An unstructured block of text + """ + def __init__(self, text: str): + super().__init__(text) + self.text = text + + def __str__(self) -> str: + return self.text + +class ValueEndif(Item): + def __init__(self, comment=""): + super().__init__("ValueEndif") + self.comment = comment + + def __str__(self): + return formatStr(False, ".endif", self.comment) + +class ValueIf(Item): + def __init__(self, value: int): + super().__init__("ValueIf") + self.value = value + + def __str__(self): + return ".if " + str(self.value) + +class ValueSet(Item): + def __init__(self, name, value, offset = 0, format = 0): + super().__init__(name) + if isinstance(value, int): + self.ref = None + self.value = value + elif isinstance(value, str): + self.ref = value + self.value = None + self.offset = offset + # -1 for no offset, 0 for dec, 1 for hex + self.format = format + + def __str__(self): + t = ".set " + self.name + ", " + if self.ref != None: + if self.format == -1: + t += str(self.ref) + else: + t += "%s+%u" % (self.ref, self.offset) + elif self.value != None: + if self.format == -1: + t += str(self.value) + elif self.format == 0: + t += str(self.value + self.offset) + elif self.format == 1: + t += "0x{0:08x}".format(self.value + self.offset) + t += "\n" + return t + +class RegSet(ValueSet): + def __init__(self, regType, name, value, offset = 0): + super().__init__(name, value, offset) + # v or s + self.regType = regType + +class BitfieldStructure(ctypes.Structure): + def field_desc(self, field): + fname = field[0] + bits = " ({}b)".format(field[2]) if len(field) > 2 else "" + value = getattr(self, fname) + return "{0}{1}: {2}".format(fname, bits, value) + + def desc(self): + return '\n'.join([self.field_desc(field) for field in self._fields_]) + +class BitfieldUnion(ctypes.Union): + def __str__(self): + return "0x{0:08x}".format(self.value) + + def getValue(self): + return self.value + + def desc(self): + return "hex: {}\n".format(self) + self.fields.desc() + +class SrdUpperFields9XX(BitfieldStructure): + _fields_ = [("dst_sel_x", ctypes.c_uint, 3), + ("dst_sel_y", ctypes.c_uint, 3), + ("dst_sel_z", ctypes.c_uint, 3), + ("dst_sel_w", ctypes.c_uint, 3), + ("num_format", ctypes.c_uint, 3), + ("data_format", ctypes.c_uint, 4), + ("user_vm_enable", ctypes.c_uint, 1), + ("user_vm_mode", ctypes.c_uint, 1), + ("index_stride", ctypes.c_uint, 2), + ("add_tid_enable", ctypes.c_uint, 1), + ("_unusedA", ctypes.c_uint, 3), + ("nv", ctypes.c_uint, 1), + ("_unusedB", ctypes.c_uint, 2), + ("type", ctypes.c_uint, 2)] + + @classmethod + def default(cls): + return cls(data_format = 4) + +class SrdUpperValue9XX(BitfieldUnion): + _fields_ = [("fields", SrdUpperFields9XX), ("value", ctypes.c_uint32)] + + @classmethod + def default(cls): + return cls(fields=SrdUpperFields9XX.default()) + +class SrdUpperFields10XX(BitfieldStructure): + _fields_ = [("dst_sel_x", ctypes.c_uint, 3), + ("dst_sel_y", ctypes.c_uint, 3), + ("dst_sel_z", ctypes.c_uint, 3), + ("dst_sel_w", ctypes.c_uint, 3), + ("format", ctypes.c_uint, 7), + ("_unusedA", ctypes.c_uint, 2), + ("index_stride", ctypes.c_uint, 2), + ("add_tid_enable", ctypes.c_uint, 1), + ("resource_level", ctypes.c_uint, 1), + ("_unusedB", ctypes.c_uint, 1), + ("LLC_noalloc", ctypes.c_uint, 2), + ("oob_select", ctypes.c_uint, 2), + ("type", ctypes.c_uint, 2)] + + + @classmethod + def default(cls): + return cls(format = 4, + resource_level = 1, + oob_select = 3) + + +class SrdUpperValue10XX(BitfieldUnion): + _fields_ = [("fields", SrdUpperFields10XX), ("value", ctypes.c_uint32)] + + @classmethod + def default(cls): + return cls(fields=SrdUpperFields10XX.default()) + + +class SrdUpperFields11XX(BitfieldStructure): + _fields_ = [("dst_sel_x", ctypes.c_uint, 3), + ("dst_sel_y", ctypes.c_uint, 3), + ("dst_sel_z", ctypes.c_uint, 3), + ("dst_sel_w", ctypes.c_uint, 3), + ("format", ctypes.c_uint, 7), + ("_unusedA", ctypes.c_uint, 2), + ("index_stride", ctypes.c_uint, 2), + ("add_tid_enable", ctypes.c_uint, 1), + ("resource_level", ctypes.c_uint, 1), + ("_unusedB", ctypes.c_uint, 1), + ("LLC_noalloc", ctypes.c_uint, 2), + ("oob_select", ctypes.c_uint, 2), + ("type", ctypes.c_uint, 2)] + + @classmethod + def default(cls): + return cls(format = 4, + resource_level = 1, + oob_select = 3) + +class SrdUpperValue11XX(BitfieldUnion): + _fields_ = [("fields", SrdUpperFields11XX), ("value", ctypes.c_uint32)] + + @classmethod + def default(cls): + return cls(fields=SrdUpperFields11XX.default()) + +def SrdUpperValue(isa): + if isa[0] == 11: + return SrdUpperValue11XX.default() + elif isa[0] == 10: + return SrdUpperValue10XX.default() + else: + return SrdUpperValue9XX.default() + +######################################## +# Signatures +######################################## + +class _SignatureArgument(Item): + + ValueTypeSizeDict = {'i8': 1, + 'i16': 2, + 'i32': 4, + 'i64': 8, + 'u8': 1, + 'u16': 2, + 'u32': 4, + 'u64': 8, + 'bf16': 2, + 'f16': 2, + 'f32': 4, + 'f64': 8, + 'pkf16': 4, + 'struct': 8 + } + + def __init__(self, name, valueKind, valueType, addrSpaceQual = None): + super().__init__(name) + self.valueKind = valueKind + self.valueType = valueType + self.size = self.valueToSize(valueKind, valueType) + + self.addrSpaceQual = addrSpaceQual + + def valueToSize(self, valueKind, valueType): + if valueKind == SignatureValueKind.SIG_GLOBALBUFFER: + return 8 + + return _SignatureArgument.ValueTypeSizeDict[valueType] + + def valueKindToStr(self): + if self.valueKind == SignatureValueKind.SIG_GLOBALBUFFER: + return "global_buffer" + elif self.valueKind == SignatureValueKind.SIG_VALUE: + return "by_value" + +class _SignatureArgumentV2(_SignatureArgument): + def __init__(self, align, name, valueKind, valueType, addrSpaceQual=None): + super().__init__(name, valueKind, valueType, addrSpaceQual) + self.align = align + + def __str__(self): + signatureIndent = " " * 8 + kStr = "" + kStr += signatureIndent[2:] + "- Name: %s\n" % self.name + kStr += signatureIndent + "Size: %s\n" % self.size + kStr += signatureIndent + "Align: %s\n" % self.align + kStr += signatureIndent + "ValueKind: %s\n" % self.valueKindToStr() + kStr += signatureIndent + "ValueType: %s\n" % self.valueType + if self.addrSpaceQual != None: + kStr += signatureIndent + "AddrSpaceQual: %s\n" % self.addrSpaceQual + return kStr + +class _SignatureArgumentV3(_SignatureArgument): + def __init__(self, offset, name, valueKind, valueType, addrSpaceQual=None): + super().__init__(name, valueKind, valueType, addrSpaceQual) + self.offset = offset + + def __str__(self): + signatureIndent = " " * 8 + kStr = "" + kStr += signatureIndent[2:] + "- .name: %s\n" % self.name + kStr += signatureIndent + ".size: %s\n" % self.size + kStr += signatureIndent + ".offset: %s\n" % self.offset + kStr += signatureIndent + ".value_kind: %s\n" % self.valueKindToStr() + kStr += signatureIndent + ".value_type: %s\n" % self.valueType + if self.addrSpaceQual != None: + kStr += signatureIndent + ".address_space: %s\n" % self.addrSpaceQual + return kStr + +class _SignatureKernelDescriptorV3(Item): + def __init__(self, name, groupSegSize, sgprWorkGroup, vgprWorkItem, \ + totalVgprs: int=0, totalAgprs: int=0, totalSgprs: int =0): + super().__init__(name) + # accumulator offset for Unified Register Files + if self.archCaps["ArchAccUnifiedRegs"]: + self.accumOffset = ceil(totalVgprs/8)*8 + self.totalVgprs = self.accumOffset + totalAgprs + else: + self.accumOffset = None + self.totalVgprs = totalVgprs + self.originalTotalVgprs = totalVgprs + self.totalAgprs = totalAgprs + self.totalSgprs = totalSgprs + self.groupSegSize = groupSegSize + self.sgprWorkGroup = sgprWorkGroup + self.vgprWorkItem = vgprWorkItem + + def setGprs(self, totalVgprs: int, totalAgprs: int, totalSgprs: int): + if self.archCaps["ArchAccUnifiedRegs"]: + self.accumOffset = ceil(totalVgprs/8)*8 + self.totalVgprs = self.accumOffset + totalAgprs + else: + self.accumOffset = None + self.totalVgprs = totalVgprs + self.originalTotalVgprs = totalVgprs + self.totalAgprs = totalAgprs + self.totalSgprs = totalSgprs + + def __str__(self): + kdIndent = " " * 2 + kStr = "" + kStr += ".amdgcn_target \"amdgcn-amd-amdhsa--%s\"\n" % getGfxName(self.kernel.isa) + kStr += ".text\n" + kStr += ".protected %s\n" % self.name + kStr += ".globl %s\n" % self.name + kStr += ".p2align 8\n" + kStr += ".type %s,@function\n" % self.name + kStr += ".section .rodata,#alloc\n" + kStr += ".p2align 6\n" + kStr += ".amdhsa_kernel %s\n" % self.name + kStr += kdIndent + ".amdhsa_user_sgpr_kernarg_segment_ptr 1\n" + if self.accumOffset != None: + kStr += kdIndent + ".amdhsa_accum_offset %u // accvgpr offset\n" % self.accumOffset + kStr += kdIndent + ".amdhsa_next_free_vgpr %u // vgprs\n" % self.totalVgprs + kStr += kdIndent + ".amdhsa_next_free_sgpr %u // sgprs\n" % self.totalSgprs + kStr += kdIndent + ".amdhsa_group_segment_fixed_size %u // lds bytes\n" % self.groupSegSize + if self.archCaps["HasWave32"]: + if self.kernel.wavefrontSize == 32: + kStr += kdIndent + ".amdhsa_wavefront_size32 1 // 32-thread wavefronts\n" + else: + kStr += kdIndent + ".amdhsa_wavefront_size32 0 // 64-thread wavefronts\n" + kStr += kdIndent + ".amdhsa_private_segment_fixed_size 0\n" + kStr += kdIndent + ".amdhsa_system_sgpr_workgroup_id_x %u\n" % self.sgprWorkGroup[0] + kStr += kdIndent + ".amdhsa_system_sgpr_workgroup_id_y %u\n" % self.sgprWorkGroup[1] + kStr += kdIndent + ".amdhsa_system_sgpr_workgroup_id_z %u\n" % self.sgprWorkGroup[2] + kStr += kdIndent + ".amdhsa_system_vgpr_workitem_id %u\n" % self.vgprWorkItem + kStr += kdIndent + ".amdhsa_float_denorm_mode_32 3\n" + kStr += kdIndent + ".amdhsa_float_denorm_mode_16_64 3\n" + kStr += ".end_amdhsa_kernel\n" + kStr += ".text\n" + kStr += block("Num VGPR =%u"%self.originalTotalVgprs) + kStr += block("Num AccVGPR=%u"%self.totalAgprs) + kStr += block("Num SGPR =%u"%self.totalSgprs) + return kStr + + def prettyPrint(self, indent=""): + ostream = "" + ostream += "%s%s "%(indent, type(self).__name__) + return ostream + +class SignatureCodeMetaV3(Item): + def __init__(self, name, groupSegSize, flatWgSize, totalVgprs = 0, totalSgprs=0): + super().__init__(name) + self.groupSegSize = groupSegSize + self.totalVgprs = totalVgprs + self.totalSgprs = totalSgprs + self.flatWgSize = flatWgSize + self.offset = 0 + self.argList = [] + + def setGprs(self, totalVgprs: int, totalSgprs: int): + self.totalVgprs = totalVgprs + self.totalSgprs = totalSgprs + + def __str__(self): + kStr = "" + kStr += ".amdgpu_metadata\n" + kStr += "---\n" + kStr += "amdhsa.version:\n" + kStr += " - 1\n" + kStr += " - 0\n" + kStr += "amdhsa.kernels:\n" + kStr += " - .name: %s\n" % self.name + kStr += " .symbol: '%s.kd'\n" % self.name + kStr += " .language: OpenCL C\n" + kStr += " .language_version:\n" + kStr += " - 2\n" + kStr += " - 0\n" + kStr += " .args:\n" + for i in self.argList: + kStr += str(i) + kStr += " .group_segment_fixed_size: %u\n" % self.groupSegSize + kStr += " .kernarg_segment_align: %u\n" % 8 + kStr += " .kernarg_segment_size: %u\n" % (((self.offset+7)//8)*8) # round up to .kernarg_segment_align + kStr += " .max_flat_workgroup_size: %u\n" % self.flatWgSize + kStr += " .private_segment_fixed_size: %u\n" % 0 + kStr += " .sgpr_count: %u\n" % self.totalSgprs + kStr += " .sgpr_spill_count: %u\n" % 0 + kStr += " .vgpr_count: %u\n" % self.totalVgprs + kStr += " .vgpr_spill_count: %u\n" % 0 + kStr += " .wavefront_size: %u\n" % self.kernel.wavefrontSize + + kStr += "...\n" + kStr += ".end_amdgpu_metadata\n" + kStr += "%s:\n" % self.name + return kStr + + def addArg(self, name: str, kind: SignatureValueKind, type: str, addrSpaceQual: Optional[str]=None): + sa = _SignatureArgumentV3(self.offset, name, kind, type, addrSpaceQual) + self.argList.append(sa) + self.offset += sa.size + + def prettyPrint(self, indent=""): + ostream = "" + ostream += "%s%s "%(indent, type(self).__name__) + return ostream + +class SignatureBase(Item): + def __init__(self, kernelName, codeObjectVersion, groupSegmentSize, sgprWorkGroup, \ + vgprWorkItem, flatWorkGroupSize, totalVgprs: int=0, totalAgprs: int=0, totalSgprs: int=0) -> None: + super().__init__(kernelName) + self.codeObjectVersion = codeObjectVersion + + # Internal data + if self.codeObjectVersion == "v2": + printExit("Does not support v2 signatures.") + elif self.codeObjectVersion == "v3": + self.kernelDescriptor = _SignatureKernelDescriptorV3(name=kernelName, + totalVgprs=totalVgprs, + totalAgprs=totalAgprs, + totalSgprs=totalSgprs, + groupSegSize=groupSegmentSize, + sgprWorkGroup=sgprWorkGroup, + vgprWorkItem=vgprWorkItem) + self.codeMeta = SignatureCodeMetaV3(name=kernelName, + groupSegSize=groupSegmentSize, + totalVgprs=totalVgprs, + totalSgprs=totalSgprs, + flatWgSize=flatWorkGroupSize) + + # Comment description + self.descriptionTopic = None + self.descriptionList = [] + + def setGprs(self, totalVgprs: int, totalAgprs: int, totalSgprs: int): + self.kernelDescriptor.setGprs(totalVgprs=totalVgprs, totalAgprs=totalAgprs, \ + totalSgprs=totalSgprs) + self.codeMeta.setGprs(totalVgprs=totalVgprs, totalSgprs=totalSgprs) + + def addArg(self, name: str, kind: SignatureValueKind, type: str, addrSpaceQual: Optional[str]=None): + self.codeMeta.addArg(name, kind, type, addrSpaceQual) + + def addDescriptionTopic(self, text: str): + self.descriptionTopic = TextBlock(block3Line(text)) + + def addDescriptionBlock(self, text: str): + self.descriptionList.append(TextBlock(block(text))) + + def addDescription(self, text: str): + self.descriptionList.append(TextBlock(slash(text))) + + def clearDescription(self, text: str): + self.descriptionList = [] + + def __str__(self): + kStr = "" + kStr += str(self.kernelDescriptor) + if self.descriptionTopic != None: + kStr += str(self.descriptionTopic) + for i in self.descriptionList: + kStr += str(i) + kStr += str(self.codeMeta) + return kStr + + def prettyPrint(self, indent=""): + ostream = "" + ostream += "%s%s "%(indent, type(self).__name__) + return ostream + +######################################## +# Signatures +######################################## + +class KernelBody(Item): + def __init__(self, name) -> None: + super().__init__(name) + + def addSignature(self, signature: SignatureBase): + self.signature = signature + + def addBody(self, body: Module): + self.body = body + + def setGprs(self, totalVgprs: int, totalAgprs: int, totalSgprs: int): + self.totalVgprs = totalVgprs + self.totalAgprs = totalAgprs + self.totalSgprs = totalSgprs + self.signature.setGprs(totalVgprs=totalVgprs, totalAgprs=totalAgprs, \ + totalSgprs=totalSgprs) + + def __str__(self) -> str: + kStr = str(TextBlock(block3Line("Begin Kernel"))) + kStr += str(self.signature) + kStr += str(self.body) + return kStr diff --git a/tensilelite/Tensile/TensileInstructions/Containers.py b/tensilelite/Tensile/TensileInstructions/Containers.py new file mode 100644 index 0000000000..2a481e21d9 --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/Containers.py @@ -0,0 +1,384 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Base import Item +from .Enums import SelectBit, UnusedBit + +from copy import deepcopy +from dataclasses import dataclass +from typing import List, Optional +import math + +class Container(Item): + def __init__(self) -> None: + super().__init__("Container") + +@dataclass +class DSModifiers(Container): + # Nums of addresses needed + na: int = 1 + # Used with DS instructions that expect a single address + offset: int = 0 + # Used with DS instructions that expect two addresses + offset0: int = 0 + offset1: int = 0 + gds: bool = False + + def __post_init__(self): + super().__init__() + + def toList(self) -> List[str]: + l = [str(self).lstrip()] + return l + + def __str__(self) -> str: + kStr = "" + if self.na == 1: + kStr += " offset:%u"%self.offset + elif self.na == 2: + kStr += " offset0:%u offset1:%u"%(self.offset0, self.offset1) + if self.gds: + kStr += " gds" + return kStr + +@dataclass +class FLATModifiers(Container): + offset12: int = 0 + glc: bool = False + slc: bool = False + lds: bool = False + + def __post_init__(self): + super().__init__() + + def toList(self) -> List[str]: + l = [str(self).lstrip()] + return l + + def __str__(self) -> str: + kStr = "" + if self.offset12 != 0: + kStr += " offset:%u"%self.offset12 + if self.glc: + kStr += " glc" + if self.slc: + kStr += " slc" + if self.lds: + kStr += " lds" + return kStr + +@dataclass +class MUBUFModifiers(Container): + offen: bool = False + offset12: int = 0 + glc: bool = False + slc: bool = False + lds: bool = False + + def __post_init__(self): + super().__init__() + + def toList(self) -> List[str]: + l = [str(self).lstrip()] + return l + + def __str__(self) -> str: + kStr = "" + if self.offen: + kStr += " offen offset:%u"%self.offset12 + if (self.glc or self.slc or self.lds): + kStr += "," + if self.glc: + kStr += " glc" + if self.slc: + kStr += " slc" + if self.lds: + kStr += " lds" + return kStr + +@dataclass +class SMEMModifiers(Container): + glc: bool = False + nv: bool = False + dlc: bool = False + offset: int = 0 # 20u 21s shaes the same + + def __post_init__(self): + super().__init__() + + def toList(self) -> List[str]: + l = [str(self).lstrip()] + return l + + def __str__(self) -> str: + kStr = "" + if self.glc: + kStr += " glc" + if self.nv: + kStr += " nv" + if self.dlc: + kStr += " dlc" + if self.offset != 0: + kStr += " offset:%d"%self.offset + return kStr + +@dataclass +class SDWAModifiers(Container): + dst_sel: Optional[SelectBit] = None + dst_unused: Optional[UnusedBit] = None + src0_sel: Optional[SelectBit] = None + src1_sel: Optional[SelectBit] = None + + def __post_init__(self): + super().__init__() + + def toList(self) -> List[str]: + l = [] + if self.dst_sel != None: + l.append("dst_sel:" + self.dst_sel.name) + if self.dst_unused != None: + l.append("dst_unused:" + self.dst_unused.name) + if self.src0_sel != None: + l.append("src0_sel:" + self.src0_sel.name) + if self.src1_sel != None: + l.append("src1_sel:" + self.src1_sel.name) + return l + + def __str__(self) -> str: + kStr = "" + if self.dst_sel != None: + kStr += " dst_sel:" + self.dst_sel.name + if self.dst_unused != None: + kStr += " dst_unused:" + self.dst_unused.name + if self.src0_sel != None: + kStr += " src0_sel:" + self.src0_sel.name + if self.src1_sel != None: + kStr += " src1_sel:" + self.src1_sel.name + return kStr + +@dataclass +class VOP3PModifiers(Container): + op_sel: Optional[List[int]] = None + op_sel_hi: Optional[List[int]] = None + + def __post_init__(self): + super().__init__() + + def toList(self) -> List[str]: + l = [] + if self.op_sel != None: + l.append("op_sel:" + str(self.op_sel).replace(" ", "")) + if self.op_sel_hi != None: + l.append("op_sel_hi:" + str(self.op_sel_hi).replace(" ", "")) + return l + + def __str__(self) -> str: + kStr = "" + if self.op_sel != None: + kStr += " op_sel:" + str(self.op_sel).replace(" ", "") + if self.op_sel_hi != None: + kStr += " op_sel_hi:" + str(self.op_sel_hi).replace(" ", "") + return kStr + +class RegName(Container): + def __init__(self, name, *args): + super().__init__() + self.name = name + self.offsets = [] + for offset in args: + self.offsets.append(offset) + + def getTotalOffsets(self): + total = 0 + if self.offsets: + for i in self.offsets: + total += i + return total + + def __key(self) -> tuple: + return (self.name, str(self.offsets)) + + def __hash__(self) -> int: + return hash(self.__key()) + + def __eq__(self, __o: object) -> bool: + if not isinstance(__o, RegName): + return False + return (self.name == __o.name) and (self.offsets == __o.offsets) + + def __str__(self) -> str: + ss = self.name + if self.offsets: + for i in self.offsets: + ss += "+%u"%i + return ss + +class RegisterContainer(Container): + def __init__(self, regType, regName, regIdx, regNum) -> None: + super().__init__() + self.regType = regType + self.regIdx = regIdx + self.regNum = int(math.ceil(regNum)) + self.regName = regName + + self.isInlineAsm = False + self.isMinus = False + + def setInlineAsm(self, setting): + self.isInlineAsm = setting + + def setMinus(self, isMinus): + self.isMinus = isMinus + + def getMinus(self): + c = deepcopy(self) + c.setMinus(True) + return c + + def replaceRegName(self, srcName, dst): + if self.regName: + if isinstance(dst, int): + if self.regName.name == srcName: # Exact match + self.regIdx = dst + self.regName.offset + self.regName = None + else: + self.regName.name = self.regName.name.replace(srcName, str(dst)) + elif isinstance(dst, str): + self.regName.name = self.regName.name.replace(srcName, dst) + else: + assert("Dst type unknown %s" % str(type(dst)) and 0) + + # This get the name without offsets + def getRegNameWithType(self): + assert(self.regName) + return "%sgpr%s" % (self.regType, str(self.regName.name)) + + # This get the name with offsets + def getCompleteRegNameWithType(self): + assert(self.regName) + return "%sgpr%s" % (self.regType, str(self.regName)) + + def splitRegContainer(self): + if self.regName: + regName = deepcopy(self.regName) + regName2 = deepcopy(self.regName) + regName2.offsets.append(1) + regIdx2 = None + else: + regName = None + regName2 = None + regIdx2 = self.regIdx + 1 + r1 = RegisterContainer(self.regType, regName, self.regIdx, 1) + r2 = RegisterContainer(self.regType, regName2, regIdx2, 1) + return r1, r2 + + def __eq__(self, o) -> bool: + if not isinstance(o, RegisterContainer): + return False + # FIXME: should compare only with regIdx + isSame = (self.regName == o.regName) if (self.regIdx == None) else (self.regIdx == o.regIdx) + return (self.regType == o.regType) and isSame and (self.regNum == o.regNum) + + def __key(self) -> tuple: + return (self.regType, self.regIdx, self.regNum, self.regName) + + def __hash__(self) -> int: + return hash(self.__key()) + + def __str__(self) -> str: + minusStr = "-" if self.isMinus else "" + if self.isInlineAsm: + assert(self.regName == None) + return "%s%%%d" % (minusStr, self.regIdx) + + if self.regName: + if self.regNum == 1: + return "%s%s[%sgpr%s]"%(minusStr, self.regType, self.regType, str(self.regName)) + else: + return "%s%s[%sgpr%s:%sgpr%s+%u]"%(minusStr, self.regType, self.regType, str(self.regName), \ + self.regType, str(self.regName), self.regNum-1) + else: + if self.regNum == 1: + return "%s%s%u" % (minusStr, self.regType, self.regIdx) + else: + return "%s%s[%u:%u]" % (minusStr, self.regType, self.regIdx, self.regIdx+self.regNum-1) + +class HolderContainer(RegisterContainer): + def __init__(self, regType, holderName, holderIdx, regNum) -> None: + super().__init__(regType, None, None, regNum) + if holderIdx != None: + assert(holderName == None) + self.holderIdx = holderIdx + self.holderType = 0 + else: + assert(holderIdx == None) + self.holderName = holderName + self.holderType = 1 + + def setRegNum(self, num): + if self.holderType == 0: + self.regIdx = self.holderIdx + num + elif self.holderType == 1: + self.regName = deepcopy(self.holderName) + self.regName.offsets.insert(0, num) + + def getCopiedRC(self): + if self.holderType == 0: + assert(self.regIdx != None) + elif self.holderType == 1: + assert(self.regName != None) + return RegisterContainer(self.regType, self.regName, self.regIdx, self.regNum) + + def splitRegContainer(self): + if self.holderName: + holderName = deepcopy(self.holderName) + holderName2 = deepcopy(self.holderName) + holderName2.offsets.append(1) + holderIdx2 = None + else: + holderName = None + holderName2 = None + holderIdx2 = self.holderIdx + 1 + r1 = HolderContainer(self.regType, holderName, self.holderIdx, 1) + r2 = HolderContainer(self.regType, holderName2, holderIdx2, 1) + return r1, r2 + +class EXEC(Container): + def __init__(self, setHi=False) -> None: + super().__init__() + self.setHi = setHi + + def __str__(self) -> str: + if self.kernel.wavefrontSize == 64: + return "exec" + else: + return "exec_lo" + +class VCC(Container): + def __init__(self, setHi=False) -> None: + super().__init__() + self.setHi = setHi + + def __str__(self) -> str: + if self.kernel.wavefrontSize == 64: + return "vcc" + else: + return "vcc_hi" if self.setHi else "vcc_lo" diff --git a/tensilelite/Tensile/TensileInstructions/DataType.py b/tensilelite/Tensile/TensileInstructions/DataType.py new file mode 100644 index 0000000000..fcb5cef9b7 --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/DataType.py @@ -0,0 +1,327 @@ +################################################################################ +# Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +import functools + +@functools.total_ordering +class DataType: + """ + Data Type (new) + Uses a list of dictionaries to organize the DataType and Properties for the kernels + Changed older properties list of lists to list of dictionaries + The inner keys (char, reg, etc) correspond with the data type properties values + Lookup table is used to store row numbers of a specific property + """ + + properties = [ + { + 'char': 'S', + 'name': 'single', + 'nameAbbrev': 'f32', + 'miOutTypeNameAbbrev': 'f32', + 'enum': 'Float', + 'reg': 1, + 'ocl': 'float', + 'hip': 'float', + 'libType': 'float', + 'libEnum': 'tensileDataTypeFloat', + 'isIntegral': False, + 'isComplex': False, + 'packing': 1 + }, + { + 'char': 'D', + 'name': 'double', + 'nameAbbrev': 'f64', + 'miOutTypeNameAbbrev': 'f64', + 'enum': 'Double', + 'reg': 2, + 'ocl': 'double', + 'hip': 'double', + 'libType': 'double', + 'libEnum': 'tensileDataTypeDouble', + 'isIntegral': False, + 'isComplex': False, + 'packing': 1 + }, + { + 'char': 'C', + 'name': 'complexSingle', + 'nameAbbrev': 'f32c', + 'miOutTypeNameAbbrev': 'f32', + 'enum': 'ComplexFloat', + 'reg': 2, + 'ocl': 'float2', + 'hip': 'TensileComplexFloat', + 'libType': 'TensileComplexFloat', + 'libEnum': 'tensileDataTypeComplexFloat', + 'isIntegral': False, + 'isComplex': True, + 'packing': 1 + }, + { + 'char': 'Z', + 'name': 'complexDouble', + 'nameAbbrev': 'f64c', + 'miOutTypeNameAbbrev': 'f64', + 'enum': 'ComplexDouble', + 'reg': 4, + 'ocl': 'double2', + 'hip': 'TensileComplexDouble', + 'libType': 'TensileComplexDouble', + 'libEnum': 'tensileDataTypeComplexDouble', + 'isIntegral': False, + 'isComplex': True, + 'packing': 1 + }, + { + 'char': 'H', + 'name': 'half', + 'nameAbbrev': 'f16', + 'miOutTypeNameAbbrev': 'f32', + 'enum': 'Half', + 'reg': 0.5, + 'ocl': 'ERROR', + 'hip': 'tensile_half', + 'libType': 'TensileHalf', + 'libEnum': 'tensileDataTypeHalf', + 'isIntegral': False, + 'isComplex': False, + 'packing': 1 + }, + { + 'char': '4xi8', + 'name': 'int8x4', + 'nameAbbrev': 'i8', + 'miOutTypeNameAbbrev': 'i32', + 'enum': 'Int8x4', + 'reg': 1, + 'ocl': 'ERROR', + 'hip': 'uint32_t', + 'libType': 'TensileInt8x4', + 'libEnum': 'tensileDataTypeInt8x4', + 'isIntegral': True, + 'isComplex': False, + 'packing': 4 + }, + { + 'char': 'I', + 'name': 'int32', + 'nameAbbrev': 'i32', + 'miOutTypeNameAbbrev': 'NONE', # not supported for MI + 'enum': 'Int32', + 'reg': 1, + 'ocl': 'ERROR', + 'hip': 'int32_t', + 'libType': 'TensileInt32', + 'libEnum': 'tensileDataTypeInt32', + 'isIntegral': True, + 'isComplex': False, + 'packing': 1 + }, + { + 'char': 'I64', + 'name': 'int64', + 'nameAbbrev': 'i64', + 'miOutTypeNameAbbrev': 'NONE', # not supported for MI + 'enum': 'Int64', + 'reg': 1, + 'ocl': 'ERROR', + 'hip': 'int64_t', + 'libType': 'TensileInt64', + 'libEnum': 'tensileDataTypeInt64', + 'isIntegral': True, + 'isComplex': False, + 'packing': 1 + }, + { + 'char': 'B', + 'name': 'bfloat16', + 'nameAbbrev': 'bf16', + 'miOutTypeNameAbbrev': 'f32', + 'enum': 'BFloat16', + 'reg': 0.5, + 'ocl': 'ERROR', + 'hip': 'tensile_bfloat16', + 'libType': 'tensile_bfloat16', + 'libEnum': 'tensileDataTypeBFloat16', + 'isIntegral': False, + 'isComplex': False, + 'packing': 1 + }, + { + 'char': 'I8', + 'name': 'int8', + 'nameAbbrev': 'i8', + 'miOutTypeNameAbbrev': 'i32', + 'enum': 'Int8', # mapping to new client c++ enum + 'reg': 0.25, + 'ocl': 'ERROR', + 'hip': 'int8_t', + 'libType': 'TensileInt8', # old client + 'libEnum': 'tensileDataTypeInt8', # old client + 'isIntegral': True, + 'isComplex': False, + 'packing': 1 + }, + ] + lookup = {} + + def __init__(self, value): + if isinstance(value, int): + self.value = value + elif isinstance(value, str): + self.value = DataType.lookup[value.lower()] + elif isinstance(value, DataType): + self.value = value.value + else: + raise RuntimeError("initializing DataType to {0} {1}".format(str(type(value)), str(value))) + + self.properties = DataType.properties[self.value] + + def toChar(self): + return self.properties['char'] + def toName(self): + return self.properties['name'] + def toNameAbbrev(self): + return self.properties['nameAbbrev'] + def toEnum(self): + return self.properties['enum'] + def toOpenCL(self): + return self.properties['ocl'] + def toHIP(self): + return self.properties['hip'] + def toDevice(self, language): + if language == "OCL": + return self.toOpenCL() + else: + return self.toHIP() + def toCpp(self): + return self.properties['libType'] + def getLibString(self): + return self.properties['libEnum'] + + ######################################## + def zeroString(self, language, vectorWidth): + """ + Returns a string containing the data output format, depending on programming language + and in the case of complex numbers, the vector width. + """ + zeroString = "(" + zeroString += self.toDevice(language) + if vectorWidth > 1: + zeroString += str(vectorWidth) + zeroString += ")(" + + """ + if self.value == self.half: + single = "0" + vectorWidth = 1 + elif self.value == self.single: + single = "0.f" + elif self.value == self.double: + single = "0.0" + elif self.value == self.complexSingle: + single = "0.f, 0.f" + elif self.value == self.complexDouble: + single = "0.0, 0.0" + """ + zeroString += "0" + zeroString += ")" + return zeroString + + def isReal(self): + return not self.isComplex() + def isComplex(self): + return self.properties['isComplex'] + def isDoubleComplex(self): + return self.value == DataType.complexDouble + def isSingleComplex(self): + return self.value == DataType.complexSingle + def isDouble(self): + return self.value == DataType.double + def isSingle(self): + return self.value == DataType.single + def isHalf(self): + return self.value == DataType.half + def isInt32(self): + return self.value == DataType.int32 + def isInt64(self): + return self.value == DataType.int64 + def isInt8x4(self): + return self.value == DataType.int8x4 + def isInt8(self): + return self.value == DataType.int8 + def isBFloat16(self): + return self.value == DataType.bfloat16 + def isNone(self): + return self.value == None + + def numRegisters(self): + return self.properties['reg'] + def numBytes(self): + return int(self.numRegisters() * 4) + def MIOutputTypeNameAbbrev(self): + return self.properties['miOutTypeNameAbbrev'] + def flopsPerMac(self): + return 2 if self.isReal() else 8 + + def state(self): return self.toEnum() + + def __str__(self): + return self.toChar() + def __repr__(self): + return self.__str__() + + def getAttributes(self): + return (self.value,) + + def __hash__(self): + return hash(self.getAttributes()) + + def __eq__(self, other): + if not isinstance(other, DataType): + return NotImplemented + + return self.getAttributes() == other.getAttributes() + + def __lt__(self, other): + if not isinstance(other, DataType): + return NotImplemented + + return self.getAttributes() < other.getAttributes() + + # Other operands are provided by functools.total_ordering. + +def _populateLookupTable(properties,lookup): + """ + Populates Lookup Table with the corresponding row number for each DataType. The row number + is assigned to self.value when a DataType object is called + """ + for i,e in enumerate(properties): + setattr(DataType, e['name'], i) + for k in ['name','char','enum','libEnum']: + lookupKey = e[k].lower() + if lookupKey in lookup and lookup[lookupKey] != i: + raise RuntimeError("Duplicate key {1} in property '{0}'".format(k,lookupKey)) + lookup[lookupKey] = i + +_populateLookupTable(DataType.properties,DataType.lookup) diff --git a/tensilelite/Tensile/TensileInstructions/Enums.py b/tensilelite/Tensile/TensileInstructions/Enums.py new file mode 100644 index 0000000000..023abe3373 --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/Enums.py @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from enum import Enum + +class SignatureValueKind(Enum): + SIG_VALUE = 1 + SIG_GLOBALBUFFER = 2 + +class InstType(Enum): + INST_F8 = 1 + INST_F16 = 2 + INST_F32 = 3 + INST_F64 = 4 + INST_I8 = 5 + INST_I16 = 6 + INST_I32 = 7 + INST_U8 = 8 + INST_U16 = 9 + INST_U32 = 10 + INST_U64 = 11 + INST_LO_I32 = 12 + INST_HI_I32 = 13 + INST_LO_U32 = 14 + INST_HI_U32 = 15 + INST_BF16 = 16 + INST_B8 = 17 + INST_B16 = 18 + INST_B32 = 19 + INST_B64 = 20 + INST_B128 = 21 + INST_B256 = 22 + INST_B512 = 23 + INST_B8_HI_D16 = 24 + INST_D16_U8 = 25 + INST_D16_HI_U8 = 26 + INST_D16_U16 = 27 + INST_D16_HI_U16 = 28 + INST_D16_B8 = 29 + INST_D16_HI_B8 = 30 + INST_D16_B16 = 31 + INST_D16_HI_B16 = 32 + INST_CVT = 33 + INST_MACRO = 34 + INST_NOTYPE = 35 + +class SelectBit(Enum): + DWORD = 1 + BYTE_0 = 2 + BYTE_1 = 3 + BYTE_2 = 4 + BYTE_3 = 5 + WORD_0 = 6 + WORD_1 = 7 + +class UnusedBit(Enum): + UNUSED_PAD = 1 + UNUSED_SEXT = 2 + UNUSED_PRESERVE = 3 + +class CvtType(Enum): + CVT_F16_to_F32 = 1 + CVT_F32_to_F16 = 2 + CVT_U32_to_F32 = 3 + CVT_F32_to_U32 = 4 diff --git a/tensilelite/Tensile/TensileInstructions/ExtInstructions.py b/tensilelite/Tensile/TensileInstructions/ExtInstructions.py new file mode 100644 index 0000000000..8547222d92 --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/ExtInstructions.py @@ -0,0 +1,619 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Code import Module, Label, TextBlock +from .Containers import RegisterContainer, VCC +from .DataType import DataType +from .RegisterPool import RegisterPoolResource +from .Utils import vgpr, sgpr, log2 +from .Instructions import * + +from enum import Enum +from typing import Union +import sys + +################################################################################ +################################################################################ +### +### Function ExtInstructions +### +################################################################################ +################################################################################ + +######################################## +# Branch +######################################## +############################################################################## +# longBranch - 32 bit offset +# s_branch class instructions take a label operand which is truncated to 16 bit +# If the target label address offset is greater than 16 bits, then +# we must use a longer 32 bit version. +# Use when erroring out "invalid operand due to label > SIMM16" +############################################################################## +def SLongBranch(label: Label, tmpSgprRes: RegisterPoolResource, postiveLabelStr: str, comment=""): + labelName = label.getLabelName() + module = Module("SLongBranch %s"%labelName) + if comment: + module.addComment(comment) + + assert tmpSgprRes.size >= 3 + tmpSgpr = tmpSgprRes.idx + positiveLabel = Label(postiveLabelStr, "") + module.add(SGetPCB64(dst=sgpr(tmpSgpr,2), comment="addr of next instr")) + module.add(SAddI32(dst=sgpr(tmpSgpr+2), src0=labelName, src1=hex(4), comment="target branch offset")) + module.add(SCmpGeI32(src0=sgpr(tmpSgpr+2), src1=hex(0), comment="check positive or negative")) + module.add(SCBranchSCC1(labelName=positiveLabel.getLabelName(), comment="jump when positive")) + + # negative offset + module.add(SAbsI32(dst=sgpr(tmpSgpr+2), src=sgpr(tmpSgpr+2), comment="abs offset")) + module.add(SSubU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=sgpr(tmpSgpr+2), comment="sub target branch offset")) + module.add(SSubBU32(dst=sgpr(tmpSgpr+1), src0=sgpr(tmpSgpr+1), src1=0, comment="sub high and carry")) + module.add(SSetPCB64(src=sgpr(tmpSgpr,2), comment="branch to %s"%labelName)) + + # positive offset + module.add(positiveLabel) + module.add(SAddU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=sgpr(tmpSgpr+2), comment="add target branch offset")) + module.add(SAddCU32(dst=sgpr(tmpSgpr+1), src0=sgpr(tmpSgpr+1), src1=0, comment="add high and carry")) + module.add(SSetPCB64(src=sgpr(tmpSgpr,2), comment="branch to %s"%labelName)) + return module + +############################################################################## +# longBranchPositive - 32 bit offset (positive offset only) +# s_branch class instructions take a label operand which is truncated to 16 bit +# If the target label address offset is greater than 16 bits, then +# we must use a longer 32 bit version. +# Use when erroring out "invalid operand due to label > SIMM16" +############################################################################## +def SGetPositivePCOffset(sgprIdx, label: Label, tmpSgprRes: RegisterPoolResource): + labelName = label.getLabelName() + module = Module("SGetPositivePCOffset %s"%labelName) + assert tmpSgprRes.size >= 1 + tmpSgpr = tmpSgprRes.idx + module.add(SGetPCB64(dst=sgpr(sgprIdx,2), comment="addr of next instr")) + module.add(SAddI32(dst=sgpr(tmpSgpr), src0=labelName, src1=hex(4), comment="target branch offset")) + + # positive offset + module.add(SAddU32(dst=sgpr(sgprIdx), src0=sgpr(sgprIdx), src1=sgpr(tmpSgpr), comment="add target branch offset")) + module.add(SAddCU32(dst=sgpr(sgprIdx+1), src0=sgpr(sgprIdx+1), src1=0, comment="add high and carry")) + return module + +def SLongBranchPositive(label: Label, tmpSgprRes: RegisterPoolResource, comment=""): + labelName = label.getLabelName() + module = Module("SLongBranchPositive %s"%labelName) + if comment: + module.addComment(comment) + + assert tmpSgprRes.size >= 3 + tmpSgpr = tmpSgprRes.idx + module.addModuleAsFlatItems(SGetPositivePCOffset(tmpSgpr, label, RegisterPoolResource(tmpSgpr+2, 1))) + module.add(SSetPCB64(src=sgpr(tmpSgpr,2), comment="branch to %s"%labelName)) + return module + +############################################################################## +# longBranchNegative - 32 bit offset (negative offset only) +# s_branch class instructions take a label operand which is truncated to 16 bit +# If the target label address offset is greater than 16 bits, then +# we must use a longer 32 bit version. +# Use when erroring out "invalid operand due to label > SIMM16" +##############################################################################0 +def SLongBranchNegative(label: Label, tmpSgprRes: RegisterPoolResource, comment=""): + labelName = label.getLabelName() + module = Module("SLongBranchNegative %s"%labelName) + if comment: + module.addComment(comment) + + assert tmpSgprRes.size >= 3 + tmpSgpr = tmpSgprRes.idx + module.add(SGetPCB64(dst=sgpr(tmpSgpr,2), comment="addr of next instr")) + module.add(SAddI32(dst=sgpr(tmpSgpr+2), src0=labelName, src1=hex(4), comment="target branch offset")) + + # negative offset + module.add(SAbsI32(dst=sgpr(tmpSgpr+2), src=sgpr(tmpSgpr+2), comment="abs offset")) + module.add(SSubU32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=sgpr(tmpSgpr+2), comment="sub target branch offset")) + module.add(SSubBU32(dst=sgpr(tmpSgpr+1), src0=sgpr(tmpSgpr+1), src1=0, comment="sub high and carry")) + module.add(SSetPCB64(src=sgpr(tmpSgpr,2), comment="branch to %s"%labelName)) + return module + +############################################################################## +# longBranchScc0 - 32 bit offset +# Conditional branch to label when SCC == 0 +# Use when erroring out "invalid operand due to label > SIMM16" +############################################################################## +def SCLongBranchScc0(label: Label, tmpSgprRes: RegisterPoolResource, \ + noBranchLabelStr: str, postiveLabelStr: str, posNeg: int = 0): + module = Module("SCLongBranchScc0 %s"%label.getLabelName()) + noBranchLabel = Label(noBranchLabelStr, "") + module.add(SCBranchSCC1(labelName=noBranchLabel.getLabelName(), \ + comment="Only branch on scc0")) + if posNeg > 0: + module.add(SLongBranchPositive(label, tmpSgprRes)) + elif posNeg < 0: + module.add(SLongBranchNegative(label, tmpSgprRes)) + else: + module.add(SLongBranch(label, tmpSgprRes, postiveLabelStr)) + module.add(noBranchLabel) + return module + +############################################################################## +# longBranchScc1 - 32 bit offset +# Conditional branch to label when SCC == 1 +# Use when erroring out "invalid operand due to label > SIMM16" +############################################################################## +def SCLongBranchScc1(label: Label, tmpSgprRes: RegisterPoolResource, \ + noBranchLabelStr: str, postiveLabelStr: str, posNeg: int = 0): + module = Module("SCLongBranchScc1 %s"%label.getLabelName()) + noBranchLabel = Label(noBranchLabelStr, "") + module.add(SCBranchSCC0(labelName=noBranchLabel.getLabelName(), \ + comment="Only branch on scc1")) + if posNeg > 0: + module.add(SLongBranchPositive(label, tmpSgprRes)) + elif posNeg < 0: + module.add(SLongBranchNegative(label, tmpSgprRes)) + else: + module.add(SLongBranch(label, tmpSgprRes, postiveLabelStr)) + module.add(noBranchLabel) + return module + +######################################## +# If else +######################################## + +def SBranchIfZero(sgprName, computeDataType: DataType, tmpSgpr, laneSC, label, waveFrontSize): + module = Module("SBranchIfZero") + sgprStr = "s[{}]".format(sgprName) + if computeDataType.isDoubleComplex(): + module.add(VCmpEQF64(dst=sgpr(tmpSgpr, laneSC), src0=sgpr(sgprName, 2), src1=0.0, comment="%s.real == 0.0 ?" % sgprStr)) + sgprVar = "%s+2" % sgprName if isinstance(sgprName, str) else sgprName + 2 + module.add(VCmpEQF64(dst=VCC(), src0=sgpr(sgprVar, 2), src1=0.0, comment="%s.imag == 0.0 ?" % sgprStr)) + if waveFrontSize == 32: + module.add(SAndB32(dst=sgpr(tmpSgpr, laneSC), src0=VCC(), src1=sgpr(tmpSgpr, laneSC), comment="%s == 0 ?" % sgprStr)) + module.add(SCmpEQU32(src0=sgpr(tmpSgpr, laneSC), src1=hex(0), comment="branch if %s == 0" % sgprStr)) + else: + module.add(SAndB64(dst=sgpr(tmpSgpr, laneSC), src0=VCC(), src1=sgpr(tmpSgpr, laneSC), comment="%s == 0 ?" % sgprStr)) + module.add(SCmpEQU64(src0=sgpr(tmpSgpr, laneSC), src1=hex(0), comment="branch if %s == 0" % sgprStr)) + module.add(SCBranchSCC0(labelName=label.getLabelName(), comment="branch if %s == 0" % sgprStr)) + elif computeDataType.isDouble(): + module.add(VCmpEQF64(dst=VCC(), src0=sgpr(sgprName, 2), src1=0.0, comment="%s == 0.0 ?" % sgprStr)) + module.add(SCBranchVCCNZ(labelName=label.getLabelName(), comment="branch if %s == 0" % sgprStr)) + elif computeDataType.isSingleComplex(): + module.add(VCmpEQF32(dst=sgpr(tmpSgpr, laneSC), src0=sgpr(sgprName), src1=0.0, comment="%s.real == 0.0f ?" % sgprStr)) + sgprVar = "%s+1" % sgprName if isinstance(sgprName, str) else sgprName + 1 + module.add(VCmpEQF32(dst=VCC(), src0=sgpr(sgprVar), src1=0.0, comment="%s.imag == 0.0f ?" % sgprStr)) + if waveFrontSize == 32: + module.add(SAndB32(dst=sgpr(tmpSgpr, laneSC), src0=VCC(), src1=sgpr(tmpSgpr, laneSC), comment="%s == 0 ?" % sgprStr)) + module.add(SCmpEQU32(src0=sgpr(tmpSgpr, laneSC), src1=hex(0), comment="branch if %s == 0" % sgprStr)) + else: + module.add(SAndB64(dst=sgpr(tmpSgpr, laneSC), src0=VCC(), src1=sgpr(tmpSgpr, laneSC), comment="%s == 0 ?" % sgprStr)) + module.add(SCmpEQU64(src0=sgpr(tmpSgpr, laneSC), src1=hex(0), comment="branch if %s == 0" % sgprStr)) + module.add(SCBranchSCC0(labelName=label.getLabelName(), comment="branch if %s == 0" % sgprStr)) + elif computeDataType.isSingle() or computeDataType.isHalf() or computeDataType.isBFloat16(): + module.add(VCmpEQF32(dst=VCC(), src0=sgpr(sgprName), src1=0.0, comment="%s == 0.0f ?" % sgprStr)) + module.add(SCBranchVCCNZ(labelName=label.getLabelName(), comment="branch if %s == 0" % sgprStr)) + elif computeDataType.isInt32(): # int32 + module.add(SCmpEQU32(src0=sgpr(sgprName), src1=0, comment="%s == 0 ?" % sgprStr)) + module.add(SCBranchSCC1(labelName=label.getLabelName(), comment="branch if %s == 0" % sgprStr)) + else: + print("Unsupported compute data type: %s" % str(computeDataType)) + sys.stdout.flush() + sys.exit(-1) + return module + +def SBranchIfNotZero(sgprName, computeDataType: DataType, label): + module = Module("SBranchIfNotZero") + sgprStr = "s[{}]".format(sgprName) + if computeDataType.isDoubleComplex(): + module.add(VCmpEQF64(dst=VCC(), src0=sgpr(sgprName, 2), src1=0.0, comment="%s.real == 0.0 ?" % sgprStr)) + module.add(SCBranchVCCZ(labelName=label.getLabelName(), comment="branch if %s.real != 0" % sgprStr)) + sgprVar = "%s+2" % sgprName if isinstance(sgprName, str) else sgprName + 2 + module.add(VCmpEQF64(dst=VCC(), src0=sgpr(sgprVar, 2), src1=0.0, comment="%s.imag == 0.0 ?" % sgprStr)) + module.add(SCBranchVCCZ(labelName=label.getLabelName(), comment="branch if %s.imag != 0" % sgprStr)) + elif computeDataType.isDouble(): + module.add(VCmpEQF64(dst=VCC(), src0=sgpr(sgprName, 2), src1=0.0, comment="%s == 0.0 ?" % sgprStr)) + module.add(SCBranchVCCZ(labelName=label.getLabelName(), comment="branch if %s != 0" % sgprStr)) + elif computeDataType.isSingleComplex(): + module.add(VCmpEQF32(dst=VCC(), src0=sgpr(sgprName), src1=0.0, comment="%s.real == 0.0f ?" % sgprStr)) + module.add(SCBranchVCCZ(labelName=label.getLabelName(), comment="branch if %s.real != 0" % sgprStr)) + sgprVar = "%s+1" % sgprName if isinstance(sgprName, str) else sgprName + 1 + module.add(VCmpEQF32(dst=VCC(), src0=sgpr(sgprVar), src1=0.0, comment="%s.imag == 0.0f ?" % sgprStr)) + module.add(SCBranchVCCZ(labelName=label.getLabelName(), comment="branch if %s.imag != 0" % sgprStr)) + elif computeDataType.isSingle() or computeDataType.isHalf() or computeDataType.isBFloat16(): + module.add(VCmpEQF32(dst=VCC(), src0=sgpr(sgprName), src1=0.0, comment="%s == 0.0f ?" % sgprStr)) + module.add(SCBranchVCCZ(labelName=label.getLabelName(), comment="branch if %s != 0" % sgprStr)) + elif computeDataType.isInt64(): + module.add(SCmpEQU64(src0=sgpr(sgprName, 2), src1=0, comment="%s == 0 ?" % sgprStr)) + module.add(SCBranchSCC0(labelName=label.getLabelName(), comment="branch if %s != 0" % sgprStr)) + else: + module.add(SCmpEQU32(src0=sgpr(sgprName), src1=0, comment="%s == 0 ?" % sgprStr)) + module.add(SCBranchSCC0(labelName=label.getLabelName(), comment="branch if %s != 0" % sgprStr)) + return module + +# Perform 32-bit scalar mul and save 64-bit result in two SGPR +# src0 and src1 are 32-bit ints in scalar sgpr or small int constants (<64?)) +# signed indicates if input and output data is signed +# return returns in dst0:dest (lower 32-bit in dst0, high 64-bit in dst1)) +# Requires 2 tmp vgprs +def SMulInt64to32(hasSMulHi, dst0, dst1, src0, src1, signed, vtmp0, comment): + module = Module("SMulInt64to32") + sign = "i" if signed else "u" + assert(dst1 != src0) # no worky since dst1 overwritten by first mul operations + assert(dst1 != src1) # no worky since dst1 overwritten by first mul operations + # the else path below has less restrictions but prefer consistency + if hasSMulHi: + SInst = SMulHII32 if signed else SMulHIU32 + module.add(SInst(dst=dst1, src0=src0, src1=src1, comment=comment)) + module.add(SMulI32(dst=dst0, src0=src0, src1=src1, comment=comment)) + else: + if (not isinstance(src1, RegisterContainer)) or (src1.regType != "s"): + # Swap operands, need a scalar sgpr in src1 (not a constant) + t = src0 + src0 = src1 + src1 = t + vtmp1 = vtmp0+1 + module.add(VMovB32(dst=vgpr(vtmp0), src=src0, comment=comment)) + VInst = VMulHII32 if signed else VMulHIU32 + module.add(VInst(dst=vgpr(vtmp1), src0=vgpr(vtmp0), src1=src1, comment=comment)) + module.add(VReadfirstlaneB32(dst=dst1, src=vgpr(vtmp1), comment=comment)) + module.add(VMulLOU32(dst=vgpr(vtmp1), src0=vgpr(vtmp0), src1=src1, comment=comment)) + module.add(VReadfirstlaneB32(dst=dst0, src=vgpr(vtmp1), comment=comment)) + return module + +######################################## +# Saturate Cast Integer +######################################## + +class SaturateCastType(Enum): + NORMAL = 1 + DO_NOTHING = 2 + UPPER = 3 + LOWER = 4 + +def VSaturateCastInt(sumIdxV, tmpVgpr, tmpSgpr, lowerBound, upperBound, type=SaturateCastType.NORMAL, initGpr=True): + # SaturateCastType = 0, normal case + # SaturateCastType = 1, do nothing + # SaturateCastType = 2, upperbound only + # SaturateCastType = 3, lowerbound only + initGprStr = "with init gpr" if initGpr else "without init gpr" + module = Module("SaturateCastInt %s"%(initGprStr)) + if type == SaturateCastType.NORMAL: + tmpLowerBound = tmpSgpr + tmpUpperBound = tmpVgpr + if initGpr: + lowerBoundHex = hex(lowerBound) + upperBoundHex = hex(upperBound) + module.add(SMovkI32(dst=sgpr(tmpLowerBound), src=lowerBoundHex, comment="%d"%lowerBound )) + module.add(VMovB32(dst=vgpr(tmpUpperBound), src=upperBoundHex, comment="%d"%upperBound )) + module.add(VMed3I32(dst=vgpr("ValuC+%u"%(sumIdxV)), src0=vgpr("ValuC+%u"%(sumIdxV)), src1=sgpr(tmpLowerBound), src2=vgpr(tmpUpperBound), comment="x= min(%d, max(%d, x))"%(upperBound, lowerBound))) + elif type == SaturateCastType.DO_NOTHING: + pass + elif type == SaturateCastType.UPPER: + module.add(VMinI32(dst=vgpr("ValuC+%u"%(sumIdxV)), src0=upperBound, src1=vgpr("ValuC+%u"%(sumIdxV)), comment="x = min(%d, x)"%upperBound)) + elif type == SaturateCastType.LOWER: + module.add(VMaxI32(dst=vgpr("ValuC+%u"%(sumIdxV)), src0=lowerBound, src1=vgpr("ValuC+%u"%(sumIdxV)), comment="x = max(%d, x)"%lowerBound)) + return module + +######################################## +# Cvt +######################################## + +def VCvtBF16toFP32(dst, src, vgprMask, vi): + if (vi % 2) == 1: + return VAndB32(dst=vgpr(dst), src0=vgpr(src), src1=vgpr(vgprMask), comment="convert bf16 to fp32") # mask = hex(0xffff0000) + else: + return VLShiftLeftB32(dst=vgpr(dst), shiftHex=16, src=vgpr(src), comment="convert bf16 to fp32") + +######################################## +# init lds state +######################################## +def DSInit(tmpVgprRes: RegisterPoolResource, numThreads: int, \ + ldsNumElements: int, numBytesPerElement: int, initValue): + assert tmpVgprRes.size > 1 + tmp = tmpVgprRes.idx + tmpAddr = tmp + 1 + module = Module("initLds") + module.addComment1("init lds state") + module.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="")) + module.add(SBarrier(comment="init LDS")) + module.add(VMovB32(dst=vgpr(tmp), src=hex(initValue), comment="Init value")) + module.add(VLShiftLeftB32(dst=vgpr(tmpAddr), shiftHex=2, src=vgpr("Serial"), \ + comment="set per-thread address to init LDS")) + writesPerThread = ((ldsNumElements*numBytesPerElement-1)//numThreads//4) + 1 + for i in range(0, writesPerThread): + module.add(DSStoreB32(dstAddr=vgpr(tmpAddr), src=vgpr(tmp), + ds=DSModifiers(offset=(i*numThreads*4)), comment="init lds")) + module.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="wait for LDS init to complete")) + module.add(SBarrier(comment="init LDS exit")) + return module + +################################################################################ +################################################################################ +### +### Class ExtInstructions +### +################################################################################ +################################################################################ + +############################################################################## +# Load arguments +############################################################################## + +class ArgumentLoader: + def __init__(self) -> None: + self.kernArgOffset = 0 + + def resetOffset(self) -> None: + self.kernArgOffset = 0 + + def getOffset(self) -> int: + return self.kernArgOffset + + ############################################################################## + # getKernArg + # Write an argument to specified SGPR and move the kernArgOffset + # if writeSgpr==0, just move the kernArgOffset - this is used to skip + # unused parms + ############################################################################## + def loadSingleKernArg(self, dst: Union[int, str], srcAddr: Union[int, str], \ + writeSgpr=True) -> Union[Instruction, TextBlock]: + item = None + size = 1*4 + if writeSgpr: + item = SLoadB32(dst=sgpr(dst), \ + base=sgpr(srcAddr, 2), soffset=hex(self.kernArgOffset)) + else: + item = TextBlock("Move offset by %u\n" % size) + self.kernArgOffset += size + return item + + def loadAllKernArg(self, sgprStartIndex: int, srcAddr: Union[int, str], \ + numSgprToLoad: int) -> Module: + module = Module("LoadAllKernArg") + while numSgprToLoad > 0: + i = 16 # 16, 8, 4, 2, 1 + while i >= 1: + if numSgprToLoad >= i: + numSgprToLoad -= i + SLoadBX = { 512: SLoadB512, + 256: SLoadB256, + 128: SLoadB128, + 64: SLoadB64, + 32: SLoadB32 + }[i * 32] + module.add(SLoadBX(dst=sgpr(sgprStartIndex, i), base=sgpr(srcAddr, 2), soffset=hex(self.kernArgOffset))) + sgprStartIndex += i + self.kernArgOffset += i * 4 + break + i = i // 2 + return module + # currently align sgpr to kernel argument memory, and use s_load_bxxx to load argument as large as possible in one instruction + # however, in order to match sgpr to kernel argument memory, some unnecessarily sgpr will also be defined, and caused wasting of sgpr. + # TODO: more efficient way is to organize both sgpr and kernel argument memory in API + +############################################################################## +# Assert +############################################################################## + +def bomb(scratchVgpr, cookie=None): + """ + Cause a GPUVM fault. + Instruction after the bomb will write the cookie to SGPR0, so you can see the cookie in the + backtrace. Useful for locating which spot in code generated the bomb + vgprAddr controls which vgpr to overwrite with the null pointer address + """ + + module = Module("bomb") + vgprAddr = scratchVgpr + + if cookie != None: + if cookie < 0: + module.add(Label("bomb_neg%u" % abs(cookie), "")) + else: + module.add(Label("bomb_%u" % abs(cookie), "")) + module.add(VMovB32(dst=vgpr(vgprAddr+0), src=0)) + module.add(VMovB32(dst=vgpr(vgprAddr+1), src=0)) + module.add(FlatLoadB32(dst=vgpr(vgprAddr), vaddr=vgpr(vgprAddr,2), comment="bomb - force fault" )) + + # This move does not execute but appears in the instruction stream immediately following + # the faulting load: + if cookie != None: + module.add(SMovB32(dst=sgpr(0), src=cookie, comment="bomb cookie=%d(0x%x)"%(cookie,cookie&0xffffffff))) + + return module + +class Assert(): + def __init__(self, laneSGPRCount, wavefrontSize, enableAsserts): + self.printedAssertCnt = 0 + self.laneSGPRCount = laneSGPRCount + self.wavefrontSize = wavefrontSize + self.enableAsserts = enableAsserts + + ############################################################################## + # assertCommon : Common routine for all assert functions. + # On entry, we have already set the exec-mask so any enabled lanes should bomb + ############################################################################## + def assertCommon(self, vtmp, cookie=-1): + module = Module("assertCommon") + if self.enableAsserts: + self.printedAssertCnt += 1 + # Default cookie for asserts is negative of printed #asserts + # Can be used to roughly identify which assert in the code is firing + module.add(bomb(vtmp, cookie if cookie != -1 else -self.printedAssertCnt)) + return module + + ############################################################################## + # assertCmpCommon : Common routine for all assert comparison functions + ############################################################################## + def assertCmpCommon(self, inst, val0, val1, vtmp, cookie=-1): + assert issubclass(inst, VCmpXInstruction) + module = Module("assertCmpCommon") + if self.enableAsserts: + SOrSaveExecBX = SOrSaveExecB64 if self.wavefrontSize == 64 else SOrSaveExecB32 + module.add(SOrSaveExecBX(dst=sgpr("SaveExecMask",self.laneSGPRCount), src=0, \ + comment="assert: saved execmask")) + module.add(inst(dst=VCC(), src0=val0, src1=val1, comment="v_cmp")) # type: ignore + module.add(self.assertCommon(vtmp, cookie)) + module.add(SOrSaveExecBX(dst=VCC(), src=sgpr("SaveExecMask",self.laneSGPRCount), \ + comment="assert: restore execmask")) + return module + + ############################################################################## + # Handle different conditions for the asserts: + # These support uin32 compare, float could be added later + # Asserts currently modify vcc + ############################################################################## + def eq(self, val0, val1, vtmp, cookie=-1): + return self.assertCmpCommon(VCmpXNeU32, val0, val1, vtmp, cookie) + + def eq_u16(self, val0, val1, vtmp, cookie=-1): + return self.assertCmpCommon(VCmpXNeU16, val0, val1, vtmp, cookie) + + def ne(self, val0, val1, vtmp, cookie=-1): + return self.assertCmpCommon(VCmpXEqU32, val0, val1, vtmp, cookie) + + def lt_u32(self, val0, val1, vtmp, cookie=-1): + return self.assertCmpCommon(VCmpXGeU32, val0, val1, vtmp, cookie) + + def gt_u32(self, val0, val1, vtmp, cookie=-1): + return self.assertCmpCommon(VCmpXLeU32, val0, val1, vtmp, cookie) + + def le_u32(self, val0, val1, vtmp, cookie=-1): + return self.assertCmpCommon(VCmpXGtU32, val0, val1, vtmp, cookie) + + def ge_u32(self, val0, val1, vtmp, cookie=-1): + return self.assertCmpCommon(VCmpXLtU32, val0, val1, vtmp, cookie) + + def ge_i32(self, val0, val1, vtmp, cookie=-1): + return self.assertCmpCommon(VCmpXLtI32, val0, val1, vtmp, cookie) + + # can left shift w/o losing non-zero bits: + def no_shift_of(self, val0, shift, stmp, vtmp, cookie=-1): + module = Module("Assert no shift of") + # TODO - use BFE here: + module.add(SMovB32(dst=stmp, src=hex((shift-1) << (32-log2(shift))), comment="assert_no_shift_of - compute mask")) + module.add(SAndB32(dst=stmp, src0=stmp, src1=val0, comment="assert_no_shift_of")) + module.add(self.eq(stmp, 0, vtmp, cookie)) + return module + + # asserts if val0 is not an integer multiple of multiple2 + # multiple2 must be a constant and power of 2 + # for example assert_multiple(A, 8) will assert if A is not multiple of 8 + def multiple_b32(self, sval, multiple2, vtmp, cookie=-1): + module = Module("Assert multiple b32") + if self.enableAsserts: + + stmp = sgpr("SaveExecMask") # repurpose to get a tmp sgpr + SAndBX = SAndB64 if self.wavefrontSize else SAndB32 + module.add(SAndBX(dst=stmp, src0=sval, src1=multiple2-1, comment="mask" )) + module.add(SCmpEQU32(src0=stmp, src1=0, comment="if maskedBits==0 then SCC=1 == no fault" )) + SMovBX = SMovB64 if self.wavefrontSize else SMovB32 + module.add(SMovBX(dst=sgpr("SaveExecMask",self.laneSGPRCount), src=-1)) + SCMovBX= SCMovB64 if self.wavefrontSize else SCMovB32 + module.add(SCMovBX(dst=sgpr("SaveExecMask", self.laneSGPRCount), src=0, comment="Clear exec mask")) + + SAndSaveExecBX = SAndSaveExecB64 if self.wavefrontSize else SAndSaveExecB32 + module.add(SAndSaveExecBX(dst=sgpr("SaveExecMask",self.laneSGPRCount), src=sgpr("SaveExecMask",self.laneSGPRCount), \ + comment="assert: saved execmask")) + + module.add(self.assertCommon(vtmp, cookie)) + + SOrSaveExecBX = SOrSaveExecB64 if self.wavefrontSize else SOrSaveExecB32 + module.add(SOrSaveExecBX(dst=VCC(), src=sgpr("SaveExecMask",self.laneSGPRCount), \ + comment="assert: restore execmask")) + + return module + + # assert v0 + expectedScalarDiff == v1 + # Verify that each element in v1 is scalar offset from v0 + def assert_vector_diff(self, v0, v1, expectedScalarDiff, cmpvtmp, vtmp, cookie=-1): + module = Module("assert_vector_diff") + module.add(VAddCOU32(dst=vgpr(cmpvtmp), \ + dst1=VCC(), \ + src0=expectedScalarDiff, \ + src1=v0, \ + comment="assert_vector_diff add expectedScalarDiff")) + module.add(self.eq(vgpr(cmpvtmp), v1, vtmp, cookie)) + return module + +############################################################################## +# Dump (Store to debug buffer) +############################################################################## + +class Dump: + def __init__(self, sgprDebugKernelItems: Union[int, str], vgprAddressDbg: Union[int, str], \ + maxItem: int, enableDump) -> None: + self.sgprDebugKernelItems = sgprDebugKernelItems + self.vgprAddressDbg = vgprAddressDbg + self.maxItem = maxItem + self.enableDump = enableDump + + def dumpVgpr(self, vgprStore: Union[int, str], labelName: str) -> Module: + module = Module("dump vgpr[%s]"%str(vgprStore)) + if self.enableDump: + afterDump = -1 + if self.maxItem != -1: + afterDump = labelName + afterDump = Label(afterDump, "skip debug target") + module.add(SCmpLtU32(src0=sgpr(self.sgprDebugKernelItems), src1=16)) + module.add(SCBranchSCC0(labelName=afterDump.getLabelName(), \ + comment="skip if already wrote enough work-items" )) + module.add(SAddU32(dst=sgpr(self.sgprDebugKernelItems), \ + src0=sgpr(self.sgprDebugKernelItems), src1=hex(1), \ + comment="inc items written" )) + + module.add(FlatStoreB32(vaddr=vgpr(self.vgprAddressDbg, 2), src=vgprStore, comment="debug dump store")) + module.add(VAddCOU32(dst=vgpr(self.vgprAddressDbg), dst1=VCC(), src0=vgpr(self.vgprAddressDbg), src1=hex(4), comment="debug dump inc")) + + if self.maxItem != -1: + assert(isinstance(afterDump, Label)) # Dummy guard in case someone remove the if above + module.add(afterDump) + + return module + + def dumpLds(self, startU: int, numU: int, tmpVgprRes: RegisterPoolResource, bpeAB: int, \ + numThreads: int, labelName: str) -> Module: + module = Module("dump lds") + if self.enableDump: + assert tmpVgprRes.size > 1 + tmp = tmpVgprRes.idx + tmpAddr = tmp + 1 + module.addComment1("dump lds state") + module.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="")) + module.add(SBarrier(comment="dump LDS")) + module.add(VLShiftLeftB32( + dst=vgpr(tmpAddr), \ + shiftHex=hex(bpeAB), \ + src=vgpr("Serial"), \ + comment="dump lds")) + for i in range(startU, startU+numU): + module.add(DSLoadB32(dst=vgpr(tmp), src=vgpr(tmpAddr), readToTempVgpr=True, + ds=DSModifiers(offset=(i*numThreads*4)), comment="dump lds")) + module.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="dump")) + module.add(self.dumpVgpr(tmp, labelName)) + return module + + def dumpSgpr(self, sgprStore: Union[int, str], tmpVgprRes: RegisterPoolResource, \ + labelName: str) -> Module: + module = Module("dump sgpr[%s]"%sgprStore) + if self.enableDump: + assert tmpVgprRes.size > 0 + tmp = tmpVgprRes.idx + module.add(VMovB32(dst=vgpr(tmp), src=sgprStore, comment="debug dump sgpr store")) + module.add(self.dumpVgpr(tmp, labelName)) + return module diff --git a/tensilelite/Tensile/TensileInstructions/Formatting.py b/tensilelite/Tensile/TensileInstructions/Formatting.py new file mode 100644 index 0000000000..2fba111f22 --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/Formatting.py @@ -0,0 +1,99 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +## Should not import any module + +import sys + +__TI_DEBUG_LEVEL__ = 0 + +######################################## +# Text format +######################################## + +def slash(comment): + """ + This comment is a single line // MYCOMMENT + """ + return "// %s\n"%comment + +def slash50(comment): + """ + This comment is a single line // MYCOMMENT + """ + return "%-50s // %s\n"%("", comment) + +def block(comment): + """ + This comment is a single line /* MYCOMMENT */ + """ + return "/* %s */\n"%comment + +def blockNewLine(comment): + """ + This comment is a blank line followed by /* MYCOMMENT */ + """ + return "\n/* %s */\n"%comment + +def block3Line(comment): + kStr = "\n/******************************************/\n" + for line in comment.split("\n"): + kStr += "/*" + kStr += " %-38s " % line + kStr += "*/\n" + kStr += "/******************************************/\n" + return kStr + +######################################## +# format string with comment +######################################## + +def formatStr(outputInlineAsm, instStr, comment) -> str: + instStr = "\"%s"%instStr if outputInlineAsm else instStr + if outputInlineAsm: + instStr += "\\n\\t\"" + if comment: + return "%-50s // %s\n" % (instStr, comment) + else: + return "%s\n" % (instStr) + +######################################## +# Terminal output +######################################## + +def print1(message): + if __TI_DEBUG_LEVEL__ >= 1: + print(message) + sys.stdout.flush() + +def print2(message): + if __TI_DEBUG_LEVEL__ >= 2: + print(message) + sys.stdout.flush() + +def printWarning(message): + print("TensileInstructions::WARNING: %s" % message) + sys.stdout.flush() + +def printExit(message): + print("TensileInstructions::FATAL: %s" % message) + sys.stdout.flush() + sys.exit(-1) diff --git a/tensilelite/Tensile/TensileInstructions/Instructions.py b/tensilelite/Tensile/TensileInstructions/Instructions.py new file mode 100644 index 0000000000..47b5994cba --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/Instructions.py @@ -0,0 +1,2085 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Base import Item +from .Enums import InstType, CvtType +from .Containers import DSModifiers, FLATModifiers, MUBUFModifiers, SMEMModifiers, SDWAModifiers, VOP3PModifiers, VCC, \ + RegisterContainer, HolderContainer +from .Formatting import formatStr, printExit +import abc +from enum import Enum +from typing import List, Optional, Union + +################################################################################ +################################################################################ +### +### Instruction classes +### +################################################################################ +################################################################################ + +class Instruction(Item, abc.ABC): + def __init__(self, instType: InstType, comment="") -> None: + Item.__init__(self, "instruction") + self.instType = instType + self.comment = comment + + # Inst string + self.instStr = "" + + # Settings + self.outputInlineAsm = False + + def setInlineAsm(self, isTrue: bool) -> None: + self.outputInlineAsm = isTrue + + def formatOnly(self, instStr, comment) -> str: + return formatStr(self.outputInlineAsm, instStr, comment) + + def formatWithComment(self, instStr) -> str: + return formatStr(self.outputInlineAsm, instStr, self.comment) + + def formatWithExtraComment(self, instStr, comment) -> str: + comment = self.comment + comment + return formatStr(self.outputInlineAsm, instStr, comment) + + def setInst(self, instStr: str): + self.instStr = instStr + + def preStr(self): + # Overwrite this if needed + pass + + @abc.abstractmethod + def getParams(self) -> list: + pass + + @abc.abstractmethod + def toList(self) -> list: + pass + + @abc.abstractmethod + def __str__(self) -> str: + pass + +class CompositeInstruction(Instruction): + def __init__(self, instType: InstType, dst, srcs: Optional[List], comment=""): + super().__init__(instType, comment) + self.instructions: List = [] + self.dst = dst + self.srcs = srcs + + def getInstructions(self) -> List[Instruction]: + self.setupInstructions() + return self.instructions + + def toList(self) -> List[List]: + self.setupInstructions() + return [inst.toList() for inst in self.instructions] + + @abc.abstractmethod + def setupInstructions(self): + pass + + def getParams(self) -> list: + assert 0 + return [] + + def preStr(self): + self.setupInstructions() + + def __str__(self): + self.preStr() + return '\n'.join([str(s) for s in self.instructions]) + +class CommonInstruction(Instruction): + def __init__(self, instType: InstType, dst, src: list, \ + sdwa: Optional[SDWAModifiers]=None, vop3: Optional[VOP3PModifiers]=None, \ + comment="") -> None: + super().__init__(instType, comment) + self.dst = dst + self.dst1 = None # Usually we don't need this + self.src = src + self.sdwa = sdwa + self.vop3 = vop3 + + def getArgStr(self) -> str: + kStr = "" + if self.dst: + kStr += str(self.dst) + if self.dst1: + if kStr: + kStr += ", " + kStr += str(self.dst1) + if self.src: + if kStr: + kStr += ", " + kStr += str(self.src[0]) + for i in self.src[1:]: + kStr += ", " + str(i) + return kStr + + def getParams(self) -> list: + l = [] + if self.dst: + l.append(self.dst) + if self.dst1: + l.append(self.dst1) + if self.src: + l.extend(self.src) + return l + + def toList(self) -> list: + self.preStr() + l = [self.instStr] + if self.dst: + l.append(self.dst) + if self.dst1: + l.append(self.dst1) + if self.src: + l.extend(self.src) + l.extend(self.sdwa.toList()) if self.sdwa else "" + l.extend(self.vop3.toList()) if self.vop3 else "" + l.append(self.comment) + return l + + def __str__(self) -> str: + self.preStr() + kStr = self.instStr + " " + self.getArgStr() + kStr += str(self.sdwa) if self.sdwa else "" + kStr += str(self.vop3) if self.vop3 else "" + return self.formatWithComment(kStr) + +class BranchInstruction(Instruction): + def __init__(self, labelName: str, comment="") -> None: + super().__init__(InstType.INST_NOTYPE, comment) + self.labelName = labelName + + def getParams(self) -> list: + return [self.labelName] + + def toList(self) -> list: + return [self.instStr, self.labelName, self.comment] + + def __str__(self) -> str: + return self.formatWithComment(self.instStr + " " + str(self.labelName)) + +class VCmpInstruction(CommonInstruction): + def __init__(self, instType: InstType, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(instType, dst, [src0, src1], sdwa, None, comment) + +class VCmpXInstruction(CommonInstruction): + def __init__(self, instType: InstType, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(instType, dst, [src0, src1], sdwa, None, comment) + + def toList(self) -> list: + self.preStr() + if self.archCaps["CMPXWritesSGPR"]: + l = [self.instStr, self.dst] + if self.src: + l.extend(self.src) + l.extend(self.sdwa.toList()) if self.sdwa else "" + l.append(self.comment) + else: + instStr = self.instStr.replace("_cmpx_", "_cmp_") + l1 = [instStr, self.dst] + if self.src: + l1.extend(self.src) + l1.extend(self.sdwa.toList()) if self.sdwa else "" + l1.append(self.comment) + if self.kernel.wavefrontSize == 64: + l2 = ["s_mov_b64", "exec", self.dst, ""] + else: + l2 = ["s_mov_b32", "exec_lo", self.dst, ""] + l = [l1, l2] + return l + + def __str__(self) -> str: + self.preStr() + if self.archCaps["CMPXWritesSGPR"]: + kStr = self.instStr + " " + self.getArgStr() + kStr += str(self.sdwa) if self.sdwa else "" + kStr += str(self.vop3) if self.vop3 else "" + kStr = self.formatWithComment(kStr) + else: + kStr = self.instStr.replace("_cmpx_", "_cmp_") + kStr += " " + self.getArgStr() + kStr += str(self.sdwa) if self.sdwa else "" + kStr = self.formatWithComment(kStr) + if self.kernel.wavefrontSize == 64: + kStr2 = "s_mov_b64 exec " + str(self.dst) + else: + kStr2 = "s_mov_b32 exec_lo " + str(self.dst) + kStr2 = self.formatWithComment(kStr2) + kStr += kStr2 + return kStr + +class VCvtInstruction(CommonInstruction): + def __init__(self, cvtType: CvtType, dst, src, sdwa: Optional[SDWAModifiers] = None, \ + comment="") -> None: + super().__init__(InstType.INST_CVT, dst, [src], sdwa, None, comment) + self.cvtType = cvtType + +class MFMAInstruction(Instruction): + def __init__(self, instType: InstType, accType: InstType, variant: List[int], mfma1k, \ + acc, a, b, acc2=None, comment="") -> None: + super().__init__(instType, comment) + self.accType = accType + self.variant = variant + self.mfma1k = mfma1k + self.acc = acc + self.a = a + self.b = b + self.acc2 = acc if acc2 == None else acc2 + + def typeConvert(self, iType) -> str: + if iType == InstType.INST_F16: + kStr = "f16" + elif iType == InstType.INST_F32: + kStr = "f32" + elif iType == InstType.INST_BF16: + kStr = "bf16" + elif iType == InstType.INST_I8: + kStr = "i8" + elif iType == InstType.INST_I32: + kStr = "i32" + else: + printExit("Type %s not found"%str(iType)) + return kStr + + def getParams(self) -> list: + return [self.acc, self.a, self.b, self.acc2] + + def preStr(self) -> None: + if len(self.variant) == 3: + variantStr = "{}x{}x{}".format(*self.variant) + mfma_1k = "_1k" if self.mfma1k else "" + self.setInst("v_mfma_%s_%s%s%s"%(self.typeConvert(self.accType), variantStr, \ + self.typeConvert(self.instType), mfma_1k)) + else: + assert("Currently does not support mfma variant != 3" and 0) + + def getArgStr(self) -> str: + return str(self.acc) + ", " + str(self.a) + ", " + str(self.b) + ", " + str(self.acc2) + + def toList(self) -> list: + self.preStr() + return [self.instStr, self.acc, self.a, self.b, self.acc2, self.comment] + + def __str__(self) -> str: + self.preStr() + kStr = self.instStr + " " + self.getArgStr() + return self.formatWithComment(kStr) + +class MacroInstruction(Instruction): + def __init__(self, name: str, args: list, comment="") -> None: + super().__init__(InstType.INST_MACRO, comment) + self.name = name + self.args = args + + def getParams(self) -> list: + return self.args + + def getArgStr(self) -> str: + kStr = "" + if self.args: + kStr += " " + str(self.args[0]) + for arg in self.args[1:]: + kStr += ", " + str(arg) + return kStr + + def toList(self) -> list: + l = [self.name] + l.extend(self.args) + l.append(self.comment) + return l + + def __str__(self) -> str: + return self.formatWithComment(self.name + self.getArgStr()) + +class ReadWriteInstruction(Instruction): + class RWType(Enum): + RW_TYPE0 = 1 + RW_TYPE1 = 2 + + def __init__(self, instType: InstType, rwType: RWType, comment="") -> None: + super().__init__(instType, comment) + self.rwType = rwType + + def typeConvert(self) -> str: + kStr = "" + if self.rwType == ReadWriteInstruction.RWType.RW_TYPE0: + if self.instType == InstType.INST_U16: + kStr = "ushort" if self.kernel.isa[0] < 11 else "u16" + elif self.instType == InstType.INST_B8: + kStr = "byte" if self.kernel.isa[0] < 11 else "b8" + elif self.instType == InstType.INST_B16: + kStr = "short" if self.kernel.isa[0] < 11 else "b16" + elif self.instType == InstType.INST_B32: + kStr = "dword" if self.kernel.isa[0] < 11 else "b32" + elif self.instType == InstType.INST_B64: + kStr = "dwordx2" if self.kernel.isa[0] < 11 else "b64" + elif self.instType == InstType.INST_B128: + kStr = "dwordx4" if self.kernel.isa[0] < 11 else "b128" + elif self.instType == InstType.INST_B256: + kStr = "dwordx8" if self.kernel.isa[0] < 11 else "b256" + elif self.instType == InstType.INST_B512: + kStr = "dwordx16" if self.kernel.isa[0] < 11 else "b512" + elif self.instType == InstType.INST_D16_U8: + kStr = "ubyte_d16" if self.kernel.isa[0] < 11 else "d16_u8" + elif self.instType == InstType.INST_D16_HI_U8: + kStr = "ubyte_d16_hi" if self.kernel.isa[0] < 11 else "d16_hi_u8" + elif self.instType == InstType.INST_D16_HI_B8: + kStr = "byte_d16_hi" if self.kernel.isa[0] < 11 else "d16_hi_b8" + elif self.instType == InstType.INST_D16_B16: + kStr = "short_d16" if self.kernel.isa[0] < 11 else "d16_b16" + elif self.instType == InstType.INST_D16_HI_B16: + kStr = "short_d16_hi" if self.kernel.isa[0] < 11 else "d16_hi_b16" + return kStr + + def preStr(self): + # Local read is set in DSLoad and DSStore + self.instStr += self.typeConvert() + + @abc.abstractmethod + def toList(self) -> list: + pass + @abc.abstractmethod + def __str__(self) -> str: + pass + + @staticmethod + def issueLatency(): + # In Quad-Cycle + return 1 + +class GlobalReadInstruction(ReadWriteInstruction): + def __init__(self, instType: InstType, dst, comment="") -> None: + super().__init__(instType, ReadWriteInstruction.RWType.RW_TYPE0, comment) + self.dst = dst + +class FLATReadInstruction(GlobalReadInstruction): + def __init__(self, instType: InstType, dst, vaddr, \ + flat: Optional[FLATModifiers] = None, \ + comment="") -> None: + super().__init__(instType, dst, comment) + self.instStr = "flat_load_" + self.vaddr = vaddr + self.flat = flat + + def getParams(self) -> list: + return [self.dst, self.vaddr] + + def getArgStr(self) -> str: + return str(self.dst) + ", " + str(self.vaddr) + + def toList(self) -> list: + self.preStr() + l = [self.instStr, self.dst, self.vaddr] + l.extend(self.flat.toList()) if self.flat else "" + l.append(self.comment) + return l + + def __str__(self) -> str: + self.preStr() + kStr = self.instStr + " " + self.getArgStr() + kStr += str(self.flat) if self.flat else "" + return self.formatWithComment(kStr) + +class MUBUFReadInstruction(GlobalReadInstruction): + def __init__(self, instType: InstType, dst, vaddr, saddr, soffset, \ + mubuf: Optional[MUBUFModifiers] = None, \ + comment="") -> None: + super().__init__(instType, dst, comment) + self.instStr = "buffer_load_" + self.vaddr = vaddr + self.saddr = saddr + self.soffset = soffset + self.mubuf = mubuf + + def getParams(self) -> list: + return [self.dst, self.vaddr, self.saddr, self.soffset] + + def getArgStr(self) -> str: + return str(self.dst) + ", " + str(self.vaddr) + ", " + str(self.saddr) + ", " + str(self.soffset) + + def toList(self) -> list: + self.preStr() + l = [self.instStr, self.dst, self.vaddr, self.saddr, self.soffset] + l.extend(self.mubuf.toList()) if self.mubuf else "" + l.append(self.comment) + return l + + def __str__(self) -> str: + self.preStr() + kStr = self.instStr + " " + self.getArgStr() + kStr += str(self.mubuf) if self.mubuf else "" + return self.formatWithComment(kStr) + +class SMemLoadInstruction(GlobalReadInstruction): + def __init__(self, instType: InstType, dst, base, soffset, + smem: Optional[SMEMModifiers]=None, comment="") -> None: + super().__init__(instType, dst, comment) + self.instStr = "s_load_" + self.base = base + self.soffset = soffset + self.smem = smem + + def getParams(self) -> list: + return [self.dst, self.base, self.soffset] + + def getArgStr(self) -> str: + return str(self.dst) + ", " + str(self.base) + ", " + str(self.soffset) + + def toList(self) -> list: + self.preStr() + l = [self.instStr, self.dst, self.base, self.soffset] + l.extend(self.smem.toList()) if self.smem else "" + l.append(self.comment) + return l + + def __str__(self) -> str: + self.preStr() + kStr = self.instStr + " " + self.getArgStr() + kStr += str(self.smem) if self.smem else "" + return self.formatWithComment(kStr) + +class GlobalWriteInstruction(ReadWriteInstruction): + def __init__(self, instType: InstType, srcData, comment="") -> None: + super().__init__(instType, ReadWriteInstruction.RWType.RW_TYPE0, comment) + self.srcData = srcData + +class FLATStoreInstruction(GlobalWriteInstruction): + def __init__(self, instType: InstType, vaddr, srcData, \ + flat: Optional[FLATModifiers] = None, \ + comment="") -> None: + super().__init__(instType, srcData, comment) + self.instStr = "flat_store_" + self.vaddr = vaddr + self.flat = flat + + def getParams(self) -> list: + return [self.vaddr, self.srcData] + + def getArgStr(self) -> str: + return str(self.vaddr) + ", " + str(self.srcData) + + def toList(self) -> list: + self.preStr() + l = [self.instStr, self.vaddr, self.srcData] + l.extend(self.flat.toList()) if self.flat else "" + l.append(self.comment) + return l + + def __str__(self) -> str: + self.preStr() + kStr = self.instStr + " " + self.getArgStr() + kStr += str(self.flat) if self.flat else "" + return self.formatWithComment(kStr) + +class MUBUFStoreInstruction(GlobalWriteInstruction): + def __init__(self, instType: InstType, srcData, vaddr, saddr, soffset, \ + mubuf: Optional[MUBUFModifiers] = None, \ + comment="") -> None: + super().__init__(instType, srcData, comment) + self.instStr = "buffer_store_" + self.vaddr = vaddr + self.saddr = saddr + self.soffset = soffset + self.mubuf = mubuf + + def getParams(self) -> list: + return [self.srcData, self.vaddr, self.saddr, self.soffset] + + def getArgStr(self) -> str: + return str(self.srcData) + ", " + str(self.vaddr) + ", " + str(self.saddr) + ", " + str(self.soffset) + + def toList(self) -> list: + self.preStr() + l = [self.instStr, self.srcData, self.vaddr, self.saddr, self.soffset] + l.extend(self.mubuf.toList()) if self.mubuf else "" + l.append(self.comment) + return l + + def __str__(self) -> str: + self.preStr() + kStr = self.instStr + " " + self.getArgStr() + kStr += str(self.mubuf) if self.mubuf else "" + return self.formatWithComment(kStr) + +class LocalReadInstruction(ReadWriteInstruction): + def __init__(self, instType: InstType, dst, src, \ + readToTempVgpr: bool, comment="") -> None: + super().__init__(instType, ReadWriteInstruction.RWType.RW_TYPE1, comment) + self.dst = dst + self.src = src + + self.readToTempVgpr = readToTempVgpr + +class DSLoadInstruction(LocalReadInstruction): + def __init__(self, instType: InstType, dst, src, \ + readToTempVgpr: bool, \ + ds: Optional[DSModifiers] = None, \ + comment="") -> None: + super().__init__(instType, dst, src, readToTempVgpr, comment) + self.ds = ds + + def getParams(self) -> list: + return [self.dst, self.src] + + def preStr(self): + if self.kernel.isa[0] < 11: + self.instStr = self.instStr.replace("load", "read") + + def getArgStr(self) -> str: + return str(self.dst) + ", " + str(self.src) + + def toList(self) -> list: + self.preStr() + l = [self.instStr, self.dst, self.src] + l.extend(self.ds.toList()) if self.ds else "" + l.append(self.comment) + return l + + def __str__(self) -> str: + self.preStr() + kStr = self.instStr + " " + self.getArgStr() + kStr += str(self.ds) if self.ds else "" + return self.formatWithComment(kStr) + +class LocalWriteInstruction(ReadWriteInstruction): + def __init__(self, instType: InstType, dstAddr, src0, src1, \ + comment="") -> None: + super().__init__(instType, ReadWriteInstruction.RWType.RW_TYPE1, comment) + self.dstAddr = dstAddr + self.src0 = src0 + self.src1 = src1 + +class DSStoreInstruction(LocalWriteInstruction): + def __init__(self, instType: InstType, dstAddr, src0, src1, \ + ds: Optional[DSModifiers] = None, \ + comment="") -> None: + super().__init__(instType, dstAddr, src0, src1, comment) + self.ds = ds + + def getParams(self) -> list: + return [self.dstAddr, self.src0, self.src1] + + def preStr(self): + if self.kernel.isa[0] < 11: + self.instStr = self.instStr.replace("store", "write") + + def getArgStr(self) -> str: + kStr = str(self.dstAddr) + ", " + str(self.src0) + if self.src1: + kStr += ", " + str(self.src1) + return kStr + + def toList(self) -> list: + self.preStr() + l = [self.instStr, self.dstAddr, self.src0] + l.extend(self.src1) if self.src1 else "" + l.extend(self.ds.toList()) if self.ds else "" + l.append(self.comment) + return l + + def __str__(self) -> str: + self.preStr() + kStr = self.instStr + " " + self.getArgStr() + kStr += str(self.ds) if self.ds else "" + return self.formatWithComment(kStr) + +################################################################################ +################################################################################ +### +### Instructions +### +################################################################################ +################################################################################ + +################################################################################ +### Read/ Write instructions +################################################################################ + +## Buffer load +class BufferLoadD16HIU8(MUBUFReadInstruction): + def __init__(self, dst, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_HI_U8, dst, vaddr, saddr, soffset, mubuf, comment) + +class BufferLoadD16U8(MUBUFReadInstruction): + def __init__(self, dst, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_U8, dst, vaddr, saddr, soffset, mubuf, comment) + +class BufferLoadD16HIB16(MUBUFReadInstruction): + def __init__(self, dst, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_HI_B16, dst, vaddr, saddr, soffset, mubuf, comment) + +class BufferLoadD16B16(MUBUFReadInstruction): + def __init__(self, dst, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_B16, dst, vaddr, saddr, soffset, mubuf, comment) + +class BufferLoadB32(MUBUFReadInstruction): + def __init__(self, dst, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, dst, vaddr, saddr, soffset, mubuf, comment) + +class BufferLoadB64(MUBUFReadInstruction): + def __init__(self, dst, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B64, dst, vaddr, saddr, soffset, mubuf, comment) + +class BufferLoadB128(MUBUFReadInstruction): + def __init__(self, dst, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B128, dst, vaddr, saddr, soffset, mubuf, comment) + +## Flat load +class FlatLoadD16HIB16(FLATReadInstruction): + def __init__(self, dst, vaddr, flat: Optional[FLATModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_HI_B16, dst, vaddr, flat, comment) + +class FlatLoadD16B16(FLATReadInstruction): + def __init__(self, dst, vaddr, flat: Optional[FLATModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_B16, dst, vaddr, flat, comment) + +class FlatLoadB32(FLATReadInstruction): + def __init__(self, dst, vaddr, flat: Optional[FLATModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, dst, vaddr, flat, comment) + +class FlatLoadB64(FLATReadInstruction): + def __init__(self, dst, vaddr, flat: Optional[FLATModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B64, dst, vaddr, flat, comment) + +class FlatLoadB128(FLATReadInstruction): + def __init__(self, dst, vaddr, flat: Optional[FLATModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B128, dst, vaddr, flat, comment) + +## Buffer store +class BufferStoreD16HIU8(MUBUFStoreInstruction): + def __init__(self, src, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_HI_U8, src, vaddr, saddr, soffset, mubuf, comment) + +class BufferStoreD16U8(MUBUFStoreInstruction): + def __init__(self, src, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_U8, src, vaddr, saddr, soffset, mubuf, comment) + +class BufferStoreD16HIB16(MUBUFStoreInstruction): + def __init__(self, src, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_HI_B16, src, vaddr, saddr, soffset, mubuf, comment) + +class BufferStoreD16B16(MUBUFStoreInstruction): + def __init__(self, src, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_B16, src, vaddr, saddr, soffset, mubuf, comment) + +class BufferStoreB16(MUBUFStoreInstruction): + def __init__(self, src, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B16, src, vaddr, saddr, soffset, mubuf, comment) + +class BufferStoreB32(MUBUFStoreInstruction): + def __init__(self, src, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, src, vaddr, saddr, soffset, mubuf, comment) + +class BufferStoreB64(MUBUFStoreInstruction): + def __init__(self, src, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B64, src, vaddr, saddr, soffset, mubuf, comment) + +class BufferStoreB128(MUBUFStoreInstruction): + def __init__(self, src, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B128, src, vaddr, saddr, soffset, mubuf, comment) + +class BufferAtomicAddF32(MUBUFStoreInstruction): + def __init__(self, src, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, src, vaddr, saddr, soffset, mubuf, comment) + self.setInst("buffer_atomic_add_f32") + + def typeConvert(self) -> str: + return "" + +class BufferAtomicCmpswapB32(MUBUFStoreInstruction): + def __init__(self, src, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, src, vaddr, saddr, soffset, mubuf, comment) + self.setInst("buffer_atomic_cmpswap") + + def typeConvert(self) -> str: + return "" + +class BufferAtomicCmpswapB64(MUBUFStoreInstruction): + def __init__(self, src, vaddr, saddr, soffset, mubuf: Optional[MUBUFModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, src, vaddr, saddr, soffset, mubuf, comment) + self.setInst("buffer_atomic_cmpswap_x2") + + def typeConvert(self) -> str: + return "" + +## Flat store +class FlatStoreD16HIB16(FLATStoreInstruction): + def __init__(self, vaddr, src, flat: Optional[FLATModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_HI_B16, vaddr, src, flat, comment) + +class FlatStoreD16B16(FLATStoreInstruction): + def __init__(self, vaddr, src, flat: Optional[FLATModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_B16, vaddr, src, flat, comment) + +class FlatStoreB32(FLATStoreInstruction): + def __init__(self, vaddr, src, flat: Optional[FLATModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, vaddr, src, flat, comment) + +class FlatStoreB64(FLATStoreInstruction): + def __init__(self, vaddr, src, flat: Optional[FLATModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B64, vaddr, src, flat, comment) + +class FlatStoreB128(FLATStoreInstruction): + def __init__(self, vaddr, src, flat: Optional[FLATModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B128, vaddr, src, flat, comment) + +class FlatAtomicCmpswapB32(FLATStoreInstruction): + def __init__(self, vaddr, tmp, src, flat: Optional[FLATModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, vaddr, src, flat, comment) + self.tmp = tmp + self.setInst("flat_atomic_cmpswap") + + def getArgStr(self) -> str: + return ", ".join(map(str, [self.vaddr, self.tmp, self.srcData])) + + def toList(self) -> List: + self.preStr() + l = [self.instStr, self.vaddr, self.tmp, self.srcData] + if self.flat: l.extend(self.flat.toList()) + l.append(self.comment) + return l + + def typeConvert(self) -> str: + return "" + + def __str__(self) -> str: + self.preStr() + kStr = " ".join([self.instStr, self.getArgStr()]) + kStr += str(self.flat) if self.flat else "" + return self.formatWithComment(kStr) + +# DS Load +class DSLoadU8(DSLoadInstruction): + def __init__(self, dst, src, readToTempVgpr: bool, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U8, dst, src, readToTempVgpr, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_load_u8") + +class DSLoadD16HIU8(DSLoadInstruction): + def __init__(self, dst, src, readToTempVgpr: bool, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_HI_U8, dst, src, readToTempVgpr, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_load_u8_d16_hi") + +class DSLoadU16(DSLoadInstruction): + def __init__(self, dst, src, readToTempVgpr: bool, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U16, dst, src, readToTempVgpr, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_load_u16") + +class DSLoadD16HIU16(DSLoadInstruction): + def __init__(self, dst, src, readToTempVgpr: bool, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_HI_U16, dst, src, readToTempVgpr, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_load_u16_d16_hi") + +class DSLoadB32(DSLoadInstruction): + def __init__(self, dst, src, readToTempVgpr: bool, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, dst, src, readToTempVgpr, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_load_b32") + +class DSLoadB64(DSLoadInstruction): + def __init__(self, dst, src, readToTempVgpr: bool, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B64, dst, src, readToTempVgpr, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_load_b64") + +class DSLoadB128(DSLoadInstruction): + def __init__(self, dst, src, readToTempVgpr: bool, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B128, dst, src, readToTempVgpr, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_load_b128") + + @staticmethod + def issueLatency(): + return 2 + +class DSLoad2B32(DSLoadInstruction): + def __init__(self, dst, src, readToTempVgpr: bool, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, dst, src, readToTempVgpr, ds, comment) + if ds: ds.na = 2 + self.setInst("ds_load2_b32") + +class DSLoad2B64(DSLoadInstruction): + def __init__(self, dst, src, readToTempVgpr: bool, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B64, dst, src, readToTempVgpr, ds, comment) + if ds: ds.na = 2 + self.setInst("ds_load2_b64") + +# DS store +class DSStoreU16(DSStoreInstruction): + def __init__(self, dstAddr, src, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U16, dstAddr, src, None, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_store_u16") + + @staticmethod + def issueLatency(): + return 2 + +class DSStoreB8(DSStoreInstruction): + def __init__(self, dstAddr, src, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B8, dstAddr, src, None, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_store_b8") + +class DSStoreB16(DSStoreInstruction): + def __init__(self, dstAddr, src, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B16, dstAddr, src, None, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_store_b16") + +class DSStoreB8HID16(DSStoreInstruction): + def __init__(self, dstAddr, src, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B8_HI_D16, dstAddr, src, None, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_store_b8_d16_hi") + +class DSStoreD16HIB16(DSStoreInstruction): + def __init__(self, dstAddr, src, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_D16_HI_B16, dstAddr, src, None, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_store_b16_d16_hi") + +class DSStoreB32(DSStoreInstruction): + def __init__(self, dstAddr, src, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, dstAddr, src, None, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_store_b32") + + @staticmethod + def issueLatency(): + return 2 + +class DSStoreB64(DSStoreInstruction): + def __init__(self, dstAddr, src, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B64, dstAddr, src, None, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_store_b64") + + @staticmethod + def issueLatency(): + return 3 + +class DSStoreB128(DSStoreInstruction): + def __init__(self, dstAddr, src, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B128, dstAddr, src, None, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_store_b128") + + @staticmethod + def issueLatency(): + return 5 + +class DSStore2B32(DSStoreInstruction): + def __init__(self, dstAddr, src0, src1, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, dstAddr, src0, src1, ds, comment) + if ds: ds.na = 2 + self.setInst("ds_store2_b32") + + @staticmethod + def issueLatency(): + return 3 + +class DSStore2B64(DSStoreInstruction): + def __init__(self, dstAddr, src0, src1, ds: Optional[DSModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B64, dstAddr, src0, src1, ds, comment) + if ds: ds.na = 2 + self.setInst("ds_store2_b64") + + @staticmethod + def issueLatency(): + return 3 + +# DS instructions +class DSBPermuteB32(DSStoreInstruction): + def __init__(self, dst, src0, src1, ds: Optional[DSModifiers]=None, comment="") -> None: + super().__init__(InstType.INST_B32, dst, src0, src1, ds, comment) + if ds: ds.na = 1 + self.setInst("ds_bpermute_b32") + +################################################################################ +### SGPR instructions +################################################################################ + +# Abs +class SAbsI32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src], None, None, comment) + self.setInst("s_abs_i32") + +# Arithmetic +class SAddI32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src0, src1], None, None, comment) + self.setInst("s_add_i32") +class SAddU32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1], None, None, comment) + self.setInst("s_add_u32") + +class SAddCU32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1], None, None, comment) + self.setInst("s_addc_u32") + +class SMulI32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src0, src1], None, None, comment) + self.setInst("s_mul_i32") + +class SMulHII32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_HI_I32, dst, [src0, src1], None, None, comment) + self.setInst("s_mul_hi_i32") + +class SMulHIU32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_HI_U32, dst, [src0, src1], None, None, comment) + self.setInst("s_mul_hi_u32") + +class SSubI32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src0, src1], None, None, comment) + self.setInst("s_sub_i32") + +class SSubU32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1], None, None, comment) + self.setInst("s_sub_u32") + +class SSubBU32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1], None, None, comment) + self.setInst("s_subb_u32") + +# Cmp +class SCmpEQI32(CommonInstruction): + def __init__(self, src0, src1, comment="") -> None: + super().__init__(InstType.INST_I32, None, [src0, src1], None, None, comment) + self.setInst("s_cmp_eq_i32") + +class SCmpEQU32(CommonInstruction): + def __init__(self, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, None, [src0, src1], None, None, comment) + self.setInst("s_cmp_eq_u32") + +class SCmpEQU64(CommonInstruction): + def __init__(self, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U64, None, [src0, src1], None, None, comment) + self.setInst("s_cmp_eq_u64") + +class SCmpGeI32(CommonInstruction): + def __init__(self, src0, src1, comment="") -> None: + super().__init__(InstType.INST_I32, None, [src0, src1], None, None, comment) + self.setInst("s_cmp_ge_i32") + +class SCmpGeU32(CommonInstruction): + def __init__(self, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, None, [src0, src1], None, None, comment) + self.setInst("s_cmp_ge_u32") + +class SCmpLeI32(CommonInstruction): + def __init__(self, src0, src1, comment="") -> None: + super().__init__(InstType.INST_I32, None, [src0, src1], None, None, comment) + self.setInst("s_cmp_le_i32") + +class SCmpLeU32(CommonInstruction): + def __init__(self, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, None, [src0, src1], None, None, comment) + self.setInst("s_cmp_le_u32") + +# SCC = (S0 != S1) +class SCmpLgU32(CommonInstruction): + def __init__(self, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, None, [src0, src1], None, None, comment) + self.setInst("s_cmp_lg_u32") + +class SCmpLtU32(CommonInstruction): + def __init__(self, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, None, [src0, src1], None, None, comment) + self.setInst("s_cmp_lt_u32") + +# S Cmp K +# SCC = (S0.u == SIMM16) +class SCmpKEQU32(CommonInstruction): + def __init__(self, src, simm16: Union[int, str], comment="") -> None: + super().__init__(InstType.INST_U32, None, [src, simm16], None, None, comment) + self.setInst("s_cmpk_eq_u32") + +class SCmpKGtU32(CommonInstruction): + def __init__(self, src, simm16: str, comment="") -> None: + super().__init__(InstType.INST_U32, None, [src, simm16], None, None, comment) + self.setInst("s_cmpk_gt_u32") + +# S Select +# D.u = SCC ? S0.u : S1.u +class SCSelectB32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src0, src1], None, None, comment) + self.setInst("s_cselect_b32") + +# Logic +class SAndB32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src0, src1], None, None, comment) + self.setInst("s_and_b32") + +class SAndB64(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_B64, dst, [src0, src1], None, None, comment) + self.setInst("s_and_b64") + +class SOrB32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src0, src1], None, None, comment) + self.setInst("s_or_b32") + +class SXorB32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src0, src1], None, None, comment) + self.setInst("s_xor_b32") +class SOrB64(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_B64, dst, [src0, src1], None, None, comment) + self.setInst("s_or_b64") + +# Branch +class SBranch(BranchInstruction): + def __init__(self, labelName: str, comment="") -> None: + super().__init__(labelName, comment) + self.setInst("s_branch") +class SCBranchSCC0(BranchInstruction): + def __init__(self, labelName: str, comment="") -> None: + super().__init__(labelName, comment) + self.setInst("s_cbranch_scc0") + +class SCBranchSCC1(BranchInstruction): + def __init__(self, labelName: str, comment="") -> None: + super().__init__(labelName, comment) + self.setInst("s_cbranch_scc1") + +class SCBranchVCCNZ(BranchInstruction): + def __init__(self, labelName: str, comment="") -> None: + super().__init__(labelName, comment) + self.setInst("s_cbranch_vccnz") + +class SCBranchVCCZ(BranchInstruction): + def __init__(self, labelName: str, comment="") -> None: + super().__init__(labelName, comment) + self.setInst("s_cbranch_vccz") + +# S PC +class SGetPCB64(CommonInstruction): + def __init__(self, dst, comment="") -> None: + super().__init__(InstType.INST_B64, dst, [], None, None, comment) + self.setInst("s_getpc_b64") + +class SSetPCB64(BranchInstruction): + def __init__(self, src, comment="") -> None: + super().__init__("", comment) + self.src = src + self.setInst("s_setpc_b64") + + def toList(self) -> list: + return [self.instStr, self.src, self.comment] + + def __str__(self) -> str: + return self.formatWithComment(self.instStr + " " + str(self.src)) + +class SSwapPCB64(BranchInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__("", comment) + self.dst = dst + self.src = src + self.setInst("s_swappc_b64") + + def toList(self) -> list: + return [self.instStr, self.dst, self.src, self.comment] + + def __str__(self) -> str: + return self.formatWithComment(self.instStr + " " + str(self.dst) + ", " + str(self.src)) + +class SCBranchExecZ(BranchInstruction): + def __init__(self, labelName: str, comment="") -> None: + super().__init__(labelName, comment) + self.setInst("s_cbranch_execz") + +class SCBranchExecNZ(BranchInstruction): + def __init__(self, labelName: str, comment="") -> None: + super().__init__(labelName, comment) + self.setInst("s_cbranch_execnz") + +# S Shift +class SLShiftLeftB32(CommonInstruction): + def __init__(self, dst, shiftHex, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src, shiftHex], None, None, comment) + self.setInst("s_lshl_b32") + +class SLShiftRightB32(CommonInstruction): + def __init__(self, dst, shiftHex, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src, shiftHex], None, None, comment) + self.setInst("s_lshr_b32") + +class SLShiftLeftB64(CommonInstruction): + def __init__(self, dst, shiftHex, src, comment="") -> None: + super().__init__(InstType.INST_B64, dst, [src, shiftHex], None, None, comment) + self.setInst("s_lshl_b64") + +class SLShiftRightB64(CommonInstruction): + def __init__(self, dst, shiftHex, src, comment="") -> None: + super().__init__(InstType.INST_B64, dst, [src, shiftHex], None, None, comment) + self.setInst("s_lshr_b64") + +# Arithmetic shift right (preserve sign bit) +class SAShiftRightI32(CommonInstruction): + def __init__(self, dst, shiftHex, src, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src, shiftHex], None, None, comment) + self.setInst("s_ashr_i32") + +# S mov +class SMovB32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src], None, None, comment) + self.setInst("s_mov_b32") + +class SMovB64(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B64, dst, [src], None, None, comment) + self.setInst("s_mov_b64") + +class SCMovB32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src], None, None, comment) + self.setInst("s_cmov_b32") + +class SCMovB64(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B64, dst, [src], None, None, comment) + self.setInst("s_cmov_b64") + +# Sign ext +class SMovkI32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src], None, None, comment) + self.setInst("s_movk_i32") + +# S exec +class SAndSaveExecB32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src], None, None, comment) + self.setInst("s_and_saveexec_b32") + +class SAndSaveExecB64(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B64, dst, [src], None, None, comment) + self.setInst("s_and_saveexec_b64") + +class SOrSaveExecB32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src], None, None, comment) + self.setInst("s_or_saveexec_b32") + +class SOrSaveExecB64(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B64, dst, [src], None, None, comment) + self.setInst("s_or_saveexec_b64") + +class SSetPrior(Instruction): + def __init__(self, prior, comment="") -> None: + super().__init__(InstType.INST_NOTYPE, comment) + self.prior = prior + self.setInst("s_setprio") + + def getParams(self) -> list: + return [self.prior] + + def toList(self) -> list: + return [self.instStr, self.prior, self.comment] + + def __str__(self) -> str: + return self.formatWithComment(self.instStr + " " + str(self.prior)) + +class SBarrier(Instruction): + def __init__(self, comment="") -> None: + super().__init__(InstType.INST_NOTYPE, comment) + self.setInst("s_barrier") + + def getParams(self) -> list: + return [] + + def toList(self) -> list: + return [self.instStr, self.comment] + + def __str__(self) -> str: + return self.formatWithComment(self.instStr) +class SNop(Instruction): + def __init__(self, waitState: int, comment="") -> None: + super().__init__(InstType.INST_NOTYPE, comment) + self.waitState = waitState + self.setInst("s_nop") + + def getParams(self) -> list: + return [self.waitState] + + def toList(self) -> list: + return [self.instStr, self.waitState, self.comment] + + def __str__(self) -> str: + return self.formatWithComment(self.instStr + " " + str(self.waitState)) + +class SEndpgm(Instruction): + def __init__(self, comment="") -> None: + super().__init__(InstType.INST_NOTYPE, comment) + self.setInst("s_endpgm") + + def getParams(self) -> list: + return [] + + def toList(self) -> list: + return [self.instStr, self.comment] + + def __str__(self) -> str: + return self.formatWithComment(self.instStr) +class SSleep(Instruction): + def __init__(self, simm16, comment=""): + super().__init__(InstType.INST_NOTYPE, comment) + self.simm16 = simm16 + self.setInst("s_sleep") + + def getParams(self) -> list: + return [self.simm16] + + def toList(self) -> List: + return [self.instStr, self.simm16, self.comment] + + def __str__(self) -> str: + return self.formatWithComment(f"{self.instStr} {self.simm16}") + +# S WaitCnt +class _SWaitCnt(Instruction): + def __init__(self, lgkmcnt: int=-1, vmcnt: int=-1, comment=""): + super().__init__(InstType.INST_NOTYPE, comment) + self.lgkmcnt = lgkmcnt + self.vmcnt = vmcnt + + def getParams(self) -> list: + return [self.lgkmcnt, self.vmcnt] + + def toList(self) -> list: + assert 0 and "Not supported." + return [] + + def __str__(self) -> str: + if self.lgkmcnt == 0 and self.vmcnt == 0: + waitStr = "0" + else: + waitStr = "" + if self.lgkmcnt != -1: + maxLgkmcnt = self.asmCaps["MaxLgkmcnt"] + waitStr = "lgkmcnt(%u)" % (min(self.lgkmcnt,maxLgkmcnt)) + if self.vmcnt != -1: + waitStr += (", " if waitStr != "" else "") + "vmcnt(%u)"%self.vmcnt + return self.formatWithComment("s_waitcnt %s"%(waitStr)) + +class _SWaitCntVscnt(Instruction): + def __init__(self, vscnt: int=-1, comment="") -> None: + super().__init__(InstType.INST_NOTYPE, comment) + self.vscnt = vscnt + + def getParams(self) -> list: + return [self.vscnt] + + def toList(self) -> list: + assert 0 and "Not supported." + return [] + + def __str__(self) -> str: + return self.formatWithComment("s_waitcnt_vscnt %u"%(self.vscnt)) + +class SWaitCnt(CompositeInstruction): + """ + Construct a waitcnt from specified lgkmcnt and vmcnt: + lgkmcnt, vmcnt: + if -1 then will not be added to the wait term. + + If lgkmcnt=vmcnt=vscnt=-1 then the waitcnt is a nop and + an instruction with a comment is returned. + """ + def __init__(self, lgkmcnt: int=-1, vmcnt: int=-1, vscnt: int=-1, comment="", waitAll=False): + super().__init__(InstType.INST_NOTYPE, None, None, comment=comment) + self.lgkmcnt = lgkmcnt + self.vmcnt = vmcnt + self.vscnt = vscnt + self.waitAll = waitAll + + def setupInstructions(self): + super().setupInstructions() + if self.waitAll: + lgkmcnt = 0 + vmcnt = 0 + vscnt = 0 + comment = "(Wait all)" + else: + lgkmcnt = self.lgkmcnt + vmcnt = self.vmcnt + vscnt = self.vscnt + comment = self.comment + + if self.archCaps["SeparateVscnt"]: + self.instructions = [_SWaitCnt(lgkmcnt, vmcnt, comment), + _SWaitCntVscnt(vscnt, comment)] + else: + vmvscnt = -1 + if vscnt != -1: + vmvscnt = vscnt + if vmcnt != -1: + vmvscnt = vmcnt + (vmvscnt if vmvscnt != -1 else 0) + self.instructions = [_SWaitCnt(lgkmcnt, vmvscnt, comment)] + +# S Load +class SLoadB32(SMemLoadInstruction): + def __init__(self, dst, base, soffset, smem: Optional[SMEMModifiers]=None, comment="") -> None: + super().__init__(InstType.INST_B32, dst, base, soffset, smem, comment) + +class SLoadB64(SMemLoadInstruction): + def __init__(self, dst, base, soffset, smem: Optional[SMEMModifiers]=None, comment="") -> None: + super().__init__(InstType.INST_B64, dst, base, soffset, smem, comment) + +class SLoadB128(SMemLoadInstruction): + def __init__(self, dst, base, soffset, smem: Optional[SMEMModifiers]=None, comment="") -> None: + super().__init__(InstType.INST_B128, dst, base, soffset, smem, comment) + +class SLoadB256(SMemLoadInstruction): + def __init__(self, dst, base, soffset, smem: Optional[SMEMModifiers]=None, comment="") -> None: + super().__init__(InstType.INST_B256, dst, base, soffset, smem, comment) + +class SLoadB512(SMemLoadInstruction): + def __init__(self, dst, base, soffset, smem: Optional[SMEMModifiers]=None, comment="") -> None: + super().__init__(InstType.INST_B512, dst, base, soffset, smem, comment) + +################################################################################ +### VGPR instructions +################################################################################ + +# Arithmetic +class VAddF16(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_add_f16") + +class VAddF32(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_add_f32") + +class VAddF64(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F64, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_add_f64") + +class VAddI32(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_add_i32") + + def preStr(self): + if self.asmBugs["ExplicitNC"]: + self.setInst("v_add_nc_i32") + elif self.asmBugs["ExplicitCO"]: + self.setInst("v_add_i32") + else: + self.setInst("v_add_i32") + +class VAddU32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1], None, None, comment) + + def preStr(self): + if self.asmBugs["ExplicitNC"]: + self.setInst("v_add_nc_u32") + self.dst1 = None + elif self.asmBugs["ExplicitCO"]: + self.setInst("v_add_u32") + self.dst1 = None + else: + self.setInst("v_add_u32") + self.dst1 = VCC() + +class VAddCOU32(CommonInstruction): + def __init__(self, dst, dst1, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1], None, None, comment) + self.dst1 = dst1 + + def preStr(self): + if self.asmBugs["ExplicitCO"]: + self.setInst("v_add_co_u32") + else: + self.setInst("v_add_u32") + +class VAddCCOU32(CommonInstruction): + def __init__(self, dst, dst1, src0, src1, src2, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1, src2], None, None, comment) + self.dst1 = dst1 + self.setInst("_v_addc_co_u32") + + def preStr(self): + if self.asmBugs["ExplicitNC"]: + self.setInst("v_add_co_ci_u32") + elif self.asmBugs["ExplicitCO"]: + self.setInst("v_addc_co_u32") + else: + self.setInst("v_addc_u32") + +class VAddPKF16(CommonInstruction): + def __init__(self, dst, src0, src1, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, [src0, src1], None, vop3, comment) + self.setInst("v_pk_add_f16") + +class _VAddPKF32(CommonInstruction): + def __init__(self, dst, src0, src1, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src0, src1], None, vop3, comment) + self.setInst("v_pk_add_f32") + +class VAddPKF32(CompositeInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src0, src1], comment) + self.setInst("v_pk_add_f32") + + def getParams(self) -> list: + assert 0 and "Not supported." + return [] + + def toList(self) -> list: + assert 0 and "Not supported." + return [] + + def setupInstructions(self): + super().setupInstructions() + assert isinstance(self.srcs, List) + if self.asmCaps["v_pk_add_f32"]: + self.instructions = [_VAddPKF32(self.dst, self.srcs[0], self.srcs[1], None, self.comment)] + else: + dst1, dst2 = self.dst.splitRegContainer() + srcs1 = [] + srcs2 = [] + for s in self.srcs: + if isinstance(s, RegisterContainer) or isinstance(s, HolderContainer): + r1, r2 = s.splitRegContainer() + srcs1.append(r1) + srcs2.append(r2) + else: + srcs1.append(s) + srcs2.append(s) + self.instructions = [VAddF32(dst1, srcs1[0], srcs1[1], None, self.comment), + VAddF32(dst2, srcs2[0], srcs2[1], None, self.comment)] + + assert all(inst.vop3 is None for inst in self.instructions), "Currently does not support with vop3 enabled" + +class VAdd3U32(CommonInstruction): + def __init__(self, dst, src0, src1, src2, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1, src2], None, vop3, comment) + self.setInst("v_add3_u32") + +class VMulF16(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_mul_f16") + +class VMulF32(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_mul_f32") + +class VMulF64(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F64, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_mul_f64") + +class VMulPKF16(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, [src0, src1], sdwa, vop3, comment) + self.setInst("v_pk_mul_f16") + +class VMulLOU32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_LO_U32, dst, [src0, src1], None, None, comment) + self.setInst("v_mul_lo_u32") + +class VMulHII32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_HI_I32, dst, [src0, src1], None, None, comment) + self.setInst("v_mul_hi_i32") + +class VMulHIU32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_HI_U32, dst, [src0, src1], None, None, comment) + self.setInst("v_mul_hi_u32") + +class VMulI32I24(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src0, src1], None, None, comment) + self.setInst("v_mul_i32_i24") + +class VMulU32U24(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1], None, None, comment) + self.setInst("v_mul_u32_u24") + +class VSubI32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src0, src1], None, None, comment) + + def preStr(self): + if self.asmBugs["ExplicitNC"]: + self.setInst("v_sub_nc_i32") + elif self.asmBugs["ExplicitCO"]: + self.setInst("v_sub_i32") + else: + self.setInst("v_sub_i32") + +class VSubU32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1], None, None, comment) + + def preStr(self): + if self.asmBugs["ExplicitNC"]: + self.setInst("v_sub_nc_u32") + elif self.asmBugs["ExplicitCO"]: + self.setInst("v_sub_u32") + else: + self.setInst("v_sub_u32") + +class VSubCoU32(CommonInstruction): + def __init__(self, dst, dst1, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1], None, None, comment) + self.dst1 = dst1 + + def preStr(self): + if self.asmBugs["ExplicitCO"]: + self.setInst("v_sub_co_u32") + else: + self.setInst("v_sub_u32") + +# MAC +class VMacF32(CommonInstruction): + def __init__(self, dst, src0, src1, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src0, src1], None, vop3, comment) + self.setInst("v_mac_f32") + self.addDstToSrc = False + + def preStr(self): + if self.asmCaps["v_fmac_f32"]: + self.setInst("v_fmac_f32") + elif self.asmCaps["v_fma_f32"]: + self.addDstToSrc = True + self.setInst("v_fmac_f32") + elif self.asmCaps["v_mac_f32"]: + self.setInst("v_mac_f32") + else: + raise RuntimeError("FMA and MAC instructions are not supported.") + + def getArgStr(self) -> str: + kStr = super().getArgStr() + if self.addDstToSrc: + kStr += ", " + str(self.dst) + return kStr + +# Dot +class VDot2CF32F16(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_dot2c_f32_f16") + + def preStr(self): + if self.kernel.isa[0] >= 11: + self.setInst("v_dot2acc_f32_f16") + +# Fma +class VFmaF16(CommonInstruction): + def __init__(self, dst, src0, src1, src2, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, [src0, src1, src2], None, vop3, comment) + self.setInst("v_fma_f16") + +class VFmaF32(CommonInstruction): + def __init__(self, dst, src0, src1, src2, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src0, src1, src2], None, vop3, comment) + self.setInst("v_fma_f32") + +class VFmaF64(CommonInstruction): + def __init__(self, dst, src0, src1, src2, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F64, dst, [src0, src1, src2], None, vop3, comment) + self.setInst("v_fma_f64") + +class VFmaPKF16(CommonInstruction): + def __init__(self, dst, src0, src1, src2, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, [src0, src1, src2], None, vop3, comment) + self.setInst("v_pk_fma_f16") + +class VFmaMixF32(CommonInstruction): + def __init__(self, dst, src0, src1, src2, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src0, src1, src2], None, vop3, comment) + self.setInst("v_fma_mix_f32") + +# V Mad +class VMadI32I24(CommonInstruction): + def __init__(self, dst, src0, src1, src2, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src0, src1, src2], None, vop3, comment) + self.setInst("v_mad_i32_i24") + +class VMadU32U24(CommonInstruction): + def __init__(self, dst, src0, src1, src2, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1, src2], None, vop3, comment) + self.setInst("v_mad_u32_u24") + +class VMadMixF32(CommonInstruction): + def __init__(self, dst, src0, src1, src2, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src0, src1, src2], None, vop3, comment) + self.setInst("v_mad_mix_f32") + +# Exp, rcp +class VExpF16(CommonInstruction): + def __init__(self, dst, src, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, [src], sdwa, None, comment) + self.setInst("v_exp_f16") + +class VExpF32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src], None, None, comment) + self.setInst("v_exp_f32") + +class VRcpF16(CommonInstruction): + def __init__(self, dst, src, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, [src], sdwa, None, comment) + self.setInst("v_rcp_f16") + +class VRcpF32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src], None, None, comment) + self.setInst("v_rcp_f32") + +# Cmp +class VCmpEQF32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_eq_f32") + +class VCmpEQF64(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F64, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_eq_f64") + +class VCmpEQU32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_eq_u32") + +class VCmpGEF16(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_ge_f16") + +class VCmpGEF32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_ge_f32") + +class VCmpGEF64(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F64, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_ge_f64") + +class VCmpGEI32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_I32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_ge_i32") + +class VCmpGEU32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_ge_u32") + +class VCmpGtU32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_gt_u32") + +class VCmpLeU32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_le_u32") + +class VCmpLtI32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_I32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_lt_i32") + +class VCmpLtU32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_lt_u32") + +# D.u64[threadId] = (isNan(S0) || isNan(S1)) +class VCmpUF32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_u_f32") + +class VCmpNeI32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_I32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_ne_i32") + +class VCmpNeU32(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_ne_u32") + +class VCmpNeU64(VCmpInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U64, dst, src0, src1, sdwa, comment) + self.setInst("v_cmp_ne_u64") + +# CmpX +class VCmpXEqU32(VCmpXInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmpx_eq_u32") + +class VCmpXGeU32(VCmpXInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmpx_ge_u32") + +class VCmpXGtU32(VCmpXInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmpx_gt_u32") + +class VCmpXLeU32(VCmpXInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmpx_le_u32") + +class VCmpXLtI32(VCmpXInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_I32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmpx_lt_i32") + +class VCmpXLtU32(VCmpXInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmpx_lt_u32") + +class VCmpXLtU64(VCmpXInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U64, dst, src0, src1, sdwa, comment) + self.setInst("v_cmpx_lt_u64") + +class VCmpXNeU16(VCmpXInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U16, dst, src0, src1, sdwa, comment) + self.setInst("v_cmpx_ne_u16") + +class VCmpXNeU32(VCmpXInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, src0, src1, sdwa, comment) + self.setInst("v_cmpx_ne_u32") + +# Min Max +class VMaxF16(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_max_f16") + +class VMaxF32(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_max_f32") + +class VMaxF64(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F64, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_max_f64") + +class VMaxI32(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_max_i32") + +class VMaxPKF16(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, [src0, src1], sdwa, vop3, comment) + self.setInst("v_pk_max_f16") + +class VMed3I32(CommonInstruction): + def __init__(self, dst, src0, src1, src2, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src0, src1, src2], None, None, comment) + self.setInst("v_med3_i32") + +class VMinF16(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F16, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_min_f16") + +class VMinF32(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F32, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_min_f32") + +class VMinF64(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_F64, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_min_f64") + +class VMinI32(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_min_i32") + +# V Logic +class VAndB32(CommonInstruction): + def __init__(self, dst, src0, src1, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src0, src1], None, None, comment) + self.setInst("v_and_b32") + +class VAndOrB32(CommonInstruction): + def __init__(self, dst, src0, src1, src2, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src0, src1, src2], None, None, comment) + self.setInst("v_and_or_b32") + +class VNotB32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src], None, None, comment) + self.setInst("v_not_b32") + +class VOrB32(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers]=None, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_or_b32") + +class VXorB32(CommonInstruction): + def __init__(self, dst, src0, src1, sdwa: Optional[SDWAModifiers]=None, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src0, src1], sdwa, None, comment) + self.setInst("v_xor_b32") + +# V Convert +class VCvtF16toF32(VCvtInstruction): + def __init__(self, dst, src, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(CvtType.CVT_F16_to_F32, dst, src, sdwa, comment) + self.setInst("v_cvt_f32_f16") + +class VCvtF32toF16(VCvtInstruction): + def __init__(self, dst, src, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(CvtType.CVT_F32_to_F16, dst, src, sdwa, comment) + self.setInst("v_cvt_f16_f32") + +class VCvtF32toU32(VCvtInstruction): + def __init__(self, dst, src, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(CvtType.CVT_F32_to_U32, dst, src, sdwa, comment) + self.setInst("v_cvt_u32_f32") + +class VCvtU32toF32(VCvtInstruction): + def __init__(self, dst, src, sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(CvtType.CVT_U32_to_F32, dst, src, sdwa, comment) + self.setInst("v_cvt_f32_u32") + +# V Mask +class VCndMaskB32(CommonInstruction): + def __init__(self, dst, src0, src1, src2 = VCC(), sdwa: Optional[SDWAModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src0, src1, src2], sdwa, None, comment) + self.setInst("v_cndmask_b32") + +# V Shift +class VLShiftLeftB16(CommonInstruction): + def __init__(self, dst, shiftHex, src, comment="") -> None: + super().__init__(InstType.INST_B16, dst, [shiftHex, src], None, None, comment) + self.setInst("v_lshlrev_b16") + +class VLShiftLeftB32(CommonInstruction): + def __init__(self, dst, shiftHex, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [shiftHex, src], None, None, comment) + self.setInst("v_lshlrev_b32") + +class VLShiftRightB32(CommonInstruction): + def __init__(self, dst, shiftHex, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [shiftHex, src], None, None, comment) + self.setInst("v_lshrrev_b32") + +class VLShiftLeftB64(CommonInstruction): + def __init__(self, dst, shiftHex, src, comment="") -> None: + super().__init__(InstType.INST_B64, dst, [shiftHex, src], None, None, comment) + self.setInst("v_lshlrev_b64") + +class VLShiftRightB64(CommonInstruction): + def __init__(self, dst, shiftHex, src, comment="") -> None: + super().__init__(InstType.INST_B64, dst, [shiftHex, src], None, None, comment) + self.setInst("v_lshrrev_b64") + +class _VLShiftLeftOrB32(CommonInstruction): + def __init__(self, dst, shiftHex, src0, src1, comment): + super().__init__(InstType.INST_B32, dst, [src0, shiftHex, src1], comment=comment) + self.setInst("v_lshl_or_b32") + +# V Arith Shift +# D.i = signext(S1.i) >> S0.i[4:0] +class VAShiftRightI32(CommonInstruction): + def __init__(self, dst, shiftHex, src, comment): + super().__init__(InstType.INST_I32, dst, [shiftHex, src], comment=comment) + self.setInst("v_ashrrev_i32") + +# V Shift + Logic +class VLShiftLeftOrB32(CompositeInstruction): + def __init__(self, dst, shiftHex, src0, src1, comment: str=""): + super().__init__(InstType.INST_B32, dst, [shiftHex, src0, src1], comment) + + @property + def shift(self): + assert isinstance(self.srcs, List) + return self.srcs[0] + + def setupInstructions(self): + super().setupInstructions() + assert isinstance(self.srcs, List) + if self.asmCaps["HasLshlOr"]: + self.instructions = [_VLShiftLeftOrB32(self.dst, self.shift, self.srcs[1], self.srcs[2], self.comment)] + else: + self.instructions = [VLShiftLeftB32(self.dst, self.shift, self.srcs[1], self.comment), \ + VOrB32(self.dst, self.dst, self.srcs[2])] + + assert all(inst.vop3 is None for inst in self.instructions), "Currently does not support with vop3 enabled" + +# V Add + Shift +class _VAddLShiftLeftU32(CommonInstruction): + def __init__(self, dst, shiftHex, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1, shiftHex], None, None, comment) + self.setInst("v_add_lshl_u32") + +class VAddLShiftLeftU32(CompositeInstruction): + def __init__(self, dst, shiftHex, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1, shiftHex], comment) + self.setInst("v_add_lshl_u32") + + @property + def shift(self): + assert isinstance(self.srcs, List) + return self.srcs[2] + + def setupInstructions(self): + super().setupInstructions() + assert isinstance(self.srcs, List) + if self.asmCaps["HasAddLshl"]: + self.instructions = [_VAddLShiftLeftU32(self.dst, self.shift, self.srcs[0], self.srcs[1], self.comment)] + else: + if self.asmBugs["ExplicitCO"]: + vadd = VAddCCOU32(self.dst, VCC(), self.srcs[0], self.srcs[1], self.comment) + else: + vadd = VAddU32(self.dst, self.srcs[0], self.srcs[1], self.comment) + self.instructions = [vadd, VLShiftLeftB32(self.dst, self.shift, self.dst, self.comment)] + + assert all(inst.vop3 is None for inst in self.instructions), "Currently does not support with vop3 enabled" + +class _VLShiftLeftAddU32(CommonInstruction): + def __init__(self, dst, shiftHex, src0, src1, vop3: Optional[VOP3PModifiers] = None, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1, shiftHex], None, vop3, comment) + self.setInst("v_lshl_add_u32") + +class VLShiftLeftAddU32(CompositeInstruction): + def __init__(self, dst, shiftHex, src0, src1, comment: str=""): + super().__init__(InstType.INST_U32, dst, [src0, src1, shiftHex], comment) + + @property + def shift(self): + assert isinstance(self.srcs, List) + return self.srcs[2] + + def setupInstructions(self): + super().setupInstructions() + assert isinstance(self.srcs, List) + if self.asmCaps["HasAddLshl"]: + self.instructions = [_VLShiftLeftAddU32(self.dst, self.shift, self.srcs[0], self.srcs[1], comment=self.comment)] + else: + if self.asmBugs["ExplicitCO"]: + inst = VAddCOU32(self.dst, VCC(), self.srcs[0], self.srcs[1]) + else: + inst = VAddU32(self.dst, self.srcs[0], self.srcs[1]) + self.instructions = [VLShiftLeftB32(self.dst, self.shift, self.dst, self.comment), inst] + + assert all(inst.vop3 is None for inst in self.instructions), "Currently does not support with vop3 enabled" + +# V Mov +class VMovB32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src], None, None, comment) + self.setInst("v_mov_b32") + +# V Bfe +class VBfeI32(CommonInstruction): + def __init__(self, dst, src0, src1, src2, comment="") -> None: + super().__init__(InstType.INST_I32, dst, [src0, src1, src2], None, None, comment) + self.setInst("v_bfe_i32") + +# D.u = (S0.u >> S1.u[4:0]) & ((1 << S2.u[4:0]) - 1) +class VBfeU32(CommonInstruction): + def __init__(self, dst, src0, src1, src2, comment="") -> None: + super().__init__(InstType.INST_U32, dst, [src0, src1, src2], None, None, comment) + self.setInst("v_bfe_u32") + +# V Bfi +# D.u = (S0.u & S1.u) | (~S0.u & S2.u) +class VBfiB32(CommonInstruction): + def __init__(self, dst, src0, src1, src2, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src0, src1, src2], None, None, comment) + self.setInst("v_bfi_b32") + +# V Pack +class VPackF16toB32(CommonInstruction): + def __init__(self, dst, src0, src1, vop3: Optional[VOP3PModifiers]=None, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src0, src1], None, vop3, comment) + self.setInst("v_pack_b32_f16") + +# Read/ Write +class VAccvgprReadB32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src], None, None, comment) + self.setInst("v_accvgpr_read_b32") + +class VAccvgprWrite(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_NOTYPE, dst, [src], None, None, comment) + self.setInst("v_accvgpr_write") + +class VAccvgprWriteB32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src], None, None, comment) + self.setInst("v_accvgpr_write_b32") + +class VReadfirstlaneB32(CommonInstruction): + def __init__(self, dst, src, comment="") -> None: + super().__init__(InstType.INST_B32, dst, [src], None, None, comment) + self.setInst("v_readfirstlane_b32") diff --git a/tensilelite/Tensile/TensileInstructions/Macros.py b/tensilelite/Tensile/TensileInstructions/Macros.py new file mode 100644 index 0000000000..71f062937a --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/Macros.py @@ -0,0 +1,85 @@ +################################################################################ +# Copyright 2019-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Code import Macro, Module +from .Containers import VCC +from .Instructions import SAndB32, SAndB64, VAddCOU32, VAddU32, \ + VCmpGEU32, VCmpLeU32, VCmpNeI32, VCndMaskB32, \ + VCvtF32toU32, VCvtU32toF32, VMulF32, \ + VMulHIU32, VMulLOU32, VRcpF32, VLShiftRightB32, \ + VLShiftRightB64, VSubCoU32 + +# Performs a division using 'magic number' computed on host +# Argument requirements: +# - dstIdx must be two consecutive registers ; on exit the lower one will contain the quotient. The upper is used as a temp. +# - First parm is passed as an integer vgpr index ; remaining are vgpr or sgpr symbolic names +# - dstIdx+1 cannot be same as dividend. dividend+0 can be same as dividend and this may be useful for chaining divides. +def MacroVMagicDiv(magicDivAlg) -> Module: + module = Module("defineMagicDivMacros") + module.addComment1("Magic div and mod functions") + macro = Macro("V_MAGIC_DIV", "dstIdx:req", "dividend:req", "magicNumber:req", "magicShift:req", "magicA:req") + if magicDivAlg==1: # TODO: remove me + macro.add(VMulHIU32(dst="v[\\dstIdx+1]", src0="\\dividend", src1="\\magicNumber")) + macro.add(VMulLOU32(dst="v[\\dstIdx+0]", src0="\\dividend", src1="\\magicNumber")) + macro.add(VLShiftRightB64(dst="v[\\dstIdx:\\dstIdx+1]", shiftHex="\\magicShift", src="v[\\dstIdx:\\dstIdx+1]")) + elif magicDivAlg==2: + macro.add(VMulHIU32(dst="v[\\dstIdx+1]", src0="\\dividend", src1="\\magicNumber")) + macro.add(VMulLOU32(dst="v[\\dstIdx+0]", src0="\\dividend", src1="\\magicA")) + macro.add(VAddU32(dst="v[\\dstIdx+0]", src0="v[\\dstIdx+0]", src1="v[\\dstIdx+1]")) + macro.add(VLShiftRightB32(dst="v[\\dstIdx+0]", shiftHex="\\magicShift", src="v[\\dstIdx+0]")) + module.add(macro) + return module + +def MacroVDynamicScalarDiv(wavefrontSize) -> Module: + module = Module("Dynamic scalar divide macros") + module.addComment1("Dynamic Scalar Divide: vQuotient=vDividend/vDivisor; vRemainder=vDividend%vDivisor;") + macro = Macro("DYNAMIC_VECTOR_DIVIDE", "vQuotient", "vRemainder", "vDividend", "vDivisor", "vTmp0", "vTmp1", "sTmp") + sTmpStr = "s[\\sTmp]" if (wavefrontSize == 32) else "s[\\sTmp:\\sTmp+1]" + macro.add(VCvtU32toF32(dst="v[\\vQuotient]", src="v[\\vDivisor]")) + macro.add(VRcpF32(dst="v[\\vQuotient]", src="v[\\vQuotient]")) + macro.add(VMulF32(dst="v[\\vQuotient]", src0=hex(0x4f800000), src1="v[\\vQuotient]")) + macro.add(VCvtF32toU32(dst="v[\\vQuotient]", src="v[\\vQuotient]")) + macro.add(VMulLOU32(dst="v[\\vRemainder]", src0="v[\\vDivisor]", src1="v[\\vQuotient]")) + macro.add(VMulHIU32(dst="v[\\vTmp0]", src0="v[\\vDivisor]", src1="v[\\vQuotient]")) + macro.add(VSubCoU32(dst="v[\\vTmp1]", dst1=VCC(), src0=hex(0), src1="v[\\vRemainder]")) + macro.add(VCmpNeI32(dst=sTmpStr, src0=hex(0), src1="v[\\vTmp0]")) + macro.add(VCndMaskB32(dst="v[\\vRemainder]", src0="v[\\vTmp1]", src1="v[\\vRemainder]", src2=sTmpStr)) # type: ignore + macro.add(VMulHIU32(dst="v[\\vRemainder]", src0="v[\\vRemainder]", src1="v[\\vQuotient]")) + macro.add(VSubCoU32(dst="v[\\vTmp0]", dst1=VCC(), src0="v[\\vQuotient]", src1="v[\\vRemainder]")) + macro.add(VAddCOU32(dst="v[\\vQuotient]", dst1=VCC(), src0="v[\\vQuotient]", src1="v[\\vRemainder]")) + macro.add(VCndMaskB32(dst="v[\\vQuotient]", src0="v[\\vQuotient]", src1="v[\\vTmp0]", src2=sTmpStr)) # type: ignore + macro.add(VMulHIU32(dst="v[\\vQuotient]", src0="v[\\vQuotient]", src1="v[\\vDividend]")) + macro.add(VMulLOU32(dst="v[\\vRemainder]", src0="v[\\vQuotient]", src1="v[\\vDivisor]")) + macro.add(VSubCoU32(dst="v[\\vTmp0]", dst1=VCC(), src0="v[\\vDividend]", src1="v[\\vRemainder]")) + macro.add(VCmpGEU32(dst=sTmpStr, src0="v[\\vDividend]", src1="v[\\vRemainder]")) + macro.add(VAddCOU32(dst="v[\\vRemainder]", dst1=VCC(), src0=hex(1), src1="v[\\vQuotient]")) + macro.add(VAddCOU32(dst="v[\\vTmp1]", dst1=VCC(), src0=-1, src1="v[\\vQuotient]")) + macro.add(VCmpLeU32(dst=VCC(), src0="v[\\vDivisor]", src1="v[\\vTmp0]")) + SAndBX = SAndB32 if wavefrontSize == 32 else SAndB64 + macro.add(SAndBX(dst=VCC(), src0=sTmpStr, src1=VCC())) + macro.add(VCndMaskB32(dst="v[\\vQuotient]", src0="v[\\vQuotient]", src1="v[\\vRemainder]", src2=VCC())) + macro.add(VCndMaskB32(dst="v[\\vQuotient]", src0="v[\\vTmp1]", src1="v[\\vQuotient]", src2=sTmpStr)) # type: ignore + macro.add(VCmpNeI32(dst=VCC(), src0=hex(0), src1="v[\\vDivisor]")) + macro.add(VCndMaskB32(dst="v[\\vQuotient]", src0=-1, src1="v[\\vQuotient]", src2=VCC(), comment="final result" )) + macro.add(VMulLOU32(dst="v[\\vRemainder]", src0="v[\\vQuotient]", src1="v[\\vDivisor]")) + macro.add(VSubCoU32(dst="v[\\vRemainder]", dst1=VCC(), src0="v[\\vDividend]", src1="v[\\vRemainder]", comment="final result" )) + module.add(macro) + return module diff --git a/tensilelite/Tensile/TensileInstructions/Math.py b/tensilelite/Tensile/TensileInstructions/Math.py new file mode 100644 index 0000000000..9c6154ba02 --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/Math.py @@ -0,0 +1,299 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from typing import Optional +from .Code import Module +from .Containers import RegisterContainer +from .ExtInstructions import SMulInt64to32 +from .Instructions import * +from .RegisterPool import RegisterPoolResource +from .Utils import vgpr, sgpr, log2 + +######################################## +# Divide & Remainder +# quotient register, remainder register, dividend register, divisor, tmpVgprx2 +######################################## + +def vectorStaticDivideAndRemainder(qReg, rReg, dReg, divisor, tmpVgprRes: Optional[RegisterPoolResource], doRemainder=True, comment=""): + dComment = "%s = %s / %s" % (vgpr(qReg), vgpr(dReg), divisor) if (comment=="") else comment + rComment = "%s = %s %% %s" % (vgpr(rReg), vgpr(dReg), divisor) if (comment=="") else comment + + module = Module("vectorStaticDivideAndRemainder") + if ((divisor & (divisor - 1)) == 0): # pow of 2 + divisor_log2 = log2(divisor) + module.add(VLShiftRightB32(dst=vgpr(qReg), shiftHex=divisor_log2, src=vgpr(dReg), comment=dComment)) + if doRemainder: + module.add(VAndB32(dst=vgpr(rReg), src0=(divisor-1), src1=vgpr(dReg), comment=rComment)) + else: + assert tmpVgprRes and tmpVgprRes.size >= 2 + tmpVgpr = tmpVgprRes.idx + """ + if divisor == 30: + shift = 32+2 + elif divisor >= 14: + shift = 32+4 + elif divisor >= 7: + shift = 32+3 + elif divisor >= 6: + shift = 32+2 # this was 32+3 but divisor hex didn't fit into 32 bits + elif divisor >= 5: + shift = 32+2 + elif divisor >= 3: + shift = 32+1 + """ + shift = 32+1 + magic = ((2**shift) // divisor) + 1 + if magic <= 64 and magic >= -16: + module.add(VMulHIU32(dst=vgpr(tmpVgpr+1), src0=vgpr(dReg), src1=hex(magic), comment=dComment)) + module.add(VMulLOU32(dst=vgpr(tmpVgpr+0), src0=vgpr(dReg), src1=hex(magic), comment=dComment)) + else: + module.add(VMovB32(dst=vgpr(tmpVgpr+0), src=hex(magic))) + module.add(VMulHIU32(dst=vgpr(tmpVgpr+1), src0=vgpr(dReg), src1=vgpr(tmpVgpr+0), comment=dComment)) + module.add(VMulLOU32(dst=vgpr(tmpVgpr+0), src0=vgpr(dReg), src1=vgpr(tmpVgpr+0), comment=dComment)) + module.add(VLShiftRightB64(dst=vgpr(tmpVgpr,2), shiftHex=hex(shift), src=vgpr(tmpVgpr,2), comment=dComment)) + module.add(VMovB32(dst=vgpr(qReg), src=vgpr(tmpVgpr), comment=dComment)) + if doRemainder: + module.add(VMulLOU32(dst=vgpr(tmpVgpr), src0=vgpr(qReg), src1=hex(divisor), comment=rComment)) + module.add(VSubU32(dst=vgpr(rReg), src0=vgpr(dReg), src1=vgpr(tmpVgpr), comment=rComment)) + return module + +def vectorStaticDivide(qReg, dReg, divisor, tmpVgprRes: Optional[RegisterPoolResource], comment=""): + rReg = -1 # unused + module = vectorStaticDivideAndRemainder(qReg, rReg, dReg, divisor, tmpVgprRes, False, comment) + module.name = "vectorStaticDivide (reg=-1)" + return module + +def vectorStaticRemainder(qReg, rReg, dReg, divisor, tmpVgprRes: Optional[RegisterPoolResource], \ + tmpSgprRes: Optional[RegisterPoolResource], comment=""): + if comment == "": + comment = "%s = %s %% %s" % (vgpr(rReg), vgpr(dReg), divisor) + + module = Module("vectorStaticRemainder") + if ((divisor & (divisor - 1)) == 0): # pow of 2 + module.add(VAndB32(dst=vgpr(rReg), src0=(divisor-1), src1=vgpr(dReg), comment=comment)) + else: + assert tmpVgprRes and tmpVgprRes.size >= 2 + tmpVgpr = tmpVgprRes.idx + assert tmpSgprRes and tmpSgprRes.size >= 1 + tmpSgpr = tmpSgprRes.idx + """ + if divisor == 30: + shift = 32+2 + elif divisor >= 14: + shift = 32+4 + elif divisor >= 7: + shift = 32+3 + elif divisor >= 6: + shift = 32+2 # this was 32+3 but divisor hex didn't fit into 32 bits + elif divisor >= 5: + shift = 32+2 + elif divisor >= 3: + shift = 32+1 + """ + shift = 32+1 + magic = ((2**shift) // divisor) + 1 + if magic <= 64 and magic >= -16: + module.add(VMulHIU32(dst=vgpr(tmpVgpr+1), src0=vgpr(dReg), src1=hex(magic), comment=comment)) + module.add(VMulLOU32(dst=vgpr(tmpVgpr+0), src0=vgpr(dReg), src1=hex(magic), comment=comment)) + else: + module.add(SMovB32(dst=sgpr(tmpSgpr), src=hex(magic), comment=comment)) + module.add(VMulHIU32(dst=vgpr(tmpVgpr+1), src0=vgpr(dReg), src1=sgpr(tmpSgpr), comment=comment)) + module.add(VMulLOU32(dst=vgpr(tmpVgpr+0), src0=vgpr(dReg), src1=sgpr(tmpSgpr), comment=comment)) + module.add(VLShiftRightB64(dst=vgpr(tmpVgpr,2), shiftHex=hex(shift), src=vgpr(tmpVgpr,2), comment=comment)) + module.add(VMovB32(dst=vgpr(qReg), src=vgpr(tmpVgpr), comment=comment)) + if divisor <= 64 and divisor >= -16: + module.add(VMulLOU32(dst=vgpr(tmpVgpr), src0=vgpr(qReg), src1=hex(divisor), comment=comment)) + else: + module.add(SMovB32(dst=sgpr(tmpSgpr), src=hex(divisor), comment=comment)) + module.add(VMulLOU32(dst=vgpr(tmpVgpr), src0=vgpr(qReg), src1=sgpr(tmpSgpr), comment=comment)) + module.add(VSubU32(dst=vgpr(rReg), src0=vgpr(dReg), src1=vgpr(tmpVgpr), comment=comment)) + return module + +# only used for loop unroll and GlobalSplitU +# doRemainder==0 : compute quotient only +# doRemainder==1 : compute quotient and remainder +# doRemainder==2 : only compute remainder (not quotient unless required for remainder) +# dreg == dividend +# tmpSgpr must be 2 SPGRs +# qReg and dReg can be "sgpr[..]" or names of sgpr (will call sgpr) +def scalarStaticDivideAndRemainder(qReg, rReg, dReg, divisor, tmpSgprRes: Optional[RegisterPoolResource], \ + doRemainder=1): + + qRegSgpr = qReg if isinstance(qReg, RegisterContainer) and qReg.regType == 's' else sgpr(qReg) + + dRegSgpr = dReg if isinstance(dReg, RegisterContainer) and dReg.regType == 's' else sgpr(dReg) + + module = Module("scalarStaticDivideAndRemainder") + if ((divisor & (divisor - 1)) == 0): # pow of 2 + divisor_log2 = log2(divisor) + if doRemainder != 2: + module.add(SLShiftRightB32(dst=qRegSgpr, shiftHex=divisor_log2, src=dRegSgpr, \ + comment="%s = %s / %u"%(qRegSgpr, dRegSgpr, divisor))) + if doRemainder: + module.add(SAndB32(dst=sgpr(rReg), src0=(divisor-1), src1=dRegSgpr, \ + comment="%s = %s %% %u"%(sgpr(rReg), dRegSgpr, divisor))) + else: + assert tmpSgprRes and tmpSgprRes.size >= 2 + tmpSgpr = tmpSgprRes.idx + assert qReg != tmpSgpr + """ + if divisor == 30: + shift = 32+2 + elif divisor >= 14: + shift = 32+4 + elif divisor >= 6: + shift = 32+3 + elif divisor >= 5: + shift = 32+2 + elif divisor >= 3: + shift = 32+1 + """ + shift = 32+1 + magic = ((2**shift) // divisor) + 1 + magicHi = magic // (2**16) + magicLo = magic & (2**16-1) + + module.add(SMovB32(dst=sgpr(tmpSgpr+1), src=hex(0), comment="STATIC_DIV: divisior=%s"%divisor)) + module.add(SMulI32(dst=sgpr(tmpSgpr+0), src0=hex(magicHi), src1=dRegSgpr, comment="tmp1 = dividend * magic hi")) + module.add(SLShiftLeftB64(dst=sgpr(tmpSgpr,2), shiftHex=hex(16), src=sgpr(tmpSgpr,2), comment="left shift 16 bits")) + module.add(SMulI32(dst=qRegSgpr, src0=dRegSgpr, src1=hex(magicLo), comment="tmp0 = dividend * magic lo")) + module.add(SAddU32(dst=sgpr(tmpSgpr+0), src0=qRegSgpr, src1=sgpr(tmpSgpr+0), comment="add lo")) + module.add(SAddCU32(dst=sgpr(tmpSgpr+1), src0=sgpr(tmpSgpr+1), src1=hex(0), comment="add hi")) + module.add(SLShiftRightB64(dst=sgpr(tmpSgpr,2), shiftHex=hex(shift), src=sgpr(tmpSgpr,2), comment="tmp1 = (dividend * magic) << shift")) + module.add(SMovB32(dst=qRegSgpr, src=sgpr(tmpSgpr), comment="quotient")) + if doRemainder: + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=qRegSgpr, src1=hex(divisor), comment="quotient*divisor")) + module.add(SSubU32(dst=sgpr(rReg), src0=dRegSgpr, src1=sgpr(tmpSgpr), comment="rReg = dividend - quotient*divisor")) + return module + +######################################## +# Scalar Magic Div +# product register, operand register, multiplier +######################################## + +# dividend is a symbol (constant or sgpr). Used directly not inside automatic sgpr(..) +# dst is 2 consecutive SGPR +# result returned in dst0. dst1 is used as a temp, +# dst[1] cannot be same as divident, dst[0] can be same as dividend and this can be useful +def scalarMagicDivExplicit(dst, dividend, magicNumber, magicAbit, magicShift): + module = Module("scalarMagicDivExplicit") + module.addComment1("dst1:0 = dividend(%s) / magicTag(%s)" % (dividend, magicNumber)) + module.add(SMulHIU32(dst=sgpr(dst+1), src0=dividend, src1=sgpr(magicNumber), comment="scalar magic div (magicnum)")) + module.add(SMulI32(dst=sgpr(dst+0), src0=dividend, src1=sgpr(magicAbit), comment="scalar magic div (abit)")) + module.add(SAddU32(dst=sgpr(dst+0), src0=sgpr(dst+0), src1=sgpr(dst+1), comment="scalar magic div (combine)")) + module.add(SLShiftRightB32(dst=sgpr(dst+0), shiftHex=sgpr(magicShift), src=sgpr(dst+0), \ + comment="scalar magic div (shift), quotient in s%s"%dst)) + return module + +def scalarMagicDiv(dst, dividend, magicTag): + return scalarMagicDivExplicit(dst, dividend, + magicNumber="MagicNumberSize"+magicTag, + magicAbit="MagicAbitSize"+magicTag, + magicShift="MagicShiftSize"+magicTag) + +############################################################################## +# Perform a magic division (mul by magic number and shift) +# dest is two consec SGPR, used for intermediate temp as well as final result +# result quotient returned in sgpr(dest,1) +# tmpVgpr: Size 2 +############################################################################## +def sMagicDiv(dest, hasSMulHi, dividend, magicNumber, magicShift, tmpVgpr): + module = Module("sMagicDiv") + module.addModuleAsFlatItems(SMulInt64to32(hasSMulHi, \ + sgpr(dest), sgpr(dest+1), dividend, magicNumber, \ + False, tmpVgpr, "s_magic mul")) + module.add(SLShiftRightB64(dst=sgpr(dest,2), shiftHex=magicShift, src=sgpr(dest,2), comment="sMagicDiv")) + return module + +############################################################################## +# Perform a sgpr version of magic division algo 2 (mul by magic number, Abit and shift) +# dest is three consec SGPR, used for intermediate temp as well as final result +# result quotient returned in sgpr(dest,1) +############################################################################## +def sMagicDivAlg2(dest, dividend, magicNumber, magicShiftAbit): + # dest+0: q, + # dest+1: intermediate for magic div + # dest+2: A tmpS to store the 'Abit' and the final Shift (use tmpS to save sgpr) + tmpS = dest+2 + + module = Module("sMagicDivAlg2") + module.add(SMulHIU32(dst=sgpr(dest+1), src0=dividend, src1=magicNumber, comment=" s_magic mul, div alg 2")) + module.add(SLShiftRightB32(dst=sgpr(tmpS), shiftHex=31, src=magicShiftAbit, comment=" tmpS = extract abit")) # tmpS = MagicAbit + module.add(SMulI32(dst=sgpr(dest), src0=dividend, src1=sgpr(tmpS), comment=" s_magic mul, div alg 2")) + module.add(SAddU32(dst=sgpr(dest), src0=sgpr(dest), src1=sgpr(dest+1), comment="")) + + module.add(SAndB32(dst=sgpr(tmpS), src0=magicShiftAbit, src1=hex(0x7fffffff), comment=" tmpS = remove abit to final shift")) # tmpS = MagicShift + module.add(SLShiftRightB32(dst=sgpr(dest), shiftHex=sgpr(tmpS), src=sgpr(dest), comment=" sMagicDiv Alg 2")) + return module + +######################################## +# Multiply +# product register, operand register, multiplier +######################################## + +def staticMultiply(product, operand, multiplier, tmpSgprRes: Optional[RegisterPoolResource], comment=""): + if comment == "": + comment = "%s = %s * %s" % (product, operand, multiplier) + + module = Module("staticMultiply") + if multiplier == 0: + module.add(VMovB32(dst=product, src=hex(multiplier), comment=comment)) + elif ((multiplier & (multiplier - 1)) == 0): # pow of 2 + multiplier_log2 = log2(multiplier) + if multiplier_log2==0 and product == operand: + module.addCommentAlign(comment + " (multiplier is 1, do nothing)") + else: + module.add(VLShiftLeftB32(dst=product, shiftHex=hex(multiplier_log2), src=operand, comment=comment)) + else: + if multiplier <= 64 and multiplier >= -16: + module.add(VMulLOU32(dst=product, src0=hex(multiplier), src1=operand, comment=comment)) + else: + assert tmpSgprRes and tmpSgprRes.size >= 1 + tmpSgpr = tmpSgprRes.idx + module.add(SMovB32(dst=sgpr(tmpSgpr), src=hex(multiplier), comment=comment)) + module.add(VMulLOU32(dst=product, src0=sgpr(tmpSgpr), src1=operand, comment=comment)) + return module + + +######################################## +# Multiply scalar for 64bit +# product register, operand register, multiplier +######################################## + +def scalarStaticMultiply(product, operand, multiplier, tmpSgpr=None, comment=""): + module = Module("scalarStaticMultiply") + if comment == "": + comment = "%s = %s * %s" % (product, operand, multiplier) + + if multiplier == 0: + module.add(SMovB64(dst=product, src=hex(multiplier), comment=comment)) + return module + + # TODO- to support non-pow2, need to use mul_32 and mul_hi_32 ? + assert ((multiplier & (multiplier - 1)) == 0) # assert pow of 2 + + multiplier_log2 = log2(multiplier) + if multiplier_log2==0 and product == operand: + module.addCommentAlign(comment + " (multiplier is 1, do nothing)") + else: + # notice that the src-order of s_lshl_b64 is different from v_lshlrev_b32. + module.add(SLShiftLeftB64(dst=product, shiftHex=hex(multiplier_log2), src=operand, comment=comment)) + return module diff --git a/tensilelite/Tensile/TensileInstructions/Pass.py b/tensilelite/Tensile/TensileInstructions/Pass.py new file mode 100644 index 0000000000..7a5d235fd7 --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/Pass.py @@ -0,0 +1,208 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Base import Item +from .Code import KernelBody, Label, Macro, Module, RegSet, TextBlock +from .Containers import RegisterContainer +from .Instructions import BranchInstruction, CommonInstruction, Instruction, \ + CompositeInstruction, MacroInstruction, \ + ReadWriteInstruction, SEndpgm, SMovB32, \ + _SWaitCnt, _SWaitCntVscnt, SSleep, SBarrier, SNop +from .Formatting import slash50 + +def TensileInstructionsPass(kernelBody: KernelBody): + module = kernelBody.body + assignDict = getAssignmentDict(module) + compositeToInstruction(module) + maxVgpr = kernelBody.totalVgprs + maxSgpr = kernelBody.totalSgprs + graph = buildGraph(module, maxVgpr, maxSgpr, assignDict) + removeDuplicateAssignment(graph) + +####################################### +# Setup +####################################### + +def compositeToInstruction(module): + itemList = [] + for item in module.items(): + if isinstance(item, CompositeInstruction): + items = item.getInstructions() + itemList.extend(items) + continue # Skip appending composite instruction back to list + elif isinstance(item, Module) or isinstance(item, Macro): + compositeToInstruction(item) + itemList.append(item) + module.setItems(itemList) + +def getAssignmentDict(module): + assignmentDict = dict() + _getAssignmentDictIter(module, assignmentDict) + return assignmentDict + +def buildGraph(module, vgprMax, sgprMax, assignmentDict): + graph = dict() + graph["v"] = [[] for _ in range(vgprMax)] + graph["s"] = [[] for _ in range(sgprMax)] + graph["m"] = [[] for _ in range(1)] + _recordGraph(module, graph, assignmentDict) + return graph + +def removeDuplicateAssignment(graph): + _removeDuplicateAssignmentGPR(graph, "s") + +# No opt item container class +class NoOptItem: + def __init__(self, item) -> None: + self._item = item + + def getItem(self) -> Item: + return self._item + +################################################################################ +################################################################################ +### +### Internal Functions +### +################################################################################ +################################################################################ + +def _getAssignmentDictIter(module, assignmentDict): + for item in module.items(): + if isinstance(item, Module): + _getAssignmentDictIter(item, assignmentDict) + elif isinstance(item, RegSet): + num = 0 + if item.ref: + num = assignmentDict[item.ref] + item.offset + else: + num = item.value + assignmentDict[item.name] = num + +def _addRegToGraph(item, assignmentDict, params: list, graph, noOpt): + for p in params: + if isinstance(p, RegisterContainer): + _setName2RegNum(p, assignmentDict) + if p.regType == "acc": + continue + for i in range(p.regIdx, p.regIdx + p.regNum): + if graph[p.regType][i] and graph[p.regType][i][-1] == item: + continue + # print("[%s] Index %d %d" %(p.regType, i, len(graph[p.regType]))) + if noOpt: + graph[p.regType][i].append(NoOptItem(item)) + else: + graph[p.regType][i].append(item) + +def _recordGraph(module, graph, assignmentDict): + for item in module.items(): + if isinstance(item, Module): + _recordGraph(item, graph, assignmentDict) + elif isinstance(item, (CommonInstruction, ReadWriteInstruction, MacroInstruction)): + _addRegToGraph(item, assignmentDict, item.getParams(), graph, module.isNoOpt()) + elif isinstance(item, (BranchInstruction, Label, _SWaitCnt, _SWaitCntVscnt, \ + SEndpgm, SBarrier, SNop, SSleep)): + for i in range(len(graph["v"])): + graph["v"][i].append(item) + for i in range(len(graph["s"])): + graph["s"][i].append(item) + +# Currently only removes s_mov_b32, does not support 2 sgpr at lvalue +def _removeDuplicateAssignmentGPR(graph, regType): + for idx, sList in enumerate(graph[regType]): + assignValue = None + newList = [] + for item in sList: + isRemoved = False + if isinstance(item, (NoOptItem, BranchInstruction, Label, MacroInstruction, \ + _SWaitCnt, _SWaitCntVscnt)): + assignValue = None + # FIXME: Need refactor. + elif isinstance(item, SMovB32): + gpr = item.dst + gprValue = item.src + if gpr.regIdx == idx and gprValue == assignValue: + if item.comment: + comment = item.comment + " (dup assign opt.)" + newItem = TextBlock(slash50(comment)) + module = item.parent + module.replaceItem(item, newItem) # type: ignore + else: + module = item.parent + module.removeItem(item) # type: ignore + isRemoved = True + assignValue = gprValue + elif isinstance(item, Instruction): + params = item.getParams() + if len(params) > 1: + gpr = params[0] + if isinstance(gpr, RegisterContainer) and (gpr.regType == regType): + for i in range(gpr.regIdx, gpr.regIdx + gpr.regNum): + if i == idx: + assignValue = None + break + if not isRemoved: + newList.append(item) + + if len(newList) != len(sList): + graph["s"][idx] = newList + +################################################################################ +################################################################################ +### +### Helper Functions +### +################################################################################ +################################################################################ + +# Find ".set AAAAA 0" and convert "s[AAAAA]" into "s0" +def _setName2RegNum(gpr, assignmentDict): + assert(isinstance(gpr, RegisterContainer)) + if gpr.regIdx == None and gpr.regName: + name = gpr.getRegNameWithType() + num = assignmentDict[name] + gpr.regName.getTotalOffsets() + gpr.regIdx = num + RegNumList = [] + for i in range(0, gpr.regNum): + RegNumList.append(i + gpr.regIdx) + return RegNumList + +def _graphDebugSaveToTxt(graph, kernelName): + f = open('%s.txt' % kernelName, 'w') + f.write("VGPR\n") + i = 0 + for d in graph["v"]: + f.write("[%d]\n" % i) + for dd in d: + ss = str(dd) + f.write(ss) + f.write("\n") + i += 1 + i = 0 + f.write("SGPR\n") + for d in graph["s"]: + f.write("[%d]\n" % i) + for dd in d: + ss = str(dd) + f.write(ss) + f.write("\n") + i += 1 + f.close() diff --git a/tensilelite/Tensile/TensileInstructions/RegisterPool.py b/tensilelite/Tensile/TensileInstructions/RegisterPool.py new file mode 100644 index 0000000000..2eb6b73d1e --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/RegisterPool.py @@ -0,0 +1,360 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Code import Module +from .Formatting import print2, printExit, printWarning +from .Instructions import SMovB32, VMovB32 +from .Utils import vgpr, sgpr, roundUpToNearestMultiple + +from contextlib import contextmanager +from dataclasses import dataclass +from enum import Enum +from typing import Optional + +import traceback + +################################################################################ +# RegisterPool +# Debugging register performance problems: +# - Enable self.db["PrintRP"] to see messages as vgprPool state changes. +# - Search for 'overlow' to see when pool grows dynamically - typically this +# indicates growth for temps or other cases. +# - checkIn, checkout take optional tag but this is not widely used in tensile. +# - checkout returns vgpr index that was returned - can search disasm to see where +# this vgpr is used. +################################################################################ +class RegisterPool: + class Status(Enum): + Unavailable = 0 + Available = 1 + InUse = 2 + + class Register: + def __init__(self, status, tag): + self.status = status + self.tag = tag + + class ResourceOverflowException(Exception): + pass + + ######################################## + # Init + # defaultPreventOverflow: control behavior of checkout and checkoutAligned when preventOverflow is not explicitly specificed. + def __init__(self, size, type, defaultPreventOverflow, printRP=0): + self.printRP=printRP + self.type = type + self.defaultPreventOverflow = defaultPreventOverflow + self.pool = [self.Register(RegisterPool.Status.Unavailable, "init") for i in range(0,size)] + self.checkOutSize = {} + + ######################################## + # Adds registers to the pool so they can be used as temps + # Convenience function that takes a range and returns it in string form + def addRange(self, start, stop, tag=""): + self.add(start, stop-start+1, tag) + if (start == stop): + return "%d"%(start) + else: + return "%d-%d" % (start, stop) + + ######################################## + # Adds registers to the pool so they can be used as temps + # Add + def add(self, start, size, tag=""): + # reserve space + if self.printRP: + print("RP::add(%u..%u for '%s')"%(start,start+size-1,tag)) + newSize = start + size + oldSize = len(self.pool) + if newSize > oldSize: + for i in range(0, newSize-oldSize): + self.pool.append(self.Register(RegisterPool.Status.Unavailable,tag)) + # mark as available + for i in range(start, start+size): + if self.pool[i].status == RegisterPool.Status.Unavailable: + self.pool[i].status = RegisterPool.Status.Available + self.pool[i].tag = tag + elif self.pool[i].status == RegisterPool.Status.Available: + printWarning("RegisterPool::add(%u,%u) pool[%u](%s) already available" % (start, size, i, self.pool[i].tag)) + elif self.pool[i].status == RegisterPool.Status.InUse: + printWarning("RegisterPool::add(%u,%u) pool[%u](%s) already in use" % (start, size, i, self.pool[i].tag)) + else: + raise RuntimeError("RegisterPool::add(%u,%u) pool[%u](%s) = %s" % (start, size, i, self.pool[i].tag, self.pool[i].status)) + if self.printRP: + print(self.state()) + ######################################## + # Remove + # Removes registers from the pool so they cannot be subsequently allocated for tmps + def remove(self, start, size, tag=""): + if self.printRP: + print("RP::remove(%u..%u) for %s"%(start,size-1,tag)) + # reserve space + newSize = start + size + oldSize = len(self.pool) + if newSize > oldSize: + printWarning("RegisterPool::remove(%u,%u) but poolSize=%u" % (start, size, oldSize)) + # mark as unavailable + for i in range(start, start+size): + if self.pool[i].status == RegisterPool.Status.Available: + self.pool[i].status = RegisterPool.Status.Unavailable + elif self.pool[i].status == RegisterPool.Status.Unavailable: + printWarning("RegisterPool::remove(%u,%u) pool[%u](%s) already unavailable" % (start, size, i, self.pool[i].tag)) + elif self.pool[i].status == RegisterPool.Status.InUse: + printWarning("RegisterPool::remove(%u,%u) pool[%u](%s) still in use" % (start, size, i, self.pool[i].tag)) + else: + printExit("RegisterPool::remove(%u,%u) pool[%u](%s) = %s" % (start, size, i, self.pool[i].tag, self.pool[i].status)) + + ######################################## + # Check Out + def checkOut(self, size, tag="_untagged_", preventOverflow=-1): + return self.checkOutAligned(size, 1, tag, preventOverflow) + + def checkOutAligned(self, size, alignment, tag="_untagged_aligned_", preventOverflow=-1): + if preventOverflow == -1: + preventOverflow = self.defaultPreventOverflow + assert(size > 0) + found = -1 + for i in range(0, len(self.pool)): + # alignment + if i % alignment != 0: + continue + # enough space + if i + size > len(self.pool): + continue + # all available + allAvailable = True + for j in range(0, size): + if self.pool[i+j].status != RegisterPool.Status.Available: + allAvailable = False + i = j+1 + break + if allAvailable: + found = i + break + else: + continue + + # success without overflowing + if found > -1: + #print "Found: %u" % found + for i in range(found, found+size): + self.pool[i].status = RegisterPool.Status.InUse + self.pool[i].tag = tag + self.checkOutSize[found] = size + if self.printRP: + print("RP::checkOut '%s' (%u,%u) @ %u avail=%u"%(tag, size,alignment, found, self.available())) + #print self.state() + return found + # need overflow + else: + #print "RegisterPool::checkOutAligned(%u,%u) overflowing past %u" % (size, alignment, len(self.pool)) + # where does tail sequence of available registers begin + assert (not preventOverflow) + start = len(self.pool) + for i in range(len(self.pool)-1, 0, -1): + if self.pool[i].status == RegisterPool.Status.Available: + self.pool[i].tag = tag + start = i + continue + else: + break + #print "Start: ", start + # move forward for alignment + + start = roundUpToNearestMultiple(start,alignment) + #print "Aligned Start: ", start + # new checkout can begin at start + newSize = start + size + oldSize = len(self.pool) + overflow = newSize - oldSize + #print "Overflow: ", overflow + for i in range(start, len(self.pool)): + self.pool[i].status = RegisterPool.Status.InUse + self.pool[i].tag = tag + for i in range(0, overflow): + if len(self.pool) < start: + # this is padding to meet alignment requirements + self.pool.append(self.Register(RegisterPool.Status.Available,tag)) + else: + self.pool.append(self.Register(RegisterPool.Status.InUse,tag)) + self.checkOutSize[start] = size + if self.printRP: + print(self.state()) + print("RP::checkOut' %s' (%u,%u) @ %u (overflow)"%(tag, size, alignment, start)) + return start + + def initTmps(self, initValue, start=0, stop=-1): + module = Module("initTmps from RegisterPool") + stop= len(self.pool) if stop== -1 or stop>len(self.pool) else stop+1 + for i in range(start, stop): + #if self.type == 's': + # print i, self.pool[i].status + if self.pool[i].status==RegisterPool.Status.Available: + if self.type == 's': + module.add(SMovB32(dst=sgpr(i), src=hex(initValue), comment="init tmp in pool")) + elif self.type == 'v': + module.add(VMovB32(dst=vgpr(i), src=hex(initValue), comment="init tmp in pool")) + else: + assert(0) # bad regpool type + + return module + + ######################################## + # Check In + def checkIn(self, start): + if start in self.checkOutSize: + size = self.checkOutSize[start] + for i in range(start, start+size): + self.pool[i].status = RegisterPool.Status.Available + self.checkOutSize.pop(start) + if self.printRP: + print("RP::checkIn('%s') @ %u +%u"%(self.pool[start].tag, start,size)) + else: + if 0: + traceback.print_stack(None) + import pdb; pdb.set_trace() + printWarning("RegisterPool::checkIn('%s',%s) but it was never checked out"%(self.pool[start].tag, start)) + #traceback.print_stack(None) + + ######################################## + # Size + def size(self): + return len(self.pool) + + + ######################################## + # Number of available registers + def available(self): + numAvailable = 0 + for s in self.pool: + if s.status == RegisterPool.Status.Available: + numAvailable += 1 + return numAvailable + + ######################################## + # Size of registers of at least specified blockSize + def availableBlock(self, blockSize, align): + if blockSize ==0: + blockSize = 1 + blocksAvail = 0 + consecAvailable = 0 + #for s in self.pool: + for i in range(0, len(self.pool)): + s = self.pool[i] + if s.status == RegisterPool.Status.Available: + if not (consecAvailable == 0 and i % align != 0): + # do not increment if the first item is not aligned + consecAvailable += 1 + else: + blocksAvail += consecAvailable // blockSize + consecAvailable = 0 + blocksAvail += consecAvailable // blockSize + #print self.state() + #print "available()=", self.available(), "availableBlock()=",maxAvailable + return blocksAvail * blockSize + + def availableBlockAtEnd(self): + availCnt = 0 + for s in reversed(self.pool): + if s.status == RegisterPool.Status.Available: + availCnt += 1 + else: + break + + return availCnt + + + ######################################## + def checkFinalState(self): + for si in range(0,len(self.pool)): + if self.pool[si].status == RegisterPool.Status.InUse: + if self.printRP: + print(self.state()) + raise RuntimeError("RegisterPool::checkFinalState: temp (%s, '%s') was never checked in." \ + %(si, self.pool[si].tag)) + print2("total vgpr count: %u\n"%self.size()) + + ######################################## + # State + def state(self): + stateStr = "" + placeValues = [1000, 100, 10, 1] + for placeValueIdx in range(1, len(placeValues)): + placeValue = placeValues[placeValueIdx] + priorPlaceValue = placeValues[placeValueIdx-1] + if len(self.pool) >= placeValue: + pvs = "" # place value string + for i in range(0, len(self.pool)): + if i % placeValue==0: + pvs += "%u"%((i%priorPlaceValue)//placeValue) + else: + pvs += " " + stateStr += pvs + "\n" + for i in range(0, len(self.pool)): + if self.pool[i].status == RegisterPool.Status.Unavailable: + stateStr += "." # 'removed', this indicates a fixed assignment from "remove", ie a non-tmp allocation + elif self.pool[i].status == RegisterPool.Status.Available: + stateStr += "|" # Can be allocated + elif self.pool[i].status == RegisterPool.Status.InUse: + stateStr += "#" # Checked out + return stateStr + + def stateDetailed(self): + for index, register in enumerate(self.pool): + print("%u: %s"%(index, register.tag)) + +@dataclass +class RegisterPoolResource: + idx: int + size: int + +@contextmanager +def allocTmpGpr(pool: RegisterPool, num: int, upperLimit: int, alignment: Optional[int]=None, tag: Optional[str]=None, overflowListener=None): + """ + A context-manager based temporary resource allocator for given RegisterPool object. + + :param pool: `RegisterPool` Resource pool + :param num: `int` Size to allocate + :param alignment: `Optional[bool]` Resource to be aligned to spcified alignment. If not specified, alignment set to 2 if `num` > 1 else 1 + :param tag: `Optional[str]` Specified tag string for code generation + :raises `RegisterPool.ResourceOverflowException` if cannot allocate resource + :returns: `(offset, size)`, offset: `int` Start offset of allocated resource + size: `int` Size of allocated resource + """ + assert alignment is None or alignment > 0 + if alignment is None: + alignment = 1 if num == 1 else 2 + if tag is None: + tag = f"allocTmp{pool.type.upper()}gpr({num})" + + try: + allocatedSgprIdx = pool.checkOutAligned(num, alignment, tag, False) + + if allocatedSgprIdx + num > upperLimit: + exception = RegisterPool.ResourceOverflowException(f"{pool.type.upper()}gpr overflow") + if overflowListener: + overflowListener(exception) + else: + raise exception + + yield RegisterPoolResource(idx=allocatedSgprIdx, size=num) + finally: + pool.checkIn(allocatedSgprIdx) # type: ignore diff --git a/tensilelite/Tensile/TensileInstructions/Utils.py b/tensilelite/Tensile/TensileInstructions/Utils.py new file mode 100644 index 0000000000..387c292f97 --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/Utils.py @@ -0,0 +1,248 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Base import getGfxName +from .Code import Module +from .Containers import HolderContainer, RegisterContainer, RegName +from .DataType import DataType +from .Enums import InstType +from .Formatting import printExit +from .Instructions import Instruction, SWaitCnt + +from math import log +from typing import List, Tuple +import os +import random +import shlex +import string + +######################################## +# Format GPRs +######################################## + +def _gpr(*args): + gprType = args[0] + args = args[1] + if isinstance(args[0], Holder): + idx = args[0].idx + name = args[0].name + if len(args) == 1: + return HolderContainer(gprType, name, idx, 1) + elif len(args) == 2: + return HolderContainer(gprType, name, idx, args[1]) + elif isinstance(args[0], int): + if len(args) == 1: + return RegisterContainer(gprType, None, args[0], 1) + elif len(args) == 2: + return RegisterContainer(gprType, None, args[0], args[1]) + elif isinstance(args[0], str): + name = _generateRegName(args[0]) + if len(args) == 1: + return RegisterContainer(gprType, name, None, 1) + elif len(args) == 2: + return RegisterContainer(gprType, name, None, args[1]) + else: + printExit("Unknown %sgpr name or index"%gprType) + +def vgpr(*args): + return _gpr("v", args) + +def sgpr(*args): + return _gpr("s", args) + +def accvgpr(*args): + return _gpr("acc", args) + +def mgpr(*args): + return _gpr("m", args) + +def _generateRegName(rawText): + splitTxt = rawText.split("+") + offsets = [] + if len(splitTxt) > 1: + for arg in splitTxt[1:]: + offsets.append(int(arg)) + return RegName(splitTxt[0], *offsets) if offsets else RegName(splitTxt[0]) + +class Holder: + def __init__(self, idx=None, name=None): + if name: + self.name = _generateRegName(name) + assert(idx == None) + else: + self.name = name + assert(name == None) + self.idx = idx + +######################################## +# mfma +######################################## + +def _dataTypeNameAbbrevToInstType(abbrev: str) -> InstType: + if abbrev == 'f64': + return InstType.INST_F64 + elif abbrev == 'f32': + return InstType.INST_F32 + elif abbrev == 'f16': + return InstType.INST_F16 + elif abbrev == 'i32': + return InstType.INST_I32 + elif abbrev == 'i8': + return InstType.INST_I8 + elif abbrev == 'bf16': + return InstType.INST_BF16 + else: + assert("Unsupported data type.") + return InstType.INST_NOTYPE + +def dataTypeToMfmaInstTypePair(dataType: DataType, Fp16AltImpl: bool) -> Tuple[InstType, InstType]: + miInTypeStr = "bf16" if Fp16AltImpl else dataType.toNameAbbrev() + miInInstType = _dataTypeNameAbbrevToInstType(miInTypeStr) # v_mfma_[...xK] + miOutInstType = _dataTypeNameAbbrevToInstType(dataType.MIOutputTypeNameAbbrev()) # v_mfma_.. + return miInInstType, miOutInstType + +######################################## +# Label Manager +######################################## + +def magicGenerator(chars=(string.ascii_uppercase + string.digits)): + return ''.join(random.choice(chars) for _ in range(16)) + +class LabelManager(): + def __init__(self): + self.labelDict = dict() + + def addName(self, name): + if name not in self.labelDict: + self.labelDict[name] = 0 + else: + self.labelDict[name] += 1 + + def getUniqueName(self): + name = magicGenerator() + while 1: + if name not in self.labelDict: + break + name = magicGenerator() + return self.getName(name) + + def getUniqueNamePrefix(self, prefix): + name = prefix + "_" + magicGenerator() + while 1: + if name not in self.labelDict: + break + name = prefix + "_" + magicGenerator() + return self.getName(name) + + def getName(self, name): + if name not in self.labelDict: + self.labelDict[name] = 0 + return name + "_" + str(self.labelDict[name]) + + def getNameInc(self, name): + self.addName(name) + if self.labelDict[name] == 0: + return name + return name + "_" + str(self.labelDict[name]) + + def getNameIndex(self, name, index): + if name not in self.labelDict: + printExit("You have to add a label first to get a label name with specific index.") + if index > self.labelDict[name]: + printExit("The index %u exceeded. (> %u)"%(index, self.labelDict[name])) + return name + "_" + str(index) + +######################################## +# Math +######################################## + +def log2(x): + return int(log(x, 2) + 0.5) + +def ceilDivide(numerator, denominator): + # import pdb + # pdb.set_trace() + try: + if numerator < 0 or denominator < 0: + raise ValueError + except ValueError: + print("ERROR: Can't have a negative register value") + return 0 + try: + div = int((numerator+denominator-1) // denominator) + except ZeroDivisionError: + print("ERROR: Divide by 0") + return 0 + return div + +def roundUpToNearestMultiple(numerator, denominator): + return ceilDivide(numerator,denominator)*int(denominator) + +######################################## +# Others +######################################## + +def replaceHolder(module, dst): + if isinstance(module, Module): + for item in module.items(): + replaceHolder(item, dst) + elif isinstance(module, Instruction): + for param in module.getParams(): + if isinstance(param, HolderContainer): + param.setRegNum(dst) + param = param.getCopiedRC() + elif isinstance(module, SWaitCnt): + assert(isinstance(dst, int)) + if isinstance(module.vmcnt, HolderContainer): + module.vmcnt = module.vmcnt.holderIdx + dst + if isinstance(module.lgkmcnt, HolderContainer): + module.lgkmcnt = module.lgkmcnt.holderIdx + dst + if isinstance(module.vscnt, HolderContainer): + module.vscnt = module.vscnt.holderIdx + dst + + return module + +def getAsmCompileArgs(assemblerPath: str, codeObjectVersion: str, archHasV3: bool, \ + isa: Tuple[int, int, int], wavefrontSize: int, \ + sourceFileName: str, objectFileName: str, *moreArgs): + launcher = shlex.split(os.environ.get('Tensile_ASM_COMPILER_LAUNCHER', '')) + rv = launcher + [assemblerPath, '-x', 'assembler', '-target', 'amdgcn-amd-amdhsa'] + + if archHasV3: + rv += ['-mcode-object-version=2' if codeObjectVersion == "V2" else '-mcode-object-version=4'] + + rv += ['-mcpu=' + getGfxName(isa)] + + if wavefrontSize == 64: + rv += ['-mwavefrontsize64'] + else: + rv += ['-mno-wavefrontsize64'] + + rv += moreArgs + rv += ['-c', '-o', objectFileName, sourceFileName] + return rv + +def getAsmLinkCodeObjectArgs(assemblerPath: str, objectFileNames: List[str], \ + coFileName: str, *moreArgs): + rv = [assemblerPath, '-target', 'amdgcn-amd-amdhsa'] + rv += moreArgs + rv += ['-o', coFileName] + objectFileNames + return rv diff --git a/tensilelite/Tensile/TensileInstructions/__init__.py b/tensilelite/Tensile/TensileInstructions/__init__.py new file mode 100644 index 0000000000..7914a51045 --- /dev/null +++ b/tensilelite/Tensile/TensileInstructions/__init__.py @@ -0,0 +1,39 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +# Init module +import sys +if not (sys.version_info[0] >= 3 and sys.version_info[1] >= 7): + raise Exception("Must be using Python 3.7 or above") + +from .Base import * +from .Code import * +from .Containers import * +from .DataType import * +from .Enums import * +from .ExtInstructions import * +from .Formatting import * +from .Instructions import * +from .Macros import * +from .Math import * +from .Pass import * +from .RegisterPool import * +from .Utils import * diff --git a/tensilelite/Tensile/TensilePass.py b/tensilelite/Tensile/TensilePass.py new file mode 100644 index 0000000000..9f353ce2e4 --- /dev/null +++ b/tensilelite/Tensile/TensilePass.py @@ -0,0 +1,102 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .TensileInstructions import Module, SAddI32, SEndpgm + +from copy import deepcopy +from dataclasses import dataclass, field + +####################################### +# Public functions +####################################### +@dataclass +class TensilePassOptions: + removeDupActFunc: bool = field(init=False) + +def TensilePass(module, options: TensilePassOptions): + if options.removeDupActFunc: + _removeDuplicatedActivationFunctions(module) + + +def getActivationFunctionModuleName(gwvw, sgpr, tmpVgpr, tmpSgpr): + return "ActFunc_VW%d_Sgpr%d_Tmp%s_%s"%(gwvw, sgpr, tmpVgpr, tmpSgpr) + +def getActivationBranchModuleName(): + return "InsertActFunctionCallAddrCalc" + +####################################### +# Internal functions +####################################### +def _findActFunc(module) -> dict: + modFunc = {} + for item in module.items(): + if isinstance(item, Module): + if "ActFunc_VW" in item.name: + if item.name in modFunc: + modFunc[item.name].append(item) + else: + modFunc[item.name] = [item] + else: + tmp = _findActFunc(item) + for key, t in tmp.items(): + if key in modFunc: + modFunc[key].extend(t) + else: + modFunc[key] = t + return modFunc + +def _replaceActBranchLabel(module, labels): + for item in module.items(): + if isinstance(item, Module): + if "InsertActFunctionCallAddrCalc" in item.name: + labelLeft = labels[1:] + replaceLabel = False + for inst in item.items(): + if isinstance(inst, SAddI32) and inst.comment == "target branch offset": + if inst.src[0] in labelLeft: + replaceLabel = True + break + if replaceLabel: + for inst in item.items(): + if isinstance(inst, SAddI32) and inst.comment == "target branch offset": + # The label is generated in the format of XXXX_1, XXXX_2 + # and string.rpartition returns ('XXXX', '_', '1'). + # We only need the first string. + part = inst.src[0].rpartition("_") + inst.src[0] = part[0] + else: + _replaceActBranchLabel(item, labels) + +def _removeDuplicatedActivationFunctions(module): + modFunc = _findActFunc(module) + moduleLast = Module("AddToLast") + for _, mlist in modFunc.items(): + if len(mlist) > 1: + moduleLast.add(deepcopy(mlist[0])) + labels = [] + for ml in mlist: + labelName = ml.items()[0].items()[0].getLabelName() + labels.append(labelName) + ml.items().clear() + _replaceActBranchLabel(module, labels) + if moduleLast.items(): + module.add(moduleLast) + module.add(SEndpgm()) diff --git a/tensilelite/Tensile/TensileRetuneLibrary.py b/tensilelite/Tensile/TensileRetuneLibrary.py new file mode 100644 index 0000000000..6fd5e1d73c --- /dev/null +++ b/tensilelite/Tensile/TensileRetuneLibrary.py @@ -0,0 +1,196 @@ +############################################################################### +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from . import BenchmarkProblems +from . import ClientExecutable +from . import ClientWriter +from . import LibraryIO +from . import LibraryLogic +from . import Common +from .Common import globalParameters, print1, printWarning, ensurePath, assignGlobalParameters, \ + pushWorkingPath, popWorkingPath, restoreDefaultGlobalParameters, HR +from .Tensile import addCommonArguments, argUpdatedGlobalParameters +from .SolutionStructs import ProblemSizes +from . import __version__ + +import argparse +import copy +import os +import shutil +import sys + + +def parseCurrentLibrary(libPath, sizePath): + libYaml = LibraryIO.readYAML(libPath) + # parseLibraryLogicData mutates the original data, so make a copy + fields = LibraryIO.parseLibraryLogicData(copy.deepcopy(libYaml), libPath) + (_, _, problemType, solutions, _, exactLogic, _, _, _) = fields + + # get performance metric + if len(libYaml) > 10: + Common.globalParameters["PerformanceMetric"] = libYaml[10] + + # process exactLogic into ProblemSizes + sizes = [] + if sizePath is None: + for (size, mapping) in exactLogic: + sizes.append({"Exact": size}) + else: + sizes = LibraryIO.readYAML(sizePath) + + # remove duplicate solutions and reindex + solutions = [v1 for i, v1 in enumerate(solutions) if not any(v1 == v2 for v2 in solutions[:i])] + for i, s in enumerate(solutions): + s["SolutionIndex"] = i + + problemSizes = ProblemSizes(problemType, sizes) + + return (libYaml, solutions, problemSizes) + + +def runBenchmarking(solutions, problemSizes, outPath, update): + # TODO some copy-pasting from BenchmarkProblems.benchmarkProblemType + # could use a refactor to elimate duplicated code + ClientExecutable.getClientExecutable() + + shortName = "benchmark" + benchmarkDir = os.path.join(outPath, shortName) + sourceDir = os.path.join(benchmarkDir, "source") + resultsDir = os.path.normpath(os.path.join(globalParameters["WorkingPath"], "Data")) + libraryFile = os.path.join(resultsDir, "benchmark.yaml") + + ensurePath(sourceDir) + ensurePath(resultsDir) + + if update: + Common.globalParameters["LibraryUpdateFile"] = os.path.join(resultsDir, "update.yaml") + + pushWorkingPath(shortName) + pushWorkingPath("source") + BenchmarkProblems.writeBenchmarkFiles(benchmarkDir, solutions, problemSizes , "", shortName, []) + popWorkingPath() # source + + libraryLogicPath = None + forBenchmark = True + # TODO make this work with TileAware selection + returncode = ClientWriter.runClient(libraryLogicPath, forBenchmark, False) + if returncode: + printWarning("Benchmarking Client exited with code {}. Trying to continue".format(returncode)) + + # write solutions yaml file + for sol in solutions: + sol["ISA"] = list(sol["ISA"]) + LibraryIO.writeSolutions(libraryFile, problemSizes, "", solutions) + + popWorkingPath() # benchmark + + # copy results to expected directory + out = os.path.join(globalParameters["WorkingPath"], "2_BenchmarkData") + ensurePath(out) + shutil.copy(os.path.join(resultsDir, "benchmark.csv"), os.path.join(out, "benchmark.csv")) + shutil.copy(os.path.join(resultsDir, "benchmark.yaml"), os.path.join(out, "benchmark.yaml")) + + +def TensileRetuneLibrary(userArgs): + print1("") + print1(HR) + print1("#") + print1("# Tensile Retune Library v{}".format(__version__)) + + # argument parsing and related setup + argParser = argparse.ArgumentParser() + argParser.add_argument("LogicFile", type=os.path.realpath, + help="Library logic file to retune") + argParser.add_argument("OutputPath", type=os.path.realpath, + help="Where to run benchmarks and output results") + argParser.add_argument("SizeFile", type=os.path.realpath, nargs="?", + help="Yaml file with sizes to tune; same format as the 'ProblemSizes' " + "section of a regular Tensile config " + "(https://github.com/ROCmSoftwarePlatform/Tensile/wiki/Benchmark-Protocol)", + default=None) + argParser.add_argument("--update-method", "-u", dest="updateMethod", + choices=["remake", "update", "both"], default="remake", + help="Method for making new library logic file") + + addCommonArguments(argParser) + args = argParser.parse_args(userArgs) + + libPath = args.LogicFile + sizePath = args.SizeFile + print1("# Library Logic: {}".format(libPath)) + print1("#") + print1(HR) + print1("") + + if args.updateMethod == "remake": + update = False + remake = True + elif args.updateMethod == "update": + update = True + remake = False + else: # args.updateMethod == "both" + update = True + remake = True + + ############################################## + # Retuning + ############################################## + outPath = ensurePath(os.path.abspath(args.OutputPath)) + restoreDefaultGlobalParameters() + assignGlobalParameters({"LibraryFormat": "msgpack", + "OutputPath": outPath, + "WorkingPath": outPath}) + + overrideParameters = argUpdatedGlobalParameters(args) + for key, value in overrideParameters.items(): + print1("Overriding {0}={1}".format(key, value)) + Common.globalParameters[key] = value + + # parse library logic then setup and run benchmarks + (rawYaml, solutions, problemSizes) = parseCurrentLibrary(libPath, sizePath) + runBenchmarking(solutions, problemSizes, outPath, update) + + if remake: + # write library logic file + LibraryLogic.main({"ScheduleName": rawYaml[1], + "ArchitectureName": rawYaml[2], + "DeviceNames": rawYaml[3] }) + + if update: + # read update yaml from benchmark client and update logic + print1("") + print1(HR) + print1("# Reading update file from Benchmarking Client") + updateFile = os.path.join(outPath, "Data", "update.yaml") + updateLogic = LibraryIO.readYAML(updateFile) + rawYaml[7] = updateLogic + + # write updated library logic (does not overwrite original) + libName = os.path.basename(libPath) + outFile = os.path.join(outPath, libName) + + print1("# Writing updated Library Logic: {}".format(outFile)) + LibraryIO.writeYAML(outFile, rawYaml, explicit_start=False, explicit_end=False) + print1(HR) + + +def main(): + TensileRetuneLibrary(sys.argv[1:]) diff --git a/tensilelite/Tensile/TensileUpdateLibrary.py b/tensilelite/Tensile/TensileUpdateLibrary.py new file mode 100644 index 0000000000..85d55f5be4 --- /dev/null +++ b/tensilelite/Tensile/TensileUpdateLibrary.py @@ -0,0 +1,103 @@ +############################################################################### +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from . import Common +from . import LibraryIO +from .Common import assignGlobalParameters, print1, restoreDefaultGlobalParameters, HR +from .Tensile import addCommonArguments, argUpdatedGlobalParameters +from . import __version__ + +import argparse +import copy +import os +import sys + + +def TensileUpdateLibrary(userArgs): + print1("") + print1(HR) + print1("#") + print1("# Tensile Update Library v{}".format(__version__)) + + # argument parsing and related setup + argParser = argparse.ArgumentParser() + argParser.add_argument("LogicFile", type=os.path.realpath, + help="Library logic file to update") + argParser.add_argument("OutputPath", type=os.path.realpath, + help="Where to place updated logic file") + + addCommonArguments(argParser) + args = argParser.parse_args(userArgs) + + libPath = args.LogicFile + print1("# Library Logic: {}".format(libPath)) + print1("#") + print1(HR) + print1("") + + # setup global parameters + restoreDefaultGlobalParameters() + assignGlobalParameters({}) + overrideParameters = argUpdatedGlobalParameters(args) + for key, value in overrideParameters.items(): + print1("Overriding {0}={1}".format(key, value)) + Common.globalParameters[key] = value + + # update logic file + outPath = Common.ensurePath(os.path.abspath(args.OutputPath)) + filename = os.path.basename(libPath) + outFile = os.path.join(outPath, filename) + + libYaml = LibraryIO.readYAML(libPath) + # parseLibraryLogicData mutates the original data, so make a copy + fields = LibraryIO.parseLibraryLogicData(copy.deepcopy(libYaml), libPath) + (_, _, problemType, solutions, _, _, _, _, _) = fields + + # problem type object to state + problemTypeState = problemType.state + problemTypeState["DataType"] = problemTypeState["DataType"].value + problemTypeState["DestDataType"] = problemTypeState["DestDataType"].value + problemTypeState["ComputeDataType"] = problemTypeState["ComputeDataType"].value + + # solution objects to state + solutionList = [] + for solution in solutions: + solutionState = solution.getAttributes() + solutionState["ProblemType"] = solutionState["ProblemType"].state + solutionState["ProblemType"]["DataType"] = \ + solutionState["ProblemType"]["DataType"].value + solutionState["ProblemType"]["DestDataType"] = \ + solutionState["ProblemType"]["DestDataType"].value + solutionState["ProblemType"]["ComputeDataType"] = \ + solutionState["ProblemType"]["ComputeDataType"].value + + solutionState["ISA"] = list(solutionState["ISA"]) + solutionList.append(solutionState) + + # update yaml + libYaml[0] = {"MinimumRequiredVersion":__version__} + libYaml[4] = problemTypeState + libYaml[5] = solutionList + LibraryIO.writeYAML(outFile, libYaml, explicit_start=False, explicit_end=False) + + +def main(): + TensileUpdateLibrary(sys.argv[1:]) diff --git a/tensilelite/Tensile/Utilities/archive/merge_rocblas_yaml_files.py b/tensilelite/Tensile/Utilities/archive/merge_rocblas_yaml_files.py new file mode 100644 index 0000000000..01827e2785 --- /dev/null +++ b/tensilelite/Tensile/Utilities/archive/merge_rocblas_yaml_files.py @@ -0,0 +1,412 @@ +################################################################################ +# Copyright 2020-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from __future__ import print_function +import os +import sys +import argparse + +HR = "################################################################################" + +################################################################################ +# Print Debug +################################################################################ + +def printWarning(message): + print("Tensile::WARNING: %s" % message) + sys.stdout.flush() + +def printExit(message): + print("Tensile::FATAL: %s" % message) + sys.stdout.flush() + sys.exit(-1) + +try: + import yaml +except ImportError: + printExit("You must install PyYAML to use Tensile (to parse config files). See http://pyyaml.org/wiki/PyYAML for installation instructions.") + +def ensurePath( path ): + if not os.path.exists(path): + os.makedirs(path) + return path + +################################################################################ +# Library Logic Container +################################################################################ +class LibraryLogic: + + def __init__(self,filename=None): + + if filename is not None: + print ("# Reading Library Logic: " + filename) + try: + stream = open(filename, "r") + except IOError: + printExit("Cannot open file: %s" % filename ) + data = yaml.load(stream, yaml.SafeLoader) + + if isinstance(data, list): + + length = len(data) + + if (length > 0): + self.__set_versionString(data[0]["MinimumRequiredVersion"]) + else: + self.__set_versionString(None) + + if (length > 1): + self.__set_scheduleName(data[1]) + else: + self.__set_scheduleName(None) + + if (length > 2): + self.__set_architectureName(data[2]) + else: + self.__set_architectureName(None) + + if (length > 3): + self.__set_deviceNames(data[3]) + else: + self.__set_deviceNames(None) + + if (length > 4): + self.__set_problemType(data[4]) + else: + self.__set_problemType(None) + + if (length > 5): + self.__set_solutionStates(data[5]) + else: + self.__set_solutionStates(None) + + if (length > 6): + self.__set_indexOrder(data[6]) + else: + self.__set_indexOrder(None) + + if (length > 7): + exactData = data[7] + exactList = list() + for exact in exactData: + size = exact[0] + if (len(size) > 4): + exactOut = [size[:4],exact[1]] + exactList.append(exactOut) + else: + exactList.append(exact) + self.__set_exactLogic(exactList) + else: + self.__set_exactLogic(None) + + if (length > 8): + self.__set_rangeLogic(data[8]) + else: + self.__set_rangeLogic(None) + + else: + printExit("Invalid Logic file: %s" % filename) + + stream.close() + + else: + self.__set_versionString(None) + self.__set_scheduleName(None) + self.__set_architectureName(None) + self.__set_deviceNames(None) + self.__set_problemType(None) + self.__set_solutionStates(None) + self.__set_indexOrder(None) + self.__set_exactLogic(None) + self.__set_rangeLogic(None) + + #versionString + def __get_versionString(self): + return self.__versionString + + def __set_versionString(self,value): + self.__versionString = value + + versionString = property(__get_versionString,__set_versionString) + + #scheduleName + def __get_scheduleName(self): + return self.__scheduleName + + def __set_scheduleName(self, value): + self.__scheduleName = value + + scheduleName = property(__get_scheduleName,__set_scheduleName) + + #architectureName + def __get_architectureName(self): + return self.__architectureName + + def __set_architectureName(self,value): + self.__architectureName = value + + architectureName = property(__get_architectureName,__set_architectureName) + + #deviceNames + def __get_deviceNames(self): + return self.__deviceNames + + def __set_deviceNames(self,value): + self.__deviceNames = value + + deviceNames = property(__get_deviceNames,__set_deviceNames) + + + #problemTypeState + def __get_problemType(self): + return self.__problemType + + def __set_problemType(self,value): + self.__problemType = value + + problemType = property(__get_problemType,__set_problemType) + + #solutionStates + def __get_solutionStates(self): + return self.__solutionStates + + def __set_solutionStates(self,value): + self.__solutionStates = value + + solutionStates = property(__get_solutionStates,__set_solutionStates) + + #indexOrder + def __get_indexOrder(self): + return self.__indexOrder + + def __set_indexOrder(self,value): + self.__indexOrder = value + + indexOrder = property(__get_indexOrder,__set_indexOrder) + + + #exactLogic + def __get_exactLogic(self): + return self.__exactLogic + + def __set_exactLogic(self,value): + self.__exactLogic = value + + exactLogic = property(__get_exactLogic,__set_exactLogic) + + #rangeLogic + def __get_rangeLogic(self): + return self.__rangeLogic + + def __set_rangeLogic(self,value): + self.__rangeLogic = value + + rangeLogic = property(__get_rangeLogic,__set_rangeLogic) + + def writeLibraryLogic(self,filename): + + data = [] + + data.append({"MinimumRequiredVersion":self.versionString}) + data.append(self.scheduleName) + data.append(self.architectureName) + data.append(self.deviceNames) + data.append(self.problemType) + data.append(self.solutionStates) + data.append(self.indexOrder) + data.append(self.exactLogic) + data.append(self.rangeLogic) + + if not data: + printExit("No data to output") + else: + try: + stream = open(filename, "w") + yaml.safe_dump(data, stream, default_flow_style=None) + stream.close() + except IOError: + printExit("Cannot open file: %s" % filename) + + +def MergeTensileLogicFiles(origionalLibraryLogic, exactLibraryLogic): + + mergedLibraryLogic = LibraryLogic() + + solutionList = origionalLibraryLogic.solutionStates + solutionListExact = exactLibraryLogic.solutionStates + + indexKey = "SolutionIndex" + # zero out solution indexes + for solution in solutionList: + solution[indexKey] = 0 + + for solution in solutionListExact: + solution[indexKey] = 0 + + newSolutionOffset = len(solutionList) + + filterdSolutionExactList = [] + replicationMapping = {} + idx = 0 + idxMapping = newSolutionOffset + + mergedSolutionList = [] + for solution in solutionList: + mergedSolutionList.append(solution) + + # construct the mappings from the old exact kernal configurations + # to their definitions in the merged files + for solution in solutionListExact: + if solution in solutionList: + # if solution exists in the origional configuration the + # its placement in the merged kernel configurations list + # gets mapped to the pre-existing configuration + idxOrg = solutionList.index(solution) + replicationMapping[idx] = idxOrg + + else: + filterdSolutionExactList.append(solution) + # if the solution does not exist in the origional configurations + # it gets mapped to the new offset + replicationMapping[idx] = idxMapping + mergedSolutionList.append(solution) + idxMapping += 1 + + idx += 1 + + exactLogic = origionalLibraryLogic.exactLogic + exactLogicExact = exactLibraryLogic.exactLogic + + filteredExactLogicExact = [] + + # use the mapping from above to remap the exact logic + # in the merged file + for exact in exactLogicExact: + # example exact entry [[123,124,1,123], [5, 4312.3]] + # the first fiedl in [5, 4312.3] is the mapping to the + # kernel configuration + kernelIndex = exact[1][0] + + if kernelIndex in replicationMapping: + exact[1][0] = replicationMapping[kernelIndex] + + filteredExactLogicExact.append(exact) + + sizeList, _ = zip(*filteredExactLogicExact) + + mergedExactLogic = [] + for logicMapping in exactLogic: + if logicMapping[0] not in sizeList: + mergedExactLogic.append(logicMapping) + + for logicMapping in filteredExactLogicExact: + mergedExactLogic.append(logicMapping) + + # re index solutions + index = 0 + for solution in mergedSolutionList: + solution[indexKey] = index + index += 1 + + + mergedLibraryLogic.versionString = origionalLibraryLogic.versionString + mergedLibraryLogic.scheduleName = origionalLibraryLogic.scheduleName + mergedLibraryLogic.architectureName = origionalLibraryLogic.architectureName + mergedLibraryLogic.deviceNames = origionalLibraryLogic.deviceNames + mergedLibraryLogic.problemType = origionalLibraryLogic.problemType + mergedLibraryLogic.solutionStates = mergedSolutionList + mergedLibraryLogic.indexOrder = origionalLibraryLogic.indexOrder + mergedLibraryLogic.exactLogic = mergedExactLogic + mergedLibraryLogic.rangeLogic = origionalLibraryLogic.rangeLogic + + return mergedLibraryLogic + + +def ProcessMergeLogicFile(exactFileName, origionalFileName, outputFileName): + + _, fileName = os.path.split(exactFileName) + + print ("processing file: " + fileName) + + libraryLogic = LibraryLogic(origionalFileName) + libraryLogicExact = LibraryLogic(exactFileName) + + mergedLibraryLogic = MergeTensileLogicFiles(libraryLogic,libraryLogicExact) + + mergedLibraryLogic.writeLibraryLogic(outputFileName) + +def RunMergeTensileLogicFiles(): + + print("") + print(HR) + print("# Merge Library Logic") + print(HR) + print("") + + ############################################################################## + # Parse Command Line Arguments + ############################################################################## + + argParser = argparse.ArgumentParser() + argParser.add_argument("OrigionalLogicPath", help="Path to the origional LibraryLogic.yaml input files.") + argParser.add_argument("ExactLogicPath", help="Path to the exact LibraryLogic.yaml input files.") + argParser.add_argument("OutputPath", help="Where to write library files?") + + args = argParser.parse_args() + + origionalLogicPath = args.OrigionalLogicPath + exactLogicPath = args.ExactLogicPath + outputPath = args.OutputPath + print ("Origional Logic Path: " + origionalLogicPath) + print ("Exact Logic Path: " + exactLogicPath) + print ("OutputPath: " + outputPath) + + print("") + ensurePath(outputPath) + if not os.path.exists(exactLogicPath): + printExit("LogicPath %s doesn't exist" % exactLogicPath) + + exactLogicFiles = [os.path.join(exactLogicPath, f) for f in os.listdir(exactLogicPath) \ + if (os.path.isfile(os.path.join(exactLogicPath, f)) \ + and os.path.splitext(f)[1]==".yaml")] + + for exactLogicFilePath in exactLogicFiles: + _, fileName = os.path.split(exactLogicFilePath) + + origionalLogicFilePath = os.path.join(origionalLogicPath, fileName) + if os.path.isfile(origionalLogicFilePath): + + outputLogicFilePath = os.path.join(outputPath, fileName) + + try: + ProcessMergeLogicFile(exactLogicFilePath, origionalLogicFilePath, outputLogicFilePath) + except Exception as ex: + print("Exception: {0}".format(ex)) + + else: + print ("# file does not exist in origional directory " + origionalLogicFilePath) + + +################################################################################ +# Main +################################################################################ +if __name__ == "__main__": + RunMergeTensileLogicFiles() diff --git a/tensilelite/Tensile/Utilities/merge.py b/tensilelite/Tensile/Utilities/merge.py new file mode 100644 index 0000000000..0d149d70b1 --- /dev/null +++ b/tensilelite/Tensile/Utilities/merge.py @@ -0,0 +1,457 @@ +################################################################################ +# Copyright 2020-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +import yaml +import os +import sys +import argparse +from copy import deepcopy +from enum import IntEnum + +verbosity = 1 + +def ensurePath(path): + if not os.path.exists(path): + os.makedirs(path) + return path + +def allFiles(startDir): + current = os.listdir(startDir) + files = [] + for filename in [_current for _current in current if os.path.splitext(_current)[-1].lower() == '.yaml']: + fullPath = os.path.join(startDir,filename) + if os.path.isdir(fullPath): + files = files + allFiles(fullPath) + else: + files.append(fullPath) + return files + +def reindexSolutions(data): + for i, _ in enumerate(data[5]): + data[5][i]["SolutionIndex"] = i + return data + +def fixSizeInconsistencies(sizes, fileType): + duplicates = list() + for i in range(0,len(sizes)): + currSize = sizes[i][0] + # >= so size will be trimmed when a SolutionTag is included + if len(currSize) >= 8: + currSize = currSize[:-4] + if currSize in (item for index in sizes for item in index): + duplicates.append(i-len(duplicates)) + else: + sizes[i][0] = currSize + sizes_ = deepcopy(sizes) + if len(duplicates) > 0: + for i in duplicates: + sizes_.pop(i) + verbose(len(duplicates), "duplicate size(s) removed from", fileType, "logic file") + return sizes_, len(sizes_) + +# remove dict keys "SolutionIndex" and "SolutionNameMin" from dict +def cmpHelper(sol): + return {k:v for k, v in sol.items() if k!="SolutionIndex" and k!="SolutionNameMin"} + +def addKernel(solutionPool, solution): + for item in solutionPool: + if cmpHelper(item) == cmpHelper(solution): + index = item["SolutionIndex"] + debug("...Reuse previously existed kernel", end="") + break + else: + index = len(solutionPool) + _solution = deepcopy(solution) # if we don't we will see some subtle errors + _solution["SolutionIndex"] = index + solutionPool.append(_solution) + debug("...A new kernel has been added", end="") + debug("({}) {}".format(index, solutionPool[index]["SolutionNameMin"] if "SolutionNameMin" in solutionPool[index] else "(SolutionName N/A)")) + return solutionPool, index + +# update dependant parameters if StaggerU == 0 +def sanitizeSolutions(solList): + for sol in solList: + if sol.get("StaggerU") == 0: + sol["StaggerUMapping"] = 0 + sol["StaggerUStride"] = 0 + sol["_staggerStrideShift"] = 0 + +def removeUnusedKernels(origData, prefix=""): + origNumSolutions = len(origData[5]) + + kernelsInUse = [ index for _, [index, _] in origData[7] ] + for i, solution in enumerate(origData[5]): + solutionIndex = solution["SolutionIndex"] + origData[5][i]["__InUse__"] = True if solutionIndex in kernelsInUse else False + + # debug prints + for o in [o for o in origData[5] if o["__InUse__"]==False]: + debug("{}Solution ({}) {} is unused".format( + prefix, + o["SolutionIndex"], + o["SolutionNameMin"] if "SolutionNameMin" in o else "(SolutionName N/A)")) + + # filter out dangling kernels + origData[5] = [ {k: v for k, v in o.items() if k != "__InUse__"} + for o in origData[5] if o["__InUse__"]==True ] + + # reindex solutions + idMap = {} # new = idMap[old] + for i, solution in enumerate(origData[5]): + idMap[solution["SolutionIndex"]] = i + origData[5][i]["SolutionIndex"] = i + for i, [size, [oldSolIndex, eff]] in enumerate(origData[7]): + origData[7][i] = [size, [idMap[oldSolIndex], eff]] + + numInvalidRemoved = origNumSolutions - len(origData[5]) + return origData, numInvalidRemoved + +def loadData(filename): + try: + stream = open(filename, "r") + except IOError: + print("Cannot open file: ", filename) + sys.stdout.flush() + sys.exit(-1) + data = yaml.load(stream, yaml.SafeLoader) + return data + +# this is for complying the behavior of legacy merge script, where incremental logic +# file always replaces the base logic file even it's slower in performance - +# in the future we may let default force merge policy = False +def defaultForceMergePolicy(incFile): + if "arcturus" in incFile: + forceMerge = False + else: + forceMerge = True + + return forceMerge + +def msg(*args, **kwargs): + for i in args: print(i, end=" ") + print(**kwargs) + +def verbose(*args, **kwargs): + if verbosity < 1: return + msg(*args, **kwargs) + +def debug(*args, **kwargs): + if verbosity < 2: return + msg(*args, **kwargs) + +# Tags distinguishing solution types +# Can be added to size key to allow solutions of each type to be present +# in logic file for a given size +class MfmaTag(IntEnum): + VALU = 0 + MFMA = 1 + + def __str__(self): + return ["VALU", "MFMA"][self] + def __repr__(self): + return str(self) + +class AlphaValueTag(IntEnum): + ANY = 0 + ONE = 1 + NEG_ONE = 2 + ZERO = 3 + + def __str__(self): + return "Alpha="+["Any", "1", "-1", "0"][self] + def __repr__(self): + return str(self) + +class BetaValueTag(IntEnum): + ANY = 0 + ONE = 1 + NEG_ONE = 2 + ZERO = 3 + + def __str__(self): + return "Beta="+["Any", "1", "-1", "0"][self] + def __repr__(self): + return str(self) + +def strToScalarValueTag(Class, value): + if value == "Any": + return Class.ANY + if value == 1: + return Class.ONE + if value == -1: + return Class.NEG_ONE + if value == 0: + return Class.ZERO + else: + raise RuntimeError("Unsupported value for Alpha/Beta scalar value") + +class CEqualsDTag(IntEnum): + C_EQ_D = 0 + C_NEQ_D = 1 + + def __str__(self): + return ["C=D", "C!=D"][self] + def __repr__(self): + return str(self) + +# Tag of form (MFMATag, AlphaValueTag, BetaValueTag, CEqualsDTag) +def getSolutionTag(solution): + tagTuple = () + if solution.get("EnableMatrixInstruction", False) or solution.get("MatrixInstruction", False): + tagTuple = tagTuple + (MfmaTag.MFMA,) + else: + tagTuple = tagTuple + (MfmaTag.VALU,) + + return tagTuple + +def findSolutionWithIndex(solutionData, solIndex): + # Check solution at the index corresponding to solIndex first + if solIndex < len(solutionData) and solutionData[solIndex]["SolutionIndex"] == solIndex: + return solutionData[solIndex] + else: + debug("Searching for index...") + solution = [s for s in solutionData if s["SolutionIndex"]==solIndex] + assert(len(solution) == 1) + return solution[0] + +def addSolutionTagToKeys(solutionMap, solutionPool): + return [[[getSolutionTag(findSolutionWithIndex(solutionPool, idx))] + keys, [idx, eff]] + for [keys, [idx, eff]] in solutionMap] + +def removeSolutionTagFromKeys(solutionMap): + return [[keys[1:], [idx, incEff]] for keys, [idx, incEff] in solutionMap] + +# To be used with add_solution_tags to allow faster general solutions to supercede slower specific ones +def findFastestCompatibleSolution(origDict, sizeMapping): + tags = sizeMapping[0] + # Tag of form (MFMATag, AlphaValueTag, BetaValueTag, CEqualsDTag) + compatibleTagList = [tags] + + # Add all compatible tags to the list + if tags[1] != AlphaValueTag.ANY: + compatibleTagList = compatibleTagList + [(t[0], AlphaValueTag.ANY) + t[2:] for t in compatibleTagList] + if tags[2] != BetaValueTag.ANY: + compatibleTagList = compatibleTagList + [t[:2] + (BetaValueTag.ANY,) + t[3:] for t in compatibleTagList] + if tags[3] != CEqualsDTag.C_NEQ_D: + compatibleTagList = compatibleTagList + [t[:3] + (CEqualsDTag.C_NEQ_D,) + t[4:] for t in compatibleTagList] + + #Find the fastest efficiency of all compatible tags + maxEfficiency = 0 + for tag in compatibleTagList: + result = origDict.get((tag,) + sizeMapping[1:], None) + if result: + _, eff = origDict[(tag,) + sizeMapping[1:]] + maxEfficiency = max(maxEfficiency, eff) + + return maxEfficiency + + +# returns merged logic data as list +def mergeLogic(origData, incData, forceMerge, trimSize=True, addSolutionTags=False): + origNumSizes = len(origData[7]) + origNumSolutions = len(origData[5]) + + incNumSizes = len(incData[7]) + incNumSolutions = len(incData[5]) + + verbose(origNumSizes, "sizes and", origNumSolutions, "kernels in base logic file") + verbose(incNumSizes, "sizes and", incNumSolutions, "kernels in incremental logic file") + + # Add SolutionTag to distinguish solutions with different requirements + origTaggedSizes = addSolutionTagToKeys(origData[7], origData[5]) + incTaggedSizes = addSolutionTagToKeys(incData[7], incData[5]) + if addSolutionTags: + origData[7] = origTaggedSizes + incData[7] = incTaggedSizes + # Print warning if addSolutionTags=False results in removed sizes + else: + origSet = {tuple(size) for size, [_, _] in origData[7]} + origTaggedSet = {tuple(size) for size, [_, _] in origTaggedSizes} + incSet = {tuple(size) for size, [_, _] in incData[7]} + incTaggedSet = {tuple(size) for size, [_, _] in incTaggedSizes} + + if len(origSet) != len(origTaggedSet): + verbose("Warning:", len(origTaggedSet) - len(origSet), "duplicate sizes are present in base logic", + "that may not be handled correctly unless --add_solution_tags is used") + if len(incSet) != len(incTaggedSet): + verbose("Warning:", len(incTaggedSet) - len(incSet), "duplicate sizes are present in incremental logic", + "that may not be handled correctly unless --add_solution_tags is used") + + + + if trimSize: + # trim 8-tuple gemm size format to 4-tuple [m, n, b, k] + # TODO future gemm size could include dictionary format so need robust preprocessing + [origData[7], origNumSizes] = fixSizeInconsistencies(origData[7], "base") + [incData[7], incNumSizes] = fixSizeInconsistencies(incData[7], "incremental") + + sanitizeSolutions(origData[5]) + sanitizeSolutions(incData[5]) + origData, numOrigRemoved = removeUnusedKernels(origData, "Base logic file: ") + incData, numIncRemoved = removeUnusedKernels(incData, "Inc logic file: ") + + solutionPool = deepcopy(origData[5]) + solutionMap = deepcopy(origData[7]) + + origDict = {tuple(origSize): [i, origEff] for i, [origSize, [origIndex, origEff]] in enumerate(origData[7])} + for incSize, [incIndex, incEff] in incData[7]: + incSolution = findSolutionWithIndex(incData[5], incIndex) + + try: + j, origEff = origDict[tuple(incSize)] + if incEff > origEff or forceMerge: + if incEff > origEff: + verbose("[O]", incSize, "already exists and has improved in performance.", end="") + elif forceMerge: + verbose("[!]", incSize, "already exists but does not improve in performance.", end="") + verbose("Efficiency:", origEff, "->", incEff, "(force_merge=True)" if forceMerge else "") + solutionPool, index = addKernel(solutionPool, incSolution) + solutionMap[j][1] = [index, incEff] + else: + verbose("[X]", incSize, "already exists but does not improve in performance.", end="") + verbose("Efficiency:", origEff, "->", incEff) + except KeyError: + if addSolutionTags and findFastestCompatibleSolution(origDict, tuple(incSize)) > incEff: + verbose("[X]", incSize, "has been rejected because a compatible solution already exists with higher performance") + else: + verbose("[-]", incSize, "has been added to solution table, Efficiency: N/A ->", incEff) + solutionPool, index = addKernel(solutionPool, incSolution) + solutionMap.append([incSize,[index, incEff]]) + + verbose(numOrigRemoved, "unused kernels removed from base logic file") + verbose(numIncRemoved, "unused kernels removed from incremental logic file") + + # Remove SolutionTag for yaml output + if addSolutionTags: + solutionMap = removeSolutionTagFromKeys(solutionMap) + + mergedData = deepcopy(origData) + mergedData[5] = solutionPool + mergedData[7] = solutionMap + mergedData, numReplaced = removeUnusedKernels(mergedData, "Merged data: ") + + numSizesAdded = len(solutionMap)-len(origData[7]) + numSolutionsAdded = len(solutionPool)-len(origData[5]) + numSolutionsRemoved = numReplaced+numOrigRemoved # incremental file not counted + + return [mergedData, numSizesAdded, numSolutionsAdded, numSolutionsRemoved] + +def avoidRegressions(originalDir, incrementalDir, outputPath, forceMerge, trimSize=True, addSolutionTags=False): + originalFiles = allFiles(originalDir) + incrementalFiles = allFiles(incrementalDir) + ensurePath(outputPath) + + # filter the incremental logic files that have the corresponding base file + incrementalFiles = [ i for i in incrementalFiles + if os.path.split(i)[-1] in [os.path.split(o)[-1] for o in originalFiles] ] + + for incFile in incrementalFiles: + basename = os.path.split(incFile)[-1] + origFile = os.path.join(originalDir, basename) + forceMerge = defaultForceMergePolicy(incFile) if forceMerge is None else forceMerge + + msg("Base logic file:", origFile, "| Incremental:", incFile, "| Merge policy: %s"%("Forced" if forceMerge else "Winner"), "| Trim size:", trimSize, + "| Add solution tags:", addSolutionTags) + origData = loadData(origFile) + incData = loadData(incFile) + + # So far "SolutionIndex" in logic yamls has zero impact on actual 1-1 size mapping (but the order of the Solution does) + # since mergeLogic() takes that value very seriously so we reindex them here so it doesn't choke on duplicated SolutionIndex + origData = reindexSolutions(origData) + incData = reindexSolutions(incData) + + mergedData, *stats = mergeLogic(origData, incData, forceMerge, trimSize, addSolutionTags) + msg(stats[0], "size(s) and", stats[1], "kernel(s) added,", stats[2], "kernel(s) removed") + + with open(os.path.join(outputPath, basename), "w") as outFile: + yaml.safe_dump(mergedData,outFile,default_flow_style=None) + msg("File written to", os.path.join(outputPath, basename)) + msg("------------------------------") + +# partialLogicFilePaths: list of full paths to partial logic files +# outputDir: Directory to write the final result to +# forceMerge: +# trimSize: +# Expects: that all the partial logic files +# have the same base name, but are located +# in different folders. +# Provides: one final logic file that is the +# merged result of all partial files. +# This is useful for when a tuning task is +# shared between multiple machines who each +# will provide a partial result. +def mergePartialLogics(partialLogicFilePaths, outputDir, forceMerge, trimSize=True, addSolutionTags=False): + logicFiles = deepcopy(partialLogicFilePaths) + ensurePath(outputDir) + + baseLogicFile = logicFiles.pop(0) + baseLogicData = loadData(baseLogicFile) + msg("Base logic file:", baseLogicFile) + for f in logicFiles: + forceMerge = defaultForceMergePolicy(f) if forceMerge is None else forceMerge + + msg("Incremental file:", f, "| Merge policy: %s"%("Forced" if forceMerge else "Winner"), "| Trim size:", trimSize) + incLogicData = loadData(f) + + # So far "SolutionIndex" in logic yamls has zero impact on actual 1-1 size mapping (but the order of the Solution does) + # since mergeLogic() takes that value very seriously so we reindex them here so it doesn't choke on duplicated SolutionIndex + baseLogicData = reindexSolutions(baseLogicData) + incLogicData = reindexSolutions(incLogicData) + + mergedData, *stats = mergeLogic(baseLogicData, incLogicData, forceMerge, trimSize, addSolutionTags) + msg(stats[0], "size(s) and", stats[1], "kernel(s) added,", stats[2], "kernel(s) removed") + + # Use the merged data as the base data for the next partial logic file + baseLogicData = deepcopy(mergedData) + + + baseFileName = os.path.basename(baseLogicFile) + outputFilePath = os.path.join(outputDir, baseFileName) + with open(outputFilePath, "w") as outFile: + yaml.safe_dump(baseLogicData, outFile, default_flow_style=None) + msg("File written to", outputFilePath) + msg("------------------------------") + +if __name__ == "__main__": + argParser = argparse.ArgumentParser() + argParser.add_argument("original_dir", help="The library logic directory without tuned sizes") + argParser.add_argument("incremental_dir", help="The incremental logic directory") + argParser.add_argument("output_dir", help="The output logic directory") + argParser.add_argument("-v", "--verbosity", help="0: summary, 1: verbose, 2: debug", default=1, type=int) + argParser.add_argument("--force_merge", help="Merge previously known sizes unconditionally. Default behavior if not arcturus", default="none") + argParser.add_argument("--notrim", help="Do not trim long size format down to short format (m,n,b,k). Default is --trim", action="store_false") + argParser.add_argument("--add_solution_tags", help="Add tags to the size key for solution properies, allowing for solutions with different requirements " + "to exist for the same size. Default doesn't add this tag.", action="store_true") + + args = argParser.parse_args(sys.argv[1:]) + originalDir = args.original_dir + incrementalDir = args.incremental_dir + outputPath = args.output_dir + verbosity = args.verbosity + forceMerge = args.force_merge.lower() + trimSize = args.notrim + add_solution_tags = args.add_solution_tags + + if forceMerge in ["none"]: forceMerge=None + elif forceMerge in ["true", "1"]: forceMerge=True + elif forceMerge in ["false", "0"]: forceMerge=False + + avoidRegressions(originalDir, incrementalDir, outputPath, forceMerge, trimSize, add_solution_tags) diff --git a/tensilelite/Tensile/Utils.py b/tensilelite/Tensile/Utils.py new file mode 100644 index 0000000000..116f5285f5 --- /dev/null +++ b/tensilelite/Tensile/Utils.py @@ -0,0 +1,118 @@ +################################################################################ +# Copyright 2019-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from .Common import ProgressBar + +import functools +import sys + +class SpinnyThing: + def __init__(self): + self.chars = ['|', '/', '-', '\\'] + self.index = 0; + + def increment(self, value=1): + sys.stdout.write('\b' + self.chars[self.index]) + sys.stdout.flush() + self.index = (self.index + 1) % len(self.chars) + + def finish(self): + sys.stdout.write('\b*\n') + sys.stdout.flush() + +def iterate_progress(obj, *args, **kwargs): + try: + progress = ProgressBar(len(obj)) + except TypeError: + progress = SpinnyThing() + for o in obj: + yield o + progress.increment() + progress.finish() + +try: + from tqdm import tqdm +except ImportError: + tqdm = iterate_progress + +def state(obj): + if hasattr(obj, 'state'): + return obj.state() + + if hasattr(obj.__class__, 'StateKeys'): + rv = {} + for key in obj.__class__.StateKeys: + attr = key + if isinstance(key, tuple): + (key, attr) = key + rv[key] = state(getattr(obj, attr)) + return rv + + if isinstance(obj, dict): + return dict([(k, state(v)) for k,v in list(obj.items())]) + + if any([isinstance(obj, cls) for cls in [str, int, float]]): + return obj + + try: + obj = [state(i) for i in obj] + return obj + except TypeError: + pass + + return obj + +def state_key_ordering(cls): + def tup(obj): + return tuple([getattr(obj, k) for k in cls.StateKeys]) + + def lt(a, b): + return tup(a) < tup(b) + def eq(a, b): + return tup(a) == tup(b) + + cls.__lt__ = lt + cls.__eq__ = eq + + return functools.total_ordering(cls) + +def hash_combine(*objs, **kwargs): + shift = 1 + if 'shift' in kwargs: + shift = kwargs['shift'] + + if len(objs) == 1: + objs = objs[0] + + rv = 0 + try: + it = iter(objs) + rv = next(it) + for value in it: + rv = (rv << shift) ^ value + except TypeError: + return objs + except StopIteration: + pass + return rv + +def hash_objs(*objs, **kwargs): + return hash(tuple(objs)) diff --git a/tensilelite/Tensile/__init__.py b/tensilelite/Tensile/__init__.py new file mode 100644 index 0000000000..87a8b79169 --- /dev/null +++ b/tensilelite/Tensile/__init__.py @@ -0,0 +1,33 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +# Even though we don't support python 2, this is still packaged sometimes with python 2. +from __future__ import print_function + +# hardcoded tensile version; also in Tensile/Source/TensileConfigVersion.cmake +__version__ = "4.33.0" + +def PrintTensileRoot(): + import os.path + + print(os.path.dirname(__file__), end='') + + diff --git a/tensilelite/Tensile/bin/Tensile b/tensilelite/Tensile/bin/Tensile new file mode 100755 index 0000000000..8f0cd27a54 --- /dev/null +++ b/tensilelite/Tensile/bin/Tensile @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + + +try: + from Tensile import Tensile +except ImportError: + import os.path + import sys + parentdir = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..")) + sys.path.append(parentdir) + + from Tensile import Tensile + +# script run from commandline +if __name__ == "__main__": + Tensile.main() diff --git a/tensilelite/Tensile/bin/TensileBenchmarkCluster b/tensilelite/Tensile/bin/TensileBenchmarkCluster new file mode 100755 index 0000000000..e4f5634692 --- /dev/null +++ b/tensilelite/Tensile/bin/TensileBenchmarkCluster @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +################################################################################ +# Copyright 2016-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + + +try: + from Tensile import TensileBenchmarkCluster +except ImportError: + import os.path + import sys + parentdir = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..")) + sys.path.append(parentdir) + + from Tensile import TensileBenchmarkCluster + +# script run from commandline +if __name__ == "__main__": + TensileBenchmarkCluster.main() diff --git a/tensilelite/Tensile/bin/TensileClientConfig b/tensilelite/Tensile/bin/TensileClientConfig new file mode 100755 index 0000000000..843d5f7515 --- /dev/null +++ b/tensilelite/Tensile/bin/TensileClientConfig @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + + +try: + from Tensile import TensileClientConfig +except ImportError: + import os.path + import sys + parentdir = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..")) + sys.path.append(parentdir) + + from Tensile import TensileClientConfig + +# script run from commandline +if __name__ == "__main__": + TensileClientConfig.main() diff --git a/tensilelite/Tensile/bin/TensileCreateLibrary b/tensilelite/Tensile/bin/TensileCreateLibrary new file mode 100755 index 0000000000..0a95e72389 --- /dev/null +++ b/tensilelite/Tensile/bin/TensileCreateLibrary @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ +# This script only gets called by CMake + +try: + from Tensile.TensileCreateLibrary import TensileCreateLibrary +except ImportError: + import os.path + import sys + parentdir = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..")) + sys.path.append(parentdir) + + from Tensile.TensileCreateLibrary import TensileCreateLibrary + +################################################################################ +# Main +################################################################################ +if __name__ == "__main__": + TensileCreateLibrary() diff --git a/tensilelite/Tensile/bin/TensileGenerateSummations b/tensilelite/Tensile/bin/TensileGenerateSummations new file mode 100755 index 0000000000..a477434caf --- /dev/null +++ b/tensilelite/Tensile/bin/TensileGenerateSummations @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +try: + from Tensile.GenerateSummations import GenerateSummations +except ImportError: + import os.path + import sys + parentdir = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..")) + sys.path.append(parentdir) + + from Tensile.GenerateSummations import GenerateSummations + +################################################################################ +# Main +################################################################################ +if __name__ == "__main__": + GenerateSummations(sys.argv[1:]) diff --git a/tensilelite/Tensile/bin/TensileRetuneLibrary b/tensilelite/Tensile/bin/TensileRetuneLibrary new file mode 100755 index 0000000000..4b71da9464 --- /dev/null +++ b/tensilelite/Tensile/bin/TensileRetuneLibrary @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +################################################################################ +# Copyright 2016-2021 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + + +try: + from Tensile import TensileRetuneLibrary +except ImportError: + import os.path + import sys + parentdir = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..")) + sys.path.append(parentdir) + + from Tensile import TensileRetuneLibrary + +# script run from commandline +if __name__ == "__main__": + TensileRetuneLibrary.main() diff --git a/tensilelite/Tensile/bin/TensileUpdateLibrary b/tensilelite/Tensile/bin/TensileUpdateLibrary new file mode 100755 index 0000000000..e9f7b39bb9 --- /dev/null +++ b/tensilelite/Tensile/bin/TensileUpdateLibrary @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ +# This script only gets called by CMake + +try: + from Tensile import TensileUpdateLibrary +except ImportError: + import os.path + import sys + parentdir = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..")) + sys.path.append(parentdir) + + from Tensile import TensileUpdateLibrary + +# script run from commandline +if __name__ == "__main__": + TensileUpdateLibrary.main() diff --git a/tensilelite/Tensile/cmake/TensileConfig.cmake b/tensilelite/Tensile/cmake/TensileConfig.cmake new file mode 100644 index 0000000000..164d546023 --- /dev/null +++ b/tensilelite/Tensile/cmake/TensileConfig.cmake @@ -0,0 +1,292 @@ +################################################################################ +# Copyright 2016-2022 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +include(CMakeParseArguments) + +if(NOT DEFINED Tensile_ROOT) + # Compute the installation prefix relative to this file. + get_filename_component(Tensile_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) + get_filename_component(Tensile_PREFIX "${Tensile_PREFIX}" PATH) + + if (WIN32) + execute_process(COMMAND "${Tensile_PREFIX}/bin/TensileGetPath.exe" OUTPUT_VARIABLE Tensile_ROOT) + else() + execute_process(COMMAND "${Tensile_PREFIX}/bin/TensileGetPath" OUTPUT_VARIABLE Tensile_ROOT) + endif() +endif() +list(APPEND CMAKE_MODULE_PATH "${Tensile_ROOT}/Source/cmake/") +list(APPEND CMAKE_MODULE_PATH "${Tensile_ROOT}/Source/") + +if("HIP" IN_LIST Tensile_FIND_COMPONENTS) + set(TENSILE_USE_HIP ON CACHE BOOL "Use HIP") +else() + set(TENSILE_USE_HIP OFF CACHE BOOL "Use HIP") +endif() + +if("LLVM" IN_LIST Tensile_FIND_COMPONENTS) + set(TENSILE_USE_LLVM ON CACHE BOOL "Use LLVM") +else() + set(TENSILE_USE_LLVM OFF CACHE BOOL "Use LLVM") +endif() + +if("Client" IN_LIST Tensile_FIND_COMPONENTS) + if(TENSILE_USE_HIP AND TENSILE_USE_LLVM) + set(TENSILE_BUILD_CLIENT ON CACHE BOOL "Build Client") + elseif(Tensile_FIND_REQUIRED_Client) + message("Tensile client requires both Hip and LLVM.") + set(Tensile_FOUND false) + else() + set(TENSILE_BUILD_CLIENT OFF CACHE BOOL "Build Client") + endif() +else() + set(TENSILE_BUILD_CLIENT OFF CACHE BOOL "Build Client") +endif() + +if("STATIC_ONLY" IN_LIST Tensile_FIND_COMPONENTS) + set(TENSILE_STATIC_ONLY ON CACHE BOOL "Disable exporting symbols from shared library.") +else() + set(TENSILE_STATIC_ONLY OFF CACHE BOOL "Disable exporting symbols from shared library.") +endif() + +add_subdirectory("${Tensile_ROOT}/Source" "Tensile") +include("${Tensile_ROOT}/Source/TensileCreateLibrary.cmake") + +# Target is created for copying dependencies +function(TensileCreateCopyTarget + Target_NAME + Tensile_OBJECTS_TO_COPY + Dest_PATH + ) + + file(MAKE_DIRECTORY "${Dest_PATH}") + add_custom_target( + ${Target_NAME} ALL + COMMENT "${Target_NAME}: Copying tensile objects to ${Dest_PATH}" + DEPENDS ${Tensile_OBJECTS_TO_COPY} + ) + foreach(OBJECT_TO_COPY ${Tensile_OBJECTS_TO_COPY}) + add_custom_command( + TARGET ${Target_NAME} PRE_BUILD + COMMAND_EXPAND_LISTS + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${OBJECT_TO_COPY} ${Dest_PATH} + DEPENDS ${OBJECT_TO_COPY} + ) + endforeach() +endfunction() + +# Output target: ${Tensile_VAR_PREFIX}_LIBRARY_TARGET. Ensures that the libs get built in Tensile_OUTPUT_PATH/library. +# Output symbol: ${Tensile_VAR_PREFIX}_ALL_FILES. List of full paths of all expected library files in manifest. +function(TensileCreateLibraryFiles + Tensile_LOGIC_PATH + Tensile_OUTPUT_PATH + ) + + # Boolean options + set(options + MERGE_FILES + NO_MERGE_FILES + SHORT_FILE_NAMES + PRINT_DEBUG + GENERATE_PACKAGE + SEPARATE_ARCHITECTURES + ) + + # Single value settings + set(oneValueArgs + CODE_OBJECT_VERSION + COMPILER + COMPILER_PATH + EMBED_KEY + EMBED_LIBRARY + LIBRARY_FORMAT + TENSILE_ROOT + VAR_PREFIX + CPU_THREADS + ) + + # Multi value settings + set(multiValueArgs + ARCHITECTURE + ) + + cmake_parse_arguments(Tensile "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if(Tensile_UNPARSED_ARGUMENTS) + message(WARNING "Unrecognized arguments: ${Tensile_UNPARSED_ARGUMENTS}") + endif() + if(Tensile_KEYWORDS_MISSING_VALUES) + message(WARNING "Malformed arguments: ${Tensile_KEYWORDS_MISSING_VALUES}") + endif() + + # Parse incoming options + if(Tensile_TENSILE_ROOT) + set(Script "${Tensile_TENSILE_ROOT}/bin/TensileCreateLibrary") + else() + set(Script "${Tensile_ROOT}/bin/TensileCreateLibrary") + endif() + + message(STATUS "Tensile script: ${Script}") + + # Older NO_MERGE_FILES flag overrides MERGE_FILES option. + if(Tensile_NO_MERGE_FILES) + set(Tensile_MERGE_FILES FALSE) + endif() + + if(Tensile_MERGE_FILES) + set(Options ${Options} "--merge-files") + else() + set(Options ${Options} "--no-merge-files") + endif() + + if(Tensile_SEPARATE_ARCHITECTURES) + set(Options ${Options} "--separate-architectures") + endif() + + if(Tensile_GENERATE_PACKAGE) + set(Options ${Options} "--package-library") + endif() + + if(Tensile_SHORT_FILE_NAMES) + set(Options ${Options} "--short-file-names") + else() + set(Options ${Options} "--no-short-file-names") + endif() + + if(Tensile_PRINT_DEBUG) + set(Options ${Options} "--library-print-debug") + else() + set(Options ${Options} "--no-library-print-debug") + endif() + + if(Tensile_EMBED_LIBRARY) + set(Options ${Options} "--embed-library=${Tensile_EMBED_LIBRARY}") + endif() + + if(Tensile_EMBED_KEY) + set(Options ${Options} "--embed-library-key=${Tensile_EMBED_KEY}") + endif() + + if(Tensile_CODE_OBJECT_VERSION) + set(Options ${Options} "--code-object-version=${Tensile_CODE_OBJECT_VERSION}") + endif() + + if(Tensile_COMPILER) + set(Options ${Options} "--cxx-compiler=${Tensile_COMPILER}") + endif() + + if(Tensile_COMPILER_PATH) + set(Options ${Options} "--cmake-cxx-compiler=${Tensile_COMPILER_PATH}") + endif() + + if(Tensile_CPU_THREADS) + set(Options ${Options} "--jobs=${Tensile_CPU_THREADS}") + endif() + + if(Tensile_LIBRARY_FORMAT) + set(Options ${Options} "--library-format=${Tensile_LIBRARY_FORMAT}") + if(Tensile_LIBRARY_FORMAT MATCHES "yaml") + target_compile_definitions( TensileHost PUBLIC -DTENSILE_YAML=1) + endif() + endif() + + if(Tensile_ARCHITECTURE) + string (REPLACE ";" "_" archString "${Tensile_ARCHITECTURE}") + # uses _ separator to avoid cmake ; list interpretation, either ; or _ decoded in TensileCreateLibrary + set(Options ${Options} "--architecture=${archString}") + endif() + + if (WIN32) + set(CommandLine ${VIRTUALENV_BIN_DIR}/${VIRTUALENV_PYTHON_EXENAME} ${Script} ${Options} ${Tensile_LOGIC_PATH} ${Tensile_OUTPUT_PATH} HIP) + else() + set(CommandLine ${Script} ${Options} ${Tensile_LOGIC_PATH} ${Tensile_OUTPUT_PATH} HIP) + endif() + message(STATUS "Tensile_CREATE_COMMAND: ${CommandLine}") + + if(Tensile_EMBED_LIBRARY) + set(Tensile_EMBED_LIBRARY_SOURCE "${Tensile_OUTPUT_PATH}/library/${Tensile_EMBED_LIBRARY}.cpp") + endif() + + if($ENV{TENSILE_SKIP_LIBRARY}) + message(STATUS "Skipping build of ${Tensile_OUTPUT_PATH}") + else() + + if(NOT Tensile_VAR_PREFIX) + set(Tensile_VAR_PREFIX TENSILE) + endif() + + set(Tensile_MANIFEST_FILE_PATH "${Tensile_OUTPUT_PATH}/library/TensileManifest.txt") + message(STATUS "Tensile_MANIFEST_FILE_PATH: ${Tensile_MANIFEST_FILE_PATH}") + + if($ENV{ENABLE_ADDRESS_SANITIZER}) + # Must populate LD_PRELOAD with ASAN runtime if ASAN is being used. + # Find the ASAN RT with compiler and update env for Tensile call. + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.asan-x86_64.so + OUTPUT_VARIABLE ASAN_LIB_PATH + COMMAND_ECHO STDOUT) + string(STRIP ${ASAN_LIB_PATH} ASAN_LIB_PATH) + set(CommandLine env LD_PRELOAD=${ASAN_LIB_PATH} ${CommandLine}) + endif() + + # Create the manifest file of the output libraries. + set(Tensile_CREATE_MANIFEST_COMMAND ${CommandLine} "--generate-manifest-and-exit") + execute_process( + COMMAND ${Tensile_CREATE_MANIFEST_COMMAND} + RESULT_VARIABLE Tensile_CREATE_MANIFEST_RESULT + COMMAND_ECHO STDOUT) + + if(Tensile_CREATE_MANIFEST_RESULT OR (NOT EXISTS ${Tensile_MANIFEST_FILE_PATH})) + message(FATAL_ERROR "Error creating Tensile library: ${Tensile_CREATE_MANIFEST_RESULT}") + endif() + + # Defer the actual call of the TensileCreateLibraries to 'make' time as needed. + # Read the manifest file and declare the files as expected output. + file(STRINGS ${Tensile_MANIFEST_FILE_PATH} Tensile_MANIFEST_CONTENTS) + add_custom_command( + COMMENT "Generating Tensile Libraries" + OUTPUT ${Tensile_EMBED_LIBRARY_SOURCE};${Tensile_MANIFEST_CONTENTS} + COMMAND ${CommandLine} + ) + + set("${Tensile_VAR_PREFIX}_ALL_FILES" ${Tensile_MANIFEST_CONTENTS} PARENT_SCOPE) + + # Create a chained library build target. + # We've declared the manifest contents as output of the custom + # command above which builds the tensile libs. Now create a + # target dependency on those files so that we force the custom + # command to be invoked at build time, not cmake time. + TensileCreateCopyTarget( + "${Tensile_VAR_PREFIX}_LIBRARY_TARGET" + "${Tensile_MANIFEST_CONTENTS}" + "${Tensile_OUTPUT_PATH}/library" + ) + + endif() + + if(Tensile_EMBED_LIBRARY) + + add_library(${Tensile_EMBED_LIBRARY} ${Tensile_EMBED_LIBRARY_SOURCE}) + target_link_libraries(${Tensile_EMBED_LIBRARY} PUBLIC TensileHost) + + endif() + +endfunction() + diff --git a/tensilelite/Tensile/cmake/TensileConfigVersion.cmake b/tensilelite/Tensile/cmake/TensileConfigVersion.cmake new file mode 100644 index 0000000000..587423359a --- /dev/null +++ b/tensilelite/Tensile/cmake/TensileConfigVersion.cmake @@ -0,0 +1,90 @@ +################################################################################ +# Copyright 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +# hardcoded tensile version; also in Tensile/__init__.py +set(TENSILE_VERSION_MAJOR 4) +set(TENSILE_VERSION_MINOR 33) +set(TENSILE_VERSION_PATCH 0) + +# export version +set(PACKAGE_VERSION "${TENSILE_VERSION_MAJOR}.${TENSILE_VERSION_MINOR}.${TENSILE_VERSION_PATCH}") + +if(PACKAGE_FIND_VERSION VERSION_EQUAL PACKAGE_VERSION) + set(PACKAGE_VERSION_EXACT TRUE) +else() + set(PACKAGE_VERSION_EXACT FALSE) +endif() + +if(PACKAGE_VERSION_EXACT} OR (PACKAGE_FIND_VERSION VERSION_GREATER PACKAGE_VERSION)) + set(PACKAGE_VERSION_COMPATIBLE TRUE) +else() + set(PACKAGE_VERSION_COMPATIBLE FALSE) +endif() + +if(PACKAGE_VERSION_COMPATIBLE) + set(PACKAGE_VERSION_UNSUTABLE FALSE) +else() + set(PACKAGE_VERSION_UNSUTABLE TRUE) +endif() + +if(false) +# set to compatible, and switch to false below if necessary +set(PACKAGE_VERSION_EXACT TRUE) +set(PACKAGE_VERSION_COMPATIBLE TRUE) +set(PACKAGE_VERSION_UNSUITABLE FALSE) + +# if major doesn't match +if (NOT PACKAGE_FIND_VERSION_MAJOR STREQUAL TENSILE_VERSION_MAJOR) + + set(PACKAGE_VERSION_EXACT FALSE) + set(PACKAGE_VERSION_COMPATIBLE FALSE) + set(PACKAGE_VERSION_UNSUITABLE TRUE) + return() +endif() + +# if minor insufficient +if (PACKAGE_FIND_VERSION_MINOR STRGREATER TENSILE_VERSION_MINOR) + + set(PACKAGE_VERSION_EXACT FALSE) + set(PACKAGE_VERSION_COMPATIBLE FALSE) + set(PACKAGE_VERSION_UNSUITABLE TRUE) + return() +endif() + +# if minor==minor but patch insufficient +if (PACKAGE_FIND_VERSION_MINOR STREQUAL TENSILE_VERSION_MINOR) + if (PACKAGE_FIND_VERSION_PATCH STRGREATER TENSILE_VERSION_PATCH) + + set(PACKAGE_VERSION_EXACT FALSE) + set(PACKAGE_VERSION_COMPATIBLE FALSE) + set(PACKAGE_VERSION_UNSUITABLE TRUE) + return() + endif() +endif() + +# check exactness +if (NOT (PACKAGE_FIND_VERSION_MINOR STREQUAL TENSILE_VERSION_MINOR + AND PACKAGE_FIND_VERSION_MINOR STREQUAL TENSILE_VERSION_MINOR) ) + + set(PACKAGE_VERSION_EXACT FALSE) +endif() +endif() + diff --git a/tensilelite/bump-version.sh b/tensilelite/bump-version.sh new file mode 100755 index 0000000000..4969beb08b --- /dev/null +++ b/tensilelite/bump-version.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# This script needs to be edited to bump version for new release. +# Version will be bumped in Tensile/__init__.py and in .yaml files + +OLD_VERSION="4.32.1" +NEW_VERSION="4.33.0" + +OLD_MINIMUM_REQUIRED_VERSION="MinimumRequiredVersion: 4.7.2" +NEW_MINIMUM_REQUIRED_VERSION="MinimumRequiredVersion: 4.8.0" + +sed -i "s/${OLD_VERSION}/${NEW_VERSION}/g" Tensile/__init__.py +sed -i "s/${OLD_VERSION}/${NEW_VERSION}/g" HostLibraryTests/CMakeLists.txt + +echo "The version number also needs to be fixed in Tensile/cmake/TensileConfigVersion.cmake ." + +#only update when there is a major version change +#for FILE in Tensile/Configs/*yaml +#do +# sed -i "s/${OLD_MINIMUM_REQUIRED_VERSION}/${NEW_MINIMUM_REQUIRED_VERSION}/" $FILE +#done diff --git a/tensilelite/pytest.ini b/tensilelite/pytest.ini new file mode 100644 index 0000000000..6ea8d81ed5 --- /dev/null +++ b/tensilelite/pytest.ini @@ -0,0 +1,100 @@ +[pytest] +junit_family = xunit2 +junit_logging = all +junit_log_passing_tests = False +xfail_strict = True +markers = + extended: Longer tests, formerly nightly. + pre_checkin: All pre-checkin tests. + disabled: Disabled tests. + regression: directory: tests for some bug hotspots + + bufferload_offset: directory + bugs: directory + dot: directory + nightly: directory + no_load_loop: Directory, tests for the OptNoLoadLoop option. + special: directory + unit: Unit tests + use_initial_strides: directory + use_initial_strides_cd: directory + vega_20: directory + weekly: directory + yaml_only: directory, all tests which simply run a YAML file. + integration: + denorm: directory, tests for denorm behavior + emulation: Quick set of essential tests to run on emulation. + emulation1: Emulation tests part 1 + emulation2: Emulation tests part 2 + emulation3: Emulation tests part 3 + emulation4: Emulation tests part 4 + emulation5: Emulation tests part 5 + + + validate: All tests which validate the results. + validateAll: All tests which validate all data points. + GEMM: All tests which run standard GEMMs + Assembly: Tests with assembly kernels. + bfloat16: Data type + complexDouble : Data type + complexSingle : Data type + double : Data type + half : Data type + int8x4 : Data type + int8 : Data type + single: Data type + activation + assertions + big_tensor + classic + classic_source + direct_to_lds + dot2 + double_complex + fast + flat + float_complex + fractional + global_split_u + global_split_u_src + hpa_source + igemm + local_split_u + mfma + multi_sum + multi_sum_psd + nonbatched + pack_tensor_dims + pbd + source + stagger_u + syntax_error + tensor_contraction + vector_width + wip + bufferload_offset + mirror_dims + xfail-gfx000: architecture + xfail-gfx900: architecture + xfail-gfx906: architecture + xfail-gfx908: architecture + xfail-gfx90a: architecture + xfail-gfx1010: architecture + xfail-gfx1011: architecture + xfail-gfx1012: architecture + xfail-gfx1030: architecture + xfail-gfx1100: architecture + xfail-gfx1101: architecture + xfail-gfx1102: architecture + skip-gfx000: architecture + skip-gfx900: architecture + skip-gfx906: architecture + skip-gfx908: architecture + skip-gfx90a: architecture + skip-gfx1010: architecture + skip-gfx1011: architecture + skip-gfx1012: architecture + skip-gfx1030: architecture + skip-gfx1100: architecture + skip-gfx1101: architecture + skip-gfx1102: architecture diff --git a/tensilelite/setup.py b/tensilelite/setup.py new file mode 100644 index 0000000000..b6818d75b2 --- /dev/null +++ b/tensilelite/setup.py @@ -0,0 +1,56 @@ +################################################################################ +# Install Tensile +# - installs python scripts, c++ source files and a few configs +# - creates executables for running benchmarking +# - installs TensileConfig.cmake so one call find_package(Tensile) +################################################################################ +from setuptools import setup +import os.path +import re + +def readRequirementsFromTxt(): + requirements = [] + with open("requirements.txt") as req_file: + for line in req_file.read().splitlines(): + if not line.strip().startswith("#"): + requirements.append(line) + return requirements + +def readVersionFromInit(): + import Tensile + return Tensile.__version__ + +setup( + name="Tensile", + version=readVersionFromInit(), + description="An auto-tuning tool for GEMMs and higher-dimensional tensor contractions on GPUs.", + url="https://github.com/RadeonOpenCompute/Tensile", + author="Advanced Micro Devices", + license="MIT", + install_requires=readRequirementsFromTxt(), + python_requires='>=3.5', + packages=["Tensile"], + package_data={ "Tensile": ["Tensile/cmake/*"] }, + data_files=[ ("cmake", ["Tensile/cmake/TensileConfig.cmake", "Tensile/cmake/TensileConfigVersion.cmake"]) ], + include_package_data=True, + entry_points={"console_scripts": [ + # user runs a benchmark + "Tensile = Tensile.Tensile:main", + "tensileBenchmarkLibraryClient = Tensile.TensileBenchmarkLibraryClient:main", + # CMake calls this to create Tensile.lib + "TensileCreateLibrary = Tensile.TensileCreateLibrary:TensileCreateLibrary", + + "TensileGetPath = Tensile:PrintTensileRoot", + # automatic benchmarking for rocblas + "tensile_rocblas_sgemm = Tensile.Tensile:TensileROCBLASSGEMM", + "tensile_rocblas_dgemm = Tensile.Tensile:TensileROCBLASDGEMM", + "tensile_rocblas_cgemm = Tensile.Tensile:TensileROCBLASCGEMM", + "tensile_rocblas_zgemm = Tensile.Tensile:TensileROCBLASZGEMM", + # automatically find fastest sgemm exhaustive search + "tensile_sgemm = Tensile.Tensile:TensileSGEMM5760", + # Run tensile benchmark from cluster + "TensileBenchmarkCluster = Tensile.TensileBenchmarkCluster:main", + # Retune library logic file + "TensileRetuneLibrary = Tensile.TensileRetuneLibrary:main" + ]} + ) diff --git a/tensilelite/tox.ini b/tensilelite/tox.ini new file mode 100644 index 0000000000..eb965b16e8 --- /dev/null +++ b/tensilelite/tox.ini @@ -0,0 +1,42 @@ +[tox] +envlist = py35,py36,py27,lint + +[testenv] +# Some versions of Pytest versions have a bug: +# https://github.com/pytest-dev/pytest/issues/5971 which causes the whole test +# process to crash if a multiprocessing job has an exception. Fixed in 5.3.3. +deps = + -r{toxinidir}/requirements.txt + pytest>=5.4.1 + pytest-xdist>=1.32.0 + filelock +commands = + python3 ./Tensile/bin/Tensile Tensile/Configs/build_client.yaml {envdir}/client + py.test -v --basetemp={envtmpdir} --junit-xml={toxinidir}/python_tests.xml --junit-prefix={envname} --color=yes -n 4 --prebuilt-client={envdir}/client/0_Build/client/tensile_client {posargs} + +[testenv:lint] +basepython = python3 +deps = + -r{toxinidir}/requirements.txt + flake8 +commands = flake8 {toxinidir}/Tensile + +[testenv:pre_commit] +basepython = python3 +deps = + -r{toxinidir}/requirements.txt + pytest>=5.4.1 + flake8 +commands = + flake8 {toxinidir}/Tensile + py.test -q --basetemp={envtmpdir} --color=yes {toxinidir}/Tensile -m unit {posargs} + +[flake8] +exclude = .git,build*,dist,.cache,*~ +max-line-length = 132 +ignore = + # All E class violations are errors reported by pycodestyle + E, + # All W class violations are warnings reported by pycodestyle + W + # This mostly just leaves pyflakes errors.