diff --git a/projects/hipblaslt/tensilelite/CMakeLists.txt b/projects/hipblaslt/tensilelite/CMakeLists.txt index 19430411d879..bcc0b8ab9c56 100644 --- a/projects/hipblaslt/tensilelite/CMakeLists.txt +++ b/projects/hipblaslt/tensilelite/CMakeLists.txt @@ -115,6 +115,7 @@ set(TENSILELITE_HOST_SOURCES ) set(TENSILELITE_HOST_MSGPACK_SOURCES "${CMAKE_CURRENT_LIST_DIR}/src/msgpack/MessagePack.cpp" + "${CMAKE_CURRENT_LIST_DIR}/src/msgpack/SolutionState.cpp" "${CMAKE_CURRENT_LIST_DIR}/src/msgpack/Writing.cpp" ) diff --git a/projects/hipblaslt/tensilelite/include/CMakeLists.txt b/projects/hipblaslt/tensilelite/include/CMakeLists.txt index 2f47a1664675..dae1f5b57547 100644 --- a/projects/hipblaslt/tensilelite/include/CMakeLists.txt +++ b/projects/hipblaslt/tensilelite/include/CMakeLists.txt @@ -10,6 +10,7 @@ target_sources(tensilelite-host "${CMAKE_CURRENT_SOURCE_DIR}/Tensile/AMDGPUPredicates.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Tensile/AMDGPU_Detail.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Tensile/Activation.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Tensile/Architectures.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Tensile/CachingLibrary.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Tensile/Comparison.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Tensile/ContractionLibrary.hpp" @@ -124,6 +125,7 @@ else() FILE_SET tensilelite_public_headers TYPE HEADERS FILES "${CMAKE_CURRENT_SOURCE_DIR}/Tensile/msgpack/MessagePack.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Tensile/msgpack/SolutionState.hpp" PRIVATE FILE_SET tensilelite_private_headers TYPE HEADERS FILES diff --git a/projects/hipblaslt/tensilelite/include/Tensile/Architectures.hpp b/projects/hipblaslt/tensilelite/include/Tensile/Architectures.hpp new file mode 100644 index 000000000000..d9789d234c80 --- /dev/null +++ b/projects/hipblaslt/tensilelite/include/Tensile/Architectures.hpp @@ -0,0 +1,31 @@ +// Copyright Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include + +namespace TensileLite +{ + // PCI device ids of the GPUs TensileLite builds libraries for, one list for + // every C++ caller. SUPPORTED_BUILD_CHIP_IDS in + // tensilelite/Common/Architectures.py is the source of the values; the + // catalog tests (test_solution_state_catalog.py and + // SolutionRecord.CatalogMatchesSolutionState) fail when the two lists + // differ. + inline constexpr std::array SupportedBuildChipIds = {"74a0", + "74a1", + "74a2", + "74a3", + "74a5", + "74a9", + "75a0", + "75b0", + "75a2", + "75b2", + "75a3", + "75b3", + "75a8", + "75b8"}; +} // namespace TensileLite diff --git a/projects/hipblaslt/tensilelite/include/Tensile/msgpack/SolutionState.hpp b/projects/hipblaslt/tensilelite/include/Tensile/msgpack/SolutionState.hpp new file mode 100644 index 000000000000..d4c128d9cb9c --- /dev/null +++ b/projects/hipblaslt/tensilelite/include/Tensile/msgpack/SolutionState.hpp @@ -0,0 +1,49 @@ +// Copyright Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include +#include + +#include + +#include + +#include + +namespace TensileLite +{ + // The static catalogs that SolutionState.cpp accepts when it rebuilds a + // record from state. + TENSILELITEHOST_EXPORT std::vector supportedBuildChipIds(); + TENSILELITEHOST_EXPORT std::vector supportedDataTypeNames(); + + /** + * Builds the library record of one fully derived solution state, the map + * Solution.FromOriginalState receives after defaults and derived parameters + * are filled in. + * + * The input is not a raw custom.config delta and not an unexpanded logic + * entry. It must already contain ProblemType.TotalIndices, + * IndicesSummation, IndicesFree, IndicesBatch, IndexAssignmentsA, + * IndexAssignmentsB, and the other derived fields the record reads. A + * metadata note may carry that derived map. A caller that passes an + * unexpanded map gets a missing-key error. + * + * The assembler types each scalar of a metadata note from its YAML text, + * so an empty string arrives as 0.0 and a null as the string "null"; both + * are read as the values the state holds. + * + * WorkgroupMappingXCCCheck stores only its value. Its compute-unit count + * stays 0 until the predicate is evaluated, which is when the device is + * read. Building the record does not query a GPU. + * + * Throws std::runtime_error naming the key when a key the record needs is + * missing or holds a value of the wrong kind. + */ + TENSILELITEHOST_EXPORT std::shared_ptr + solutionFromState(msgpack::object const& state); +} diff --git a/projects/hipblaslt/tensilelite/src/msgpack/SolutionState.cpp b/projects/hipblaslt/tensilelite/src/msgpack/SolutionState.cpp new file mode 100644 index 000000000000..ae281b5b9c2e --- /dev/null +++ b/projects/hipblaslt/tensilelite/src/msgpack/SolutionState.cpp @@ -0,0 +1,2088 @@ +// Copyright Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +// ContractionProblemGemm declares `enum CONST`, a name the Windows SDK headers +// define as a macro, and msgpack.hpp reaches those headers. Every header that +// declares the enum therefore has to be seen before , which is why +// this block sits above the one holding SolutionState.hpp. clang-format sorts +// within an include block and preserves the order of the blocks themselves. +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace TensileLite +{ + namespace + { + using msgpack::type::object_type; + + using ProblemPredicate = Predicates::Predicate; + using TaskPredicate = Predicates::Predicate; + using HardwarePredicate = Predicates::Predicate; + using AMDGPUPredicate = Predicates::Predicate; + + // internalParameters["GlobalSplitUPGR"] in Tensile/Common/GlobalParameters.py. + constexpr size_t GlobalSplitUPGR = 16; + + // DataType.properties in Tensile/Common/DataType.py, by enum value: the + // enum name and char it is looked up by, and whether numBytes() > 1. + struct DataTypeProperties + { + char const* name; + char const* abbrev; + bool wide; + }; + constexpr DataTypeProperties DataTypes[] = { + {"Float", "S", true}, + {"Double", "D", true}, + {"ComplexFloat", "C", true}, + {"ComplexDouble", "Z", true}, + {"Half", "H", true}, + {"Int8x4", "4xi8", true}, + {"Int32", "I", true}, + {"BFloat16", "B", true}, + {"Int8", "I8", false}, + {"Int64", "I64", true}, + {"XFloat32", "X", true}, + {"Float8_fnuz", "F8N", false}, + {"BFloat8_fnuz", "B8N", false}, + {"Float8BFloat8_fnuz", "F8B8N", false}, + {"BFloat8Float8_fnuz", "B8F8N", false}, + {"Float8", "F8", false}, + {"BFloat8", "B8", false}, + {"Float8BFloat8", "F8B8", false}, + {"BFloat8Float8", "B8F8", false}, + {"Float6", "F6", false}, + {"BFloat6", "B6", false}, + {"Float4", "F4", false}, + {"E8", "E8", false}, + {"E5M3", "E5M3", false}, + {"Int4", "I4", false}, + }; + static_assert(std::size(DataTypes) == static_cast(rocisa::DataType::Count)); + + std::string_view view(msgpack::object const& object) + { + return {object.via.str.ptr, object.via.str.size}; + } + + bool isFloat(msgpack::object const& object) + { + return object.type == object_type::FLOAT32 || object.type == object_type::FLOAT64; + } + + // One value of the state and where it sits, read with Python's semantics. + class Value + { + public: + Value(msgpack::object const& object, std::string path) + : m_object(&object) + , m_path(std::move(path)) + { + } + + msgpack::object const& object() const + { + return *m_object; + } + + std::string const& path() const + { + return m_path; + } + + [[noreturn]] void fail(std::string const& what) const + { + throw std::runtime_error("Solution state " + (m_path.empty() ? "" : m_path + " ") + + "is not " + what); + } + + bool isNone() const + { + return m_object->type == object_type::NIL + || (m_object->type == object_type::STR && view(*m_object) == "null"); + } + + bool isNumber() const + { + switch(m_object->type) + { + case object_type::BOOLEAN: + case object_type::POSITIVE_INTEGER: + case object_type::NEGATIVE_INTEGER: + return true; + default: + return isFloat(*m_object); + } + } + + bool truthy() const + { + if(isNone()) + return false; + switch(m_object->type) + { + case object_type::BOOLEAN: + return m_object->via.boolean; + case object_type::POSITIVE_INTEGER: + return m_object->via.u64 != 0; + case object_type::NEGATIVE_INTEGER: + return true; + case object_type::FLOAT32: + case object_type::FLOAT64: + return m_object->via.f64 != 0; + case object_type::STR: + return m_object->via.str.size != 0; + case object_type::BIN: + return m_object->via.bin.size != 0; + case object_type::ARRAY: + return m_object->via.array.size != 0; + case object_type::MAP: + return m_object->via.map.size != 0; + default: + return true; + } + } + + int64_t integer() const + { + switch(m_object->type) + { + case object_type::BOOLEAN: + return m_object->via.boolean ? 1 : 0; + case object_type::POSITIVE_INTEGER: + if(m_object->via.u64 > uint64_t(std::numeric_limits::max())) + fail("a 64-bit integer"); + return int64_t(m_object->via.u64); + case object_type::NEGATIVE_INTEGER: + return m_object->via.i64; + default: + fail("an integer"); + } + } + + double number() const + { + if(!isNumber()) + fail("a number"); + return isFloat(*m_object) ? m_object->via.f64 : double(integer()); + } + + std::string text() const + { + if(m_object->type == object_type::STR) + return std::string(view(*m_object)); + if(isFloat(*m_object) && m_object->via.f64 == 0) + return ""; + fail("a string"); + } + + bool isText(std::string_view text) const + { + return m_object->type == object_type::STR && view(*m_object) == text; + } + + bool equals(double number) const + { + return isNumber() && this->number() == number; + } + + // Python's value == other. + bool equals(Value const& other) const + { + if(isNumber() && other.isNumber()) + return number() == other.number(); + return *m_object == other.object(); + } + + std::vector list() const + { + if(m_object->type != object_type::ARRAY) + fail("a list"); + std::vector rv; + rv.reserve(m_object->via.array.size); + for(uint32_t i = 0; i < m_object->via.array.size; i++) + rv.emplace_back(m_object->via.array.ptr[i], + m_path + "[" + std::to_string(i) + "]"); + return rv; + } + + std::vector integers() const + { + std::vector rv; + for(auto const& item : list()) + rv.push_back(item.integer()); + return rv; + } + + rocisa::DataType dataType() const + { + if(m_object->type == object_type::STR) + { + std::string name = lower(text()); + for(size_t i = 0; i < std::size(DataTypes); i++) + if(lower(DataTypes[i].name) == name || lower(DataTypes[i].abbrev) == name) + return static_cast(i); + fail("a data type"); + } + auto value = integer(); + if(value < 0 || value >= static_cast(rocisa::DataType::Count)) + fail("a data type"); + return static_cast(value); + } + + static std::string lower(std::string text) + { + std::transform(text.begin(), text.end(), text.begin(), [](unsigned char c) { + return std::tolower(c); + }); + return text; + } + + private: + msgpack::object const* m_object; + std::string m_path; + }; + + // A map of the state, keeping the stored order Python iterates it in. + class Dict + { + public: + explicit Dict(Value const& value) + : m_path(value.path()) + { + if(value.object().type != object_type::MAP) + value.fail("a map"); + auto const& map = value.object().via.map; + for(uint32_t i = 0; i < map.size; i++) + { + auto const& key = map.ptr[i].key; + if(key.type != object_type::STR) + value.fail("a map with string keys"); + std::string name(view(key)); + m_items.emplace_back(name, Value(map.ptr[i].val, child(name))); + } + } + + std::vector> const& items() const + { + return m_items; + } + + Value const* find(std::string_view key) const + { + for(auto const& item : m_items) + if(item.first == key) + return &item.second; + return nullptr; + } + + bool has(std::string_view key) const + { + return find(key) != nullptr; + } + + Value const& operator[](std::string_view key) const + { + if(auto const* value = find(key)) + return *value; + throw std::runtime_error("Solution state has no " + child(std::string(key))); + } + + Dict dict(std::string_view key) const + { + return Dict((*this)[key]); + } + + int64_t integer(std::string_view key) const + { + return (*this)[key].integer(); + } + + int64_t integer(std::string_view key, int64_t fallback) const + { + auto const* value = find(key); + return value ? value->integer() : fallback; + } + + bool truthy(std::string_view key) const + { + return (*this)[key].truthy(); + } + + bool truthy(std::string_view key, bool fallback) const + { + auto const* value = find(key); + return value ? value->truthy() : fallback; + } + + std::string text(std::string_view key) const + { + return (*this)[key].text(); + } + + rocisa::DataType dataType(std::string_view key) const + { + return (*this)[key].dataType(); + } + + private: + std::string child(std::string const& key) const + { + return m_path.empty() ? key : m_path + "." + key; + } + + std::string m_path; + std::vector> m_items; + }; + + Dict emptyDict() + { + static msgpack::object const map = [] { + msgpack::object rv; + rv.type = object_type::MAP; + rv.via.map.size = 0; + rv.via.map.ptr = nullptr; + return rv; + }(); + return Dict(Value(map, "")); + } + + // Python's a / b, which raises on a zero divisor. + double quotient(Value const& a, Value const& b) + { + if(b.number() == 0) + b.fail("a nonzero divisor"); + return a.number() / b.number(); + } + + // Python's a // b on integers. + int64_t floorQuotient(Value const& a, Value const& b) + { + auto numerator = a.integer(); + auto denominator = b.integer(); + if(denominator == 0) + b.fail("a nonzero divisor"); + auto rv = numerator / denominator; + if(numerator % denominator != 0 && ((numerator < 0) != (denominator < 0))) + rv--; + return rv; + } + + template + Enum enumByName(Value const& value, int count, Name name, char const* what) + { + auto text = value.text(); + for(int i = 0; i < count; i++) + if(name(static_cast(i)) == text) + return static_cast(i); + value.fail(what); + } + + // ActivationType.lookup in Tensile/Activation.py, in order. + struct ActivationEntry + { + char const* name; + bool isGradient; + int extraArgs; + // Half, Single, Double, BFloat16, Int8, Int16, Int32. + std::array can; + bool tensile; + bool hipblaslt; + }; + + constexpr ActivationEntry Activations[] = { + {"none", false, 0, {true, true, true, true, true, true, true}, true, true}, + {"abs", false, 0, {true, true, true, true, false, false, true}, true, false}, + {"clippedrelu", false, 2, {true, true, true, false, false, false, true}, true, false}, + {"gelu", false, 0, {true, true, false, false, false, false, false}, true, true}, + {"leakyrelu", false, 1, {true, true, true, false, false, false, true}, true, false}, + {"relu", false, 0, {true, true, true, false, false, false, true}, true, true}, + {"sigmoid", false, 0, {true, true, false, false, false, false, false}, true, true}, + {"tanh", false, 2, {true, true, false, false, false, false, false}, true, false}, + {"dgelu", true, 0, {false, true, false, false, false, false, false}, true, true}, + {"drelu", true, 0, {false, true, false, false, false, false, false}, true, true}, + {"geluscaling", false, 1, {true, true, false, false, false, false, false}, true, false}, + {"silu", false, 0, {true, true, false, false, false, false, false}, true, true}, + {"swish", false, 1, {true, true, false, false, false, false, false}, true, false}, + {"clamp", false, 2, {true, true, true, false, false, false, true}, true, true}, + {"hipblaslt_all", false, 0, {}, false, true}, + {"all", false, 0, {}, true, true}, + }; + + // ActivationType.lookupVeri. + constexpr char const* VerificationActivation = "exp"; + + ActivationEntry const* findActivation(std::string const& name) + { + for(auto const& entry : Activations) + if(name == entry.name) + return &entry; + return nullptr; + } + + // ActivationType(value): the lower-case name. + std::string activationName(Value const& value) + { + if(value.object().type != object_type::STR) + value.fail("an activation type"); + auto name = Value::lower(value.text()); + if(!findActivation(name) && name != VerificationActivation) + value.fail("an activation type"); + return name; + } + + bool isAllActivations(std::string const& name) + { + return name == "all" || name == "hipblaslt_all"; + } + + // ActivationType.getAdditionalArgNum() for Export.NORMAL. + int additionalArgNum(std::string const& name) + { + if(isAllActivations(name)) + { + int rv = 0; + for(auto const& entry : Activations) + if(!entry.isGradient) + rv = std::max(rv, entry.extraArgs); + return rv; + } + auto const* entry = findActivation(name); + return entry ? entry->extraArgs : 0; + } + + bool typeAvailable(ActivationEntry const& entry, rocisa::DataType type) + { + return (type == rocisa::DataType::Half && entry.can[0]) + || (type == rocisa::DataType::Float && entry.can[1]) + || (type == rocisa::DataType::Double && entry.can[2]) + || (type == rocisa::DataType::BFloat16 && entry.can[3]) + || (type == rocisa::DataType::Int8 && entry.can[4]) + || (type == rocisa::DataType::Int32 && entry.can[6]); + } + + // The C++ activation an ActivationType state names, by its capitalize(). + ActivationType activationType(std::string const& name) + { + std::string capitalized = name; + if(!capitalized.empty()) + capitalized[0] = std::toupper(static_cast(capitalized[0])); + for(int i = 0; i < static_cast(ActivationType::Count); i++) + if(ToString(static_cast(i)) == capitalized) + return static_cast(i); + throw std::runtime_error("No activation type is named " + capitalized); + } + + // ActivationType.getEnumStrList(dataType, supportedBy, exportType=...). + std::vector + activationEnumList(rocisa::DataType type, bool all, bool gradientOnly) + { + std::vector rv; + for(auto const& entry : Activations) + { + if(gradientOnly ? !entry.isGradient : entry.isGradient) + continue; + if(isAllActivations(entry.name)) + continue; + bool supported = entry.hipblaslt || (all && entry.tensile); + if(typeAvailable(entry, type) && supported) + rv.push_back(activationType(entry.name)); + } + return rv; + } + + bool wideType(rocisa::DataType type) + { + return DataTypes[static_cast(type)].wide; + } + + // list.sort() on a list of data types as the state spells them, then + // DataType() on each. + std::vector sortedDataTypes(Value const& value) + { + auto items = value.list(); + bool texts = !items.empty() && items.front().object().type == object_type::STR; + for(auto const& item : items) + if((item.object().type == object_type::STR) != texts) + value.fail("a list of data types of one kind"); + std::stable_sort(items.begin(), items.end(), [texts](Value const& a, Value const& b) { + return texts ? a.text() < b.text() : a.number() < b.number(); + }); + std::vector rv; + for(auto const& item : items) + rv.push_back(item.dataType()); + return rv; + } + + // sorted(set(types)). + std::vector uniqueSorted(std::vector types) + { + std::sort(types.begin(), types.end()); + types.erase(std::unique(types.begin(), types.end()), types.end()); + return types; + } + + // getBiasDataTypeListDefault in Tensile/SolutionStructs/Problem.py. + std::vector biasDataTypeListDefault(Dict const& problem) + { + std::vector types; + for(auto key : {"DataType", "ComputeDataType", "DestDataType"}) + { + auto type = problem.dataType(key); + if(wideType(type)) + types.push_back(type); + } + return uniqueSorted(types); + } + + // getGateResidualDataTypeListDefault in Tensile/SolutionStructs/Problem.py. + std::vector gateResidualDataTypeListDefault(Dict const& problem) + { + return uniqueSorted( + {problem.dataType("DataType"), problem.dataType("ComputeDataType")}); + } + + template + void assignSequence(Value const& value, T& target) + { + auto items = value.list(); + if(items.size() > 3) + value.fail("a list of at most 3 values"); + using Element = decltype(target.x); + if(items.size() > 0) + target.x = static_cast(items[0].integer()); + if(items.size() > 1) + target.y = static_cast(items[1].integer()); + if(items.size() > 2) + target.z = static_cast(items[2].integer()); + } + + template + void assignArray(Value const& value, std::array& target) + { + auto items = value.list(); + if(items.size() > N) + value.fail("a list of at most " + std::to_string(N) + " values"); + for(size_t i = 0; i < items.size(); i++) + target[i] = static_cast(items[i].integer()); + } + + template + std::shared_ptr

withValue(V value) + { + auto rv = std::make_shared

(); + rv->value = std::move(value); + return rv; + } + + template + std::shared_ptr

withIndexValue(I index, V value) + { + auto rv = std::make_shared

(); + rv->index = index; + rv->value = std::move(value); + return rv; + } + + // Properties.Predicate.And. + template + std::shared_ptr> + conjunction(std::vector>> predicates) + { + if(predicates.empty()) + return std::make_shared>(); + if(predicates.size() == 1) + return predicates.front(); + return std::make_shared>(predicates); + } + + // Properties.Predicate.Or. + template + std::shared_ptr> + disjunction(std::vector>> predicates) + { + if(predicates.empty()) + return std::make_shared>(); + if(predicates.size() == 1) + return predicates.front(); + return std::make_shared>(predicates); + } + + // Tensile/ExecutionPolicy.py. + struct ExecutionPolicy + { + TileProcessingStrategy strategy = TileProcessingStrategy::None; + WorkAssignment assignment = WorkAssignment::StaticGrid; + + bool persistent() const + { + return strategy != TileProcessingStrategy::None; + } + bool persistentDataParallel() const + { + return strategy == TileProcessingStrategy::DataParallel; + } + bool streamK() const + { + return strategy == TileProcessingStrategy::StreamK; + } + }; + + // The selectors resolve_policy reads, by value. + struct PolicySelectors + { + std::string strategy = "None"; + std::string assignment = "StaticGrid"; + Value const* persistentLoop = nullptr; + }; + + ExecutionPolicy resolvePolicy(PolicySelectors const& selectors) + { + ExecutionPolicy policy; + try + { + policy.strategy = parseTileProcessingStrategy(selectors.strategy); + } + catch(std::runtime_error const&) + { + throw std::runtime_error( + "TileProcessingStrategy must be 'None', DataParallel, or StreamK"); + } + try + { + policy.assignment = parseWorkAssignment(selectors.assignment); + } + catch(std::runtime_error const&) + { + throw std::runtime_error( + "WorkAssignment must be StaticGrid, DynamicWorkQueue, or Hybrid"); + } + if(policy.strategy == TileProcessingStrategy::None) + policy.assignment = WorkAssignment::StaticGrid; + else if(policy.persistentDataParallel() + && policy.assignment != WorkAssignment::StaticGrid) + throw std::runtime_error(std::string(toString(policy.strategy)) + + " supports WorkAssignment=StaticGrid only"); + if(selectors.persistentLoop + && !selectors.persistentLoop->equals(policy.persistent() ? 1 : 0)) + throw std::runtime_error( + "_PersistentLoop disagrees with the derived execution policy"); + return policy; + } + + // A type(x) is int check: True and False are not ints there. + bool isInt(Value const& value) + { + return value.object().type == object_type::POSITIVE_INTEGER + || value.object().type == object_type::NEGATIVE_INTEGER; + } + + // What SizeMapping reads of normalize_execution_policy(d, regenerate=...). + struct NormalizedPolicy + { + ExecutionPolicy policy; + int64_t streamKAtomic = 0; + int64_t prefetchAcrossPersistent = 0; + }; + + NormalizedPolicy normalizeExecutionPolicy(Dict const& config, bool regenerate) + { + if(config.has("PersistentLoop")) + throw std::runtime_error( + "PersistentLoop is derived from TileProcessingStrategy and WorkAssignment; " + "it is not a tuning parameter"); + bool legacy = config.has("StreamK") || config.has("StreamKForceDPOnly"); + + PolicySelectors selectors; + if(auto const* value = config.find("TileProcessingStrategy")) + selectors.strategy = value->text(); + if(auto const* value = config.find("WorkAssignment")) + selectors.assignment = value->text(); + selectors.persistentLoop = config.find("_PersistentLoop"); + + if(legacy) + { + // _translate_legacy_streamk_selectors. + auto const* modeValue = config.find("StreamK"); + auto const* forceValue = config.find("StreamKForceDPOnly"); + int64_t mode = 0; + int64_t force = 0; + if(modeValue) + { + if(!isInt(*modeValue)) + throw std::runtime_error("Legacy StreamK must be 0, 3, 4, or 5; " + "modes 1 and 2 are retired"); + mode = modeValue->integer(); + } + if(mode != 0 && mode != 3 && mode != 4 && mode != 5) + throw std::runtime_error( + "Legacy StreamK must be 0, 3, 4, or 5; modes 1 and 2 are retired"); + if(forceValue) + { + if(!isInt(*forceValue) && forceValue->object().type != object_type::BOOLEAN) + throw std::runtime_error("StreamKForceDPOnly must be 0 or 1"); + force = forceValue->integer(); + } + if(force != 0 && force != 1) + throw std::runtime_error("StreamKForceDPOnly must be 0 or 1"); + if(force && (mode != 3 || config.truthy("StreamKAtomic", false))) + throw std::runtime_error("StreamKForceDPOnly requires non-atomic StreamK=3"); + std::string strategy = force ? "DataParallel" : mode == 0 ? "None" : "StreamK"; + std::string assignment = mode == 4 ? "DynamicWorkQueue" + : mode == 5 ? "Hybrid" + : "StaticGrid"; + if(config.has("TileProcessingStrategy") && selectors.strategy != strategy) + throw std::runtime_error( + "Conflicting legacy StreamK and TileProcessingStrategy=" + + selectors.strategy + " (expected " + strategy + ")"); + selectors.strategy = strategy; + if(!(strategy == "None" && config.has("WorkAssignment"))) + { + if(config.has("WorkAssignment") && selectors.assignment != assignment) + throw std::runtime_error("Conflicting legacy StreamK and WorkAssignment=" + + selectors.assignment + " (expected " + assignment + + ")"); + selectors.assignment = assignment; + } + } + + // ALIASES: the old spelling wins, and must agree with the new one. + Value const* workQueueStealing = config.find("WorkQueueStealing"); + for(auto alias : + {std::pair{"StreamKXCCMapping", "PersistentXCCMapping"}, + std::pair{"StreamKWorkStealing", "WorkQueueStealing"}}) + { + auto const* old = config.find(alias.first); + if(!old) + continue; + auto const* current = config.find(alias.second); + if(current && !old->equals(*current)) + throw std::runtime_error(std::string("Conflicting ") + alias.first + " and " + + alias.second); + if(std::string_view(alias.second) == "WorkQueueStealing") + workQueueStealing = old; + } + + NormalizedPolicy rv; + rv.policy = resolvePolicy(selectors); + auto const& policy = rv.policy; + + for(auto option : {"StreamKAtomic", "StreamKFixupTreeReduction", "DebugStreamK"}) + { + if(!policy.streamK() && !legacy && config.truthy(option, false)) + throw std::runtime_error(std::string(option) + + " requires TileProcessingStrategy=StreamK"); + } + rv.streamKAtomic = policy.streamK() ? config.integer("StreamKAtomic", 0) : 0; + + if(workQueueStealing && workQueueStealing->truthy() + && (!policy.streamK() || policy.assignment == WorkAssignment::StaticGrid)) + { + if(!legacy || policy.persistent()) + throw std::runtime_error( + "WorkQueueStealing requires StreamK with DynamicWorkQueue or Hybrid"); + } + + if(!policy.persistent()) + { + for(auto option : {"PrefetchAcrossPersistent", + "ReuseAcrossPersistent", + "DebugPersistentKernelLoopForever"}) + { + if(!legacy && config.truthy(option, false)) + throw std::runtime_error(std::string(option) + + " requires a persistent TileProcessingStrategy"); + } + } + rv.prefetchAcrossPersistent + = policy.persistent() ? config.integer("PrefetchAcrossPersistent", 0) : 0; + + Dict support = config.has("InternalSupportParams") + ? config.dict("InternalSupportParams") + : emptyDict(); + int64_t version = 0; + int64_t outerVersion = 3; + if(auto const* value = support.find("PersistentLoopArgsVersion")) + { + if(!isInt(*value)) + throw std::runtime_error("Unsupported PersistentLoopArgsVersion"); + version = value->integer(); + } + if(auto const* value = support.find("KernArgsVersion")) + { + if(!isInt(*value)) + throw std::runtime_error("Unsupported KernArgsVersion"); + outerVersion = value->integer(); + } + if(version != 0 && version != 1 && version != 2) + throw std::runtime_error("Unsupported PersistentLoopArgsVersion"); + if(outerVersion < 0 || outerVersion > 3) + throw std::runtime_error("Unsupported KernArgsVersion"); + + bool handwritten = false; + auto const* custom = config.find("CustomKernel"); + if(custom && custom->object().type == object_type::MAP + && Dict(*custom).truthy("name", false)) + handwritten = !Dict(*custom).truthy("generated", false); + else + handwritten = config.truthy("CustomKernelName", false); + + if(version == 1 && !policy.persistentDataParallel() + && (!regenerate || handwritten || config.has("InternalSupportParams"))) + throw std::runtime_error( + "PersistentLoopArgsVersion=1 requires DataParallel/StaticGrid"); + if(version == 1 && outerVersion != 3 && (!regenerate || handwritten)) + throw std::runtime_error("PersistentLoopArgsVersion=1 requires KernArgsVersion=3"); + + // ExecutionPolicy.isStreamKSpatialCluster: StreamK, a static grid, + // and a ClusterDim axis above 1. + bool spatialCluster = false; + if(auto const* cluster = config.find("ClusterDim")) + { + for(auto const& axis : cluster->list()) + if(axis.number() > 1) + spatialCluster = true; + } + bool clusterStreamK = policy.streamK() + && policy.assignment == WorkAssignment::StaticGrid + && spatialCluster; + if(version == 2 && !clusterStreamK + && (!regenerate || handwritten || config.has("InternalSupportParams"))) + throw std::runtime_error("PersistentLoopArgsVersion=2 requires StreamK/StaticGrid " + "with a spatial cluster"); + if(version == 2 && outerVersion != 3 && (!regenerate || handwritten)) + throw std::runtime_error("PersistentLoopArgsVersion=2 requires KernArgsVersion=3"); + return rv; + } + + // ExecutionPolicy._policy, behind isPersistent and isStreamK. + ExecutionPolicy policyOf(Dict const& state) + { + if(!state.has("TileProcessingStrategy") + && (state.has("StreamK") || state.has("StreamKForceDPOnly"))) + return normalizeExecutionPolicy(state, true).policy; + PolicySelectors selectors; + if(auto const* value = state.find("TileProcessingStrategy")) + selectors.strategy = value->text(); + if(auto const* value = state.find("WorkAssignment")) + selectors.assignment = value->text(); + selectors.persistentLoop = state.find("_PersistentLoop"); + return resolvePolicy(selectors); + } + + // Contractions.ProblemType, with what its predicates read beyond the record. + struct ProblemType + { + ContractionSolution::ProblemType record; + std::string activation; + bool batched = false; + std::vector scaleBlockSizesA; + bool scaleZeroPointA = false; + ContractionProblemGemm::Int4Encoding int4EncodingA + = ContractionProblemGemm::Int4Encoding::Signed; + }; + + // Contractions._operandIsPerInputMxType. FP8, BF8 (including fnuz), + // FP6, BF6, and FP4. The packed F8B8 types are not in this set. + bool perInputMxType(rocisa::DataType type) + { + switch(type) + { + case rocisa::DataType::Float8: + case rocisa::DataType::Float8_fnuz: + case rocisa::DataType::BFloat8: + case rocisa::DataType::BFloat8_fnuz: + case rocisa::DataType::Float6: + case rocisa::DataType::BFloat6: + case rocisa::DataType::Float4: + return true; + default: + return false; + } + } + + // Contractions._computeInputsFromOperandTypes. Mixed MX operands whose + // MAC inputs are still the DataType default take the operand types. + void applyMixedMxComputeInputs(Dict const& d, + rocisa::DataType srcType, + ContractionSolution::ProblemType& record) + { + if(!d.has("DataTypeA") || !d.has("DataTypeB")) + return; + auto operandA = d.dataType("DataTypeA"); + auto operandB = d.dataType("DataTypeB"); + if(operandA == operandB) + return; + if(!perInputMxType(operandA) || !perInputMxType(operandB)) + return; + if(record.computeInputTypeA != srcType || record.computeInputTypeB != srcType) + return; + if(srcType != operandA && srcType != operandB) + return; + record.computeInputTypeA = operandA; + record.computeInputTypeB = operandB; + } + + [[noreturn]] void inconsistentIndices() + { + throw std::runtime_error( + "Solution state ProblemType index assignments are inconsistent"); + } + + struct ProblemIndexSlot + { + int kind = 0; + int64_t problemIndex = 0; + int64_t kindOrder = 0; + bool isA = false; + int64_t i = 0; + int64_t c = 0; + int64_t d = 0; + bool hasA = false; + bool hasB = false; + int64_t a = 0; + int64_t b = 0; + bool aMirror = false; + bool bMirror = false; + }; + + struct ProblemIndexLayout + { + std::string operationIdentifier; + std::string aNames; + std::string bNames; + std::string cNames; + std::string dNames; + std::string sumNames; + int64_t aDims = 0; + int64_t bDims = 0; + int64_t cDims = 0; + std::vector indices; + }; + + // The reconstruction of a problem type's index layout from one fully + // derived state. + ProblemIndexLayout indexLayout(Dict const& d) + { + struct Index + { + enum Kind + { + Free, + Batch, + Bound + } kind; + int64_t problemIndex = 0; + int64_t kindOrder = 0; + bool isA = false; + int64_t i = 0; + int64_t c = 0; + int64_t d = 0; + std::optional a; + std::optional b; + bool aMirror = false; + bool bMirror = false; + }; + + auto contains = [](std::vector const& list, int64_t value) { + return std::find(list.begin(), list.end(), value) != list.end(); + }; + auto position = [](std::vector const& list, int64_t value) { + return int64_t(std::find(list.begin(), list.end(), value) - list.begin()); + }; + + auto total = d.integer("TotalIndices"); + auto summation = d["IndicesSummation"].integers(); + auto mirrorA + = d.has("MirrorDimsA") ? d["MirrorDimsA"].integers() : std::vector{}; + auto mirrorB + = d.has("MirrorDimsB") ? d["MirrorDimsB"].integers() : std::vector{}; + auto numIndicesC = d.integer("NumIndicesC"); + auto batch = d["IndicesBatch"].integers(); + auto free = d["IndicesFree"].integers(); + auto assignA = d["IndexAssignmentsA"].integers(); + auto assignB = d["IndexAssignmentsB"].integers(); + if(total < 0 || numIndicesC < 0) + inconsistentIndices(); + + using IndexPtr = std::shared_ptr; + std::vector indices(total); + std::vector freeIndices, batchIndices, boundIndices; + auto normalize = [&](int64_t i) { + if(i < 0) + i += total; + if(i < 0 || i >= total) + inconsistentIndices(); + return i; + }; + auto slot = [&](int64_t i) -> IndexPtr& { return indices[normalize(i)]; }; + + for(auto i : summation) + { + auto bound = std::make_shared(Index{Index::Bound}); + bound->aMirror = contains(mirrorA, i); + bound->bMirror = contains(mirrorB, i); + bound->problemIndex = normalize(i); + bound->kindOrder = int64_t(boundIndices.size()); + slot(i) = bound; + boundIndices.push_back(bound); + } + for(int64_t i = 0; i < numIndicesC; i++) + { + auto index = std::make_shared(Index{Index::Free}); + index->c = index->d = i; + index->problemIndex = i; + if(contains(batch, i)) + { + index->kind = Index::Batch; + index->kindOrder = int64_t(batchIndices.size()); + slot(i) = index; + batchIndices.push_back(index); + continue; + } + if(!contains(free, i)) + inconsistentIndices(); + if(contains(assignA, i)) + { + index->isA = true; + index->i = position(assignA, i); + } + else if(contains(assignB, i)) + { + index->isA = false; + index->i = position(assignB, i); + } + else + { + throw std::runtime_error("free index " + std::to_string(i) + + " not in ia or ib"); + } + index->kindOrder = int64_t(freeIndices.size()); + slot(i) = index; + freeIndices.push_back(index); + } + for(size_t ia = 0; ia < assignA.size(); ia++) + { + auto& index = slot(assignA[ia]); + if(!index) + inconsistentIndices(); + index->a = int64_t(ia); + } + for(size_t ib = 0; ib < assignB.size(); ib++) + { + auto& index = slot(assignB[ib]); + if(!index) + inconsistentIndices(); + index->b = int64_t(ib); + } + for(auto const& index : indices) + if(!index || (index->kind != Index::Free && (!index->a || !index->b))) + inconsistentIndices(); + + static std::string const allIndexNames = "ijklmnopqrstuvwxyz"; + if(numIndicesC + int64_t(boundIndices.size()) > int64_t(allIndexNames.size())) + inconsistentIndices(); + std::string aNames(assignA.size(), '_'); + std::string bNames(assignB.size(), '_'); + std::string cNames(size_t(numIndicesC), '_'); + std::string dNames = allIndexNames.substr(0, numIndicesC); + std::string sumNames = allIndexNames.substr(numIndicesC, boundIndices.size()); + + auto name = [](std::string& names, int64_t i) -> char& { + if(i < 0) + i += int64_t(names.size()); + if(i < 0 || i >= int64_t(names.size())) + inconsistentIndices(); + return names[i]; + }; + for(auto const& index : freeIndices) + { + name(index->isA ? aNames : bNames, index->i) = name(dNames, index->d); + name(cNames, index->c) = name(dNames, index->d); + } + for(auto const& index : batchIndices) + { + char letter = name(dNames, index->d); + name(aNames, *index->a) = letter; + name(bNames, *index->b) = letter; + name(cNames, index->c) = letter; + } + for(size_t n = 0; n < boundIndices.size(); n++) + { + auto const& index = *boundIndices[n]; + if(!index.a || !index.b) + inconsistentIndices(); + char letter = sumNames[n]; + char upper = char(std::toupper(static_cast(letter))); + name(aNames, *index.a) = index.aMirror ? upper : letter; + name(bNames, *index.b) = index.bMirror ? upper : letter; + } + + ProblemIndexLayout layout; + layout.aNames = aNames; + layout.bNames = bNames; + layout.cNames = cNames; + layout.dNames = dNames; + layout.sumNames = sumNames; + layout.aDims = int64_t(assignA.size()); + layout.bDims = int64_t(assignB.size()); + layout.cDims = numIndicesC; + layout.operationIdentifier = "Contraction_" + sumNames + "_A" + aNames + + (d.truthy("ComplexConjugateA") ? "C" : "") + "_B" + + bNames + (d.truthy("ComplexConjugateB") ? "C" : "") + + "_C" + cNames + "_D" + dNames; + layout.indices.reserve(indices.size()); + for(auto const& index : indices) + { + ProblemIndexSlot slot; + if(index->kind == Index::Free) + slot.kind = 0; + else if(index->kind == Index::Batch) + slot.kind = 1; + else + slot.kind = 2; + slot.problemIndex = index->problemIndex; + slot.kindOrder = index->kindOrder; + slot.isA = index->isA; + slot.i = index->i; + slot.c = index->c; + slot.d = index->d; + slot.hasA = index->a.has_value(); + slot.hasB = index->b.has_value(); + slot.a = slot.hasA ? *index->a : 0; + slot.b = slot.hasB ? *index->b : 0; + slot.aMirror = index->aMirror; + slot.bMirror = index->bMirror; + layout.indices.push_back(slot); + } + return layout; + } + + // Contractions.ProblemType.FromOriginalState. + ProblemType problemTypeFromState(Dict const& d) + { + ProblemType rv; + auto& record = rv.record; + + auto opIdentifier = indexLayout(d).operationIdentifier; + + auto srcType = d.dataType("DataType"); + auto dstType = d.has("DestDataType") ? d.dataType("DestDataType") : srcType; + auto computeType = d.has("ComputeDataType") ? d.dataType("ComputeDataType") : dstType; + + record.transA = d.truthy("TransposeA"); + record.transB = d.truthy("TransposeB"); + + record.computeInputTypeA = d.has("MacDataTypeA") ? d.dataType("MacDataTypeA") : srcType; + record.computeInputTypeB = d.has("MacDataTypeB") ? d.dataType("MacDataTypeB") : srcType; + record.aType = d.has("DataTypeA") ? d.dataType("DataTypeA") : record.computeInputTypeA; + record.bType = d.has("DataTypeB") ? d.dataType("DataTypeB") : record.computeInputTypeB; + + auto either = [&](rocisa::DataType type) { + return record.aType == type || record.bType == type; + }; + if(either(rocisa::DataType::Float8BFloat8)) + { + record.aType = rocisa::DataType::Float8; + record.bType = rocisa::DataType::BFloat8; + } + else if(either(rocisa::DataType::Float8BFloat8_fnuz)) + { + record.aType = rocisa::DataType::Float8_fnuz; + record.bType = rocisa::DataType::BFloat8_fnuz; + } + else if(either(rocisa::DataType::BFloat8Float8)) + { + record.aType = rocisa::DataType::BFloat8; + record.bType = rocisa::DataType::Float8; + } + else if(either(rocisa::DataType::BFloat8Float8_fnuz)) + { + record.aType = rocisa::DataType::BFloat8_fnuz; + record.bType = rocisa::DataType::Float8_fnuz; + } + + applyMixedMxComputeInputs(d, srcType, record); + + record.eType = d.has("DataTypeE") ? d.dataType("DataTypeE") : dstType; + if(d.has("DataTypeAmaxD")) + d.dataType("DataTypeAmaxD"); + + record.cType = dstType; + record.dType = dstType; + record.computeType = computeType; + + record.highPrecisionAccumulate = d.truthy("HighPrecisionAccumulate", false); + record.useInitialStridesAB = d.truthy("UseInitialStridesAB", false); + record.useInitialStridesCD = d.truthy("UseInitialStridesCD", false); + record.stridedBatched = d.truthy("StridedBatched", true); + record.groupedGemm = d.truthy("GroupedGemm", false); + record.useBeta = d.truthy("UseBeta", true); + + record.useBias = 0; + if(d.has("UseBias")) + { + record.useBias = d.integer("UseBias"); + record.biasDataTypeWhiteList = d.has("BiasDataTypeList") + ? sortedDataTypes(d["BiasDataTypeList"]) + : biasDataTypeListDefault(d); + if(auto const* biasSrc = d.find("BiasSrc")) + { + static char const* const sources[] = {"A", "B", "C", "D"}; + auto found = std::find_if(std::begin(sources), + std::end(sources), + [&](char const* s) { return biasSrc->isText(s); }); + if(found == std::end(sources)) + biasSrc->fail("one of A, B, C and D"); + record.biasSrcWhiteList = {int(found - std::begin(sources))}; + } + } + + record.useE = d.truthy("UseE", false); + + record.useGateResidual = false; + if(d.has("UseGateResidual")) + { + record.useGateResidual = d.truthy("UseGateResidual"); + record.gateResidualDataTypeWhiteList + = d.has("GateResidualDataTypeList") + ? sortedDataTypes(d["GateResidualDataTypeList"]) + : gateResidualDataTypeListDefault(d); + } + + record.useGradient = d.truthy("Gradient", false); + record.outputAmaxD = d.truthy("OutputAmaxD", false); + record.useScaleAB = d.has("UseScaleAB") ? d.text("UseScaleAB") : ""; + + // Contractions.ProblemType.FromOriginalState group-scale checks. + if(d.has("ScaleBlockSizesA")) + { + auto groups = d["ScaleBlockSizesA"]; + if(groups.object().type != object_type::ARRAY) + throw std::runtime_error("ScaleBlockSizesA must be a list of supported groups"); + for(auto const& group : groups.list()) + { + auto size = group.integer(); + if(size != 32 && size != 64 && size != 128) + throw std::runtime_error("ScaleBlockSizesA entries must be 32, 64 or 128"); + if(std::find(rv.scaleBlockSizesA.begin(), rv.scaleBlockSizesA.end(), int(size)) + == rv.scaleBlockSizesA.end()) + rv.scaleBlockSizesA.push_back(int(size)); + } + std::sort(rv.scaleBlockSizesA.begin(), rv.scaleBlockSizesA.end()); + } + if(record.aType == rocisa::DataType::Int4 && rv.scaleBlockSizesA.empty()) + throw std::runtime_error("ScaleBlockSizesA must be non-empty for int4 A (W4A16)"); + if(!rv.scaleBlockSizesA.empty() && record.aType != rocisa::DataType::Int4) + throw std::runtime_error("ScaleBlockSizesA requires int4 A"); + rv.scaleZeroPointA = d.truthy("ScaleZeroPointA", false); + if(d.has("Int4EncodingA")) + { + auto encoding = d.text("Int4EncodingA"); + if(encoding == "Signed") + rv.int4EncodingA = ContractionProblemGemm::Int4Encoding::Signed; + else if(encoding == "UnsignedBias8") + rv.int4EncodingA = ContractionProblemGemm::Int4Encoding::UnsignedBias8; + else + throw std::runtime_error("Int4EncodingA must be Signed or UnsignedBias8"); + } + + record.useScaleCD = d.truthy("UseScaleCD", false); + record.useScaleAlphaVec = d.integer("UseScaleAlphaVec", 0); + + rv.batched = d.truthy("Batched"); + + rv.activation = d.has("ActivationType") ? activationName(d["ActivationType"]) : "none"; + record.activationType = activationType(rv.activation); + record.activationArgLength = additionalArgNum(rv.activation); + record.activationComputeDataType = d.dataType("ActivationComputeDataType"); + record.activationNoGuard = d.truthy("ActivationNoGuard", false); + record.sparse = d.integer("Sparse", 0); + + record.f32XdlMathOp + = d.has("F32XdlMathOp") ? d.dataType("F32XdlMathOp") : rocisa::DataType::Float; + + record.supportDeviceUserArguments = d.truthy("SupportUserArgs", false); + + record.swizzleTensorA = d.truthy("SwizzleTensorA", false); + record.swizzleTensorB = d.truthy("SwizzleTensorB", false); + record.fusedGemmA2A = d.truthy("FusedGemmA2A", false); + + record.mxBlockA = d.integer("MXBlockA", 0); + record.mxBlockB = d.integer("MXBlockB", 0); + record.mxBlockFreeA = d.integer("MXBlockFreeA", 1); + record.mxBlockFreeB = d.integer("MXBlockFreeB", 1); + record.mxTypeA + = d.has("DataTypeMXSA") ? d.dataType("DataTypeMXSA") : rocisa::DataType::Float; + record.mxTypeB + = d.has("DataTypeMXSB") ? d.dataType("DataTypeMXSB") : rocisa::DataType::Float; + + record.metadataLayout = d.integer("MetadataLayout", 0); + record.stochasticRounding = d.truthy("StochasticRounding", false); + + record.operationIdentifier = opIdentifier; + return rv; + } + + // Contractions.ProblemType.predicates(True, True, True). + std::vector> problemTypePredicates(ProblemType const& pt) + { + using namespace Predicates::Contraction; + auto const& r = pt.record; + + std::vector> rv; + rv.push_back(withValue(r.operationIdentifier)); + if(!r.useBeta) + rv.push_back(std::make_shared()); + rv.push_back(withValue(r.biasDataTypeWhiteList)); + rv.push_back(withValue(r.gateResidualDataTypeWhiteList)); + rv.push_back(withValue(r.biasSrcWhiteList)); + rv.push_back(withValue(r.outputAmaxD)); + if(isAllActivations(pt.activation)) + rv.push_back(withValue(activationEnumList( + r.activationComputeDataType, pt.activation == "all", r.useGradient))); + + rv.push_back(withValue(std::array{ + r.aType, r.bType, r.cType, r.dType, r.computeInputTypeA, r.computeInputTypeB})); + rv.push_back(withValue(r.highPrecisionAccumulate)); + rv.push_back(withValue(r.activationType)); + rv.push_back(withValue(r.activationComputeDataType)); + rv.push_back(withValue(r.activationNoGuard)); + rv.push_back(withValue(r.useGradient)); + rv.push_back(withValue(r.useBias)); + rv.push_back(withValue(r.useE)); + rv.push_back(withValue(r.useGateResidual)); + rv.push_back(withValue(r.eType)); + rv.push_back(withValue(r.stridedBatched)); + rv.push_back(withValue(r.groupedGemm)); + rv.push_back(withValue(r.useScaleAB)); + if(!pt.scaleBlockSizesA.empty()) + { + if(pt.scaleBlockSizesA.size() == 1) + rv.push_back(withValue(pt.scaleBlockSizesA.front())); + else + { + std::vector> sizes; + for(auto group : pt.scaleBlockSizesA) + sizes.push_back(withValue(group)); + rv.push_back(disjunction(sizes)); + } + rv.push_back(withValue(pt.scaleZeroPointA)); + rv.push_back(withValue(pt.int4EncodingA)); + } + rv.push_back(withValue(r.useScaleCD)); + rv.push_back(withValue(r.useScaleAlphaVec)); + rv.push_back(withValue(r.sparse)); + rv.push_back(withValue(r.f32XdlMathOp)); + rv.push_back(withValue(r.supportDeviceUserArguments)); + rv.push_back(withValue(r.swizzleTensorA)); + rv.push_back(withValue(r.swizzleTensorB)); + rv.push_back(withValue(r.fusedGemmA2A)); + if(r.fusedGemmA2A && pt.batched) + rv.push_back(withIndexValue(size_t(0), size_t(1))); + rv.push_back(withValue(r.mxBlockA)); + if(r.mxBlockA) + { + rv.push_back(withValue(r.mxTypeA)); + rv.push_back(withValue(r.mxBlockFreeA)); + } + rv.push_back(withValue(r.mxBlockB)); + if(r.mxBlockB) + { + rv.push_back(withValue(r.mxTypeB)); + rv.push_back(withValue(r.mxBlockFreeB)); + } + return rv; + } + + // Contractions.ASSERT_DIM_MAP_PREDICATES. A value of -1 skips that index. + bool isDimAssertKey(std::string const& key) + { + return key == "AssertSizeEqual" || key == "AssertSizeGreaterThan" + || key == "AssertSizeLessThan" || key == "AssertStrideAEqual" + || key == "AssertStrideBEqual" || key == "AssertStrideCEqual" + || key == "AssertStrideDEqual"; + } + + // Contractions.extractDimPredicate. One constraint is that predicate; + // more than one is their conjunction. An empty map, or a map whose + // values are all -1, contributes nothing. + std::shared_ptr dimAssertPredicate(std::string const& key, + Value const& value) + { + using namespace Predicates::Contraction; + if(value.object().type != object_type::MAP) + throw std::runtime_error(key + " must be a dict of {index: value}"); + + auto make = [&](size_t index, size_t size) -> std::shared_ptr { + if(key == "AssertSizeEqual") + return withIndexValue(index, size); + if(key == "AssertSizeGreaterThan") + return withIndexValue(index, size); + if(key == "AssertSizeLessThan") + return withIndexValue(index, size); + if(key == "AssertStrideAEqual") + return withIndexValue(index, size); + if(key == "AssertStrideBEqual") + return withIndexValue(index, size); + if(key == "AssertStrideCEqual") + return withIndexValue(index, size); + if(key == "AssertStrideDEqual") + return withIndexValue(index, size); + throw std::runtime_error("Unknown assertion key: " + key); + }; + + std::vector> predicates; + auto const& map = value.object().via.map; + for(uint32_t i = 0; i < map.size; i++) + { + Value index(map.ptr[i].key, value.path() + " index"); + Value entry(map.ptr[i].val, value.path()); + if(index.object().type != object_type::POSITIVE_INTEGER + && index.object().type != object_type::NEGATIVE_INTEGER) + index.fail("an integer index"); + auto position = index.integer(); + if(position < 0) + index.fail("an index >= 0"); + if(entry.equals(-1)) + continue; + if(entry.object().type != object_type::POSITIVE_INTEGER + && entry.object().type != object_type::NEGATIVE_INTEGER) + entry.fail("an integer"); + auto size = entry.integer(); + if(size < 0) + entry.fail("a size >= 0"); + predicates.push_back(make(size_t(position), size_t(size))); + } + if(predicates.empty()) + return nullptr; + return conjunction(predicates); + } + + // Contractions.ProblemPredicate.FromOriginalKeyPair. + std::shared_ptr problemPredicateFromKeyPair(std::string const& key, + Value const& value) + { + using namespace Predicates::Contraction; + if(key == "AssertAIGreaterThanEqual") + return value.number() > 0 ? withValue(value.number()) : nullptr; + if(key == "AssertAILessThanEqual") + return value.number() > 0 ? withValue(value.number()) : nullptr; + if(isDimAssertKey(key)) + return dimAssertPredicate(key, value); + + auto endsWith = [&](std::string_view suffix) { + return key.size() >= suffix.size() + && key.compare(key.size() - suffix.size(), suffix.size(), suffix) == 0; + }; + if(endsWith("Multiple")) + { + if(value.equals(1)) + return nullptr; + if(key == "AssertFree0ElementMultiple") + return withIndexValue(size_t(0), size_t(value.integer())); + if(key == "AssertFree1ElementMultiple") + return withIndexValue(size_t(0), size_t(value.integer())); + if(key == "AssertSummationElementMultiple") + return withIndexValue(int64_t(-1), size_t(value.integer())); + throw std::runtime_error("Unknown Multiple Value: " + key); + } + + if(key.rfind("Assert", 0) == 0) + throw std::runtime_error("Unknown assertion key: " + key); + return nullptr; + } + + // Contractions.ProblemPredicate.CompoundPredicates. + std::vector> + compoundPredicates(Dict const& state, ProblemType const& problemType) + { + using namespace Predicates::Contraction; + std::vector> rv; + auto const problem = state.dict("ProblemType"); + + if(state.has("BatchSizeEqual")) + rv.push_back(withIndexValue( + size_t(0), size_t(state.integer("BatchSizeEqual")))); + + if(state.has("SynchronizerSizeCheck") && state.truthy("SynchronizerSizeCheck")) + { + std::array value; + value[0] = int(state.integer("MacroTile0")); + value[1] = int(state.integer("MacroTile1")); + if(state.truthy("EnableMatrixInstruction")) + { + auto miWaveTile = state["MIWaveTile"].list(); + if(miWaveTile.size() < 2) + state["MIWaveTile"].fail("a list of 2 values"); + value[2] = int(miWaveTile[0].integer() * miWaveTile[1].integer()); + } + else + value[2] = int(state.integer("ThreadTile0") * state.integer("ThreadTile1")); + if(!state["NumElementsPerBatchStore"].equals(0)) + value[3] = int( + quotient(state["NumElementsPerThread"], state["NumElementsPerBatchStore"])); + else + value[3] = int(state.integer("NumElementsPerThread")); + value[4] = int(std::ceil(quotient(state["NumThreads"], state["WavefrontSize"]))); + value[5] = int(state.integer("GlobalSplitU")); + rv.push_back(withIndexValue(size_t(0), value)); + } + + if(state.dict("InternalSupportParams").integer("KernArgsVersion") >= 1 + && !policyOf(state).persistent()) + rv.push_back(withIndexValue( + size_t(0), + std::array{int(state.integer("MacroTile0")), + int(state.integer("MacroTile1")), + int(state.integer("GlobalSplitU"))})); + + if(problemType.record.aType != rocisa::DataType::Int8x4) + { + auto minFree0 + = problem.truthy("TLUA") ? state.integer("GlobalReadVectorWidthA") : 1; + auto minFree1 + = problem.truthy("TLUB") ? state.integer("GlobalReadVectorWidthB") : 1; + rv.push_back(withValue(size_t(minFree0))); + rv.push_back(withValue(size_t(minFree1))); + } + + if(state["PackedC0IndicesX"].list().size() > 1) + rv.push_back(std::make_shared()); + + if(state.has("KernelLanguage")) + rv.push_back(withValue(enumByName( + state["KernelLanguage"], + static_cast(KernelLanguage::Count), + [](KernelLanguage k) { return KernelLanguageTypeInfo::Get(k).name; }, + "a kernel language"))); + + if(state.has("GlobalSplitU") + && (state["GlobalSplitU"].number() > 1 || state["GlobalSplitU"].equals(-1))) + { + if(!state.has("_GlobalAccumulation") + || !state["_GlobalAccumulation"].isText("MultipleBuffer")) + rv.push_back(withValue(false)); + } + + if(policyOf(state).streamK() && state.has("StreamKAtomic") + && state["StreamKAtomic"].equals(1)) + rv.push_back(withValue(false)); + + bool bufferLoad = state.has("BufferLoad") && state["BufferLoad"].equals(1); + if(bufferLoad) + { + bool tlua = problem.truthy("TLUA"); + bool tlub = problem.truthy("TLUB"); + bool mayShiftA = tlua + && state["AssertFree0ElementMultiple"].number() + < state["GlobalReadVectorWidthA"].number(); + bool mayShiftB = tlub + && state["AssertFree1ElementMultiple"].number() + < state["GlobalReadVectorWidthB"].number(); + BufferLoadCheckPacket packet; + packet.shiftPtrElemB = mayShiftB ? state.integer("GlobalReadVectorWidthB") : 0; + packet.shiftPtrElemA = mayShiftA ? state.integer("GlobalReadVectorWidthA") : 0; + packet.depthUorMT1 = tlub ? state.integer("DepthU") : state.integer("MacroTile1"); + packet.depthUorMT0 = tlua ? state.integer("DepthU") : state.integer("MacroTile0"); + rv.push_back(withValue(packet)); + } + + if(bufferLoad) + rv.push_back(withValue( + size_t(state.integer("MacroTile1")))); + + if(state.has("BufferStore") && state["BufferStore"].equals(1)) + rv.push_back( + withValue(size_t(state.integer("MacroTile1")))); + + if(state.has("_GlobalAccumulation") && !state["_GlobalAccumulation"].isNone() + && !policyOf(state).persistent()) + rv.push_back(withValue( + std::array{32, int(state.integer("GlobalSplitU"))})); + + if(state.has("WorkGroupMappingXCC") && state.has("WorkGroupMappingXCCGroup")) + { + // Value only. Construction does not call hipGetDevice; the first + // evaluation does. The test constructor freezes a count of 0 + // and then rejects every kernel. + rv.push_back(std::make_shared( + std::array{int(state.integer("WorkGroupMappingXCC")), + int(state.integer("WorkGroupMappingXCCGroup"))})); + } + + if(problem.truthy("SwizzleTensorA")) + rv.push_back(withValue(true)); + + if(problem.truthy("SwizzleTensorB")) + rv.push_back(withValue(true)); + + if(problem.truthy("FusedGemmA2A")) + rv.push_back(withValue(state.integer("MacroTile0"))); + + // Exact-size custom kernels record the M/N/K window they won. + if(auto const* bounds = state.find("CustomKernelSizePredicate")) + { + for(auto const& boundValue : bounds->list()) + { + Dict bound(boundValue); + auto tag = bound.text("tag"); + auto index = size_t(bound.integer("index")); + auto value = size_t(bound.integer("value")); + if(tag == "SizeGreaterThan") + rv.push_back(withIndexValue(index, value)); + else if(tag == "SizeLessThan") + rv.push_back(withIndexValue(index, value)); + else if(tag == "SizeEqual") + rv.push_back(withIndexValue(index, value)); + else + throw std::runtime_error("Unknown CustomKernelSizePredicate tag: " + tag); + } + } + + if(state.truthy("ReuseAcrossPersistent", false)) + { + auto kIdx = size_t(problem.integer("NumIndicesC")); + auto kTiles = state.integer("_RAPNumResidentKTiles"); + auto depthU = state.integer("DepthU"); + rv.push_back(withIndexValue(kIdx, size_t(0))); + rv.push_back(withIndexValue(kIdx, size_t(kTiles * depthU + 1))); + } + + return rv; + } + + // Contractions.ProblemPredicate.FromOriginalState. + std::shared_ptr problemPredicateFromState(Dict const& d, + ProblemType const& problemType) + { + auto problemTypePreds = problemTypePredicates(problemType); + auto compoundPreds = compoundPredicates(d, problemType); + + std::vector> predicates; + for(auto const& item : d.items()) + if(auto predicate = problemPredicateFromKeyPair(item.first, item.second)) + predicates.push_back(predicate); + predicates.insert(predicates.end(), problemTypePreds.begin(), problemTypePreds.end()); + predicates.insert(predicates.end(), compoundPreds.begin(), compoundPreds.end()); + return conjunction(predicates); + } + + // Contractions.TaskPredicate.FromOriginalState. + std::shared_ptr taskPredicateFromState(Dict const& d) + { + std::vector> predicates; + for(auto const& item : d.items()) + if(item.first == "_WorkspaceSizePerElemC" && item.second.number() > 0) + predicates.push_back( + std::make_shared()); + if(!policyOf(d).persistent()) + predicates.push_back(std::make_shared()); + return conjunction(predicates); + } + + // Hardware.HardwarePredicate._createPciChipIdPredicate. + std::shared_ptr pciChipIdPredicate(Value const* deviceNames) + { + std::vector names; + if(deviceNames && !deviceNames->isNone()) + { + if(deviceNames->object().type == object_type::STR) + names.push_back(*deviceNames); + else if(deviceNames->truthy()) + names = deviceNames->list(); + } + + static std::regex const deviceName("^Device\\s+([0-9a-fA-F]+)$"); + std::vector> predicates; + for(auto const& name : names) + { + if(name.isNone()) + continue; + auto text = name.text(); + auto first = text.find_first_not_of(" \t\n\r\f\v"); + auto last = text.find_last_not_of(" \t\n\r\f\v"); + std::string stripped + = first == std::string::npos ? "" : text.substr(first, last - first + 1); + std::smatch match; + if(!std::regex_match(stripped, match, deviceName)) + throw std::runtime_error("Invalid device name format: '" + text + + "', expected 'Device XXXX'"); + auto chipId = match[1].str(); + if(std::find(SupportedBuildChipIds.begin(), + SupportedBuildChipIds.end(), + Value::lower(chipId)) + == SupportedBuildChipIds.end()) + continue; + predicates.push_back(std::make_shared( + int(std::stoll(chipId, nullptr, 16)))); + } + if(predicates.empty()) + return nullptr; + return disjunction(predicates); + } + + // Hardware.HardwarePredicate.FromHardware. + std::shared_ptr + hardwarePredicate(Value const& isaValue, Value const* cuCount, Value const* deviceNames) + { + auto isa = isaValue.list(); + if(isa.size() < 3) + isaValue.fail("an ISA version"); + auto stepping = isa[2].integer(); + std::ostringstream step; + if(stepping < 0) + step << '-'; + step << std::hex << (stepping < 0 ? -uint64_t(stepping) : uint64_t(stepping)); + auto gfxArch = "gfx" + std::to_string(isa[0].integer()) + + std::to_string(isa[1].integer()) + step.str(); + + auto processor = AMDGPU::toProcessor(gfxArch); + if(AMDGPU::toString(processor) != gfxArch) + throw std::runtime_error("Solution state ISA names an unknown processor " + + gfxArch); + + std::vector> props; + props.push_back(std::make_shared(processor)); + if(cuCount && !cuCount->isNone()) + props.push_back( + std::make_shared(int(cuCount->integer()))); + if(gfxArch == "gfx950") + if(auto pciChipId = pciChipIdPredicate(deviceNames)) + props.push_back(pciChipId); + + return std::make_shared>( + props.size() == 1 ? props.front() : conjunction(props)); + } + + // Contractions.SizeMapping.FromOriginalState. + SizeMapping sizeMappingFromState(Dict const& d, SizeMapping s) + { + auto normalized = normalizeExecutionPolicy(d, false); + + int globalAccum = 0; + auto const& accumulation = d["_GlobalAccumulation"]; + if(accumulation.isText("SingleBuffer")) + globalAccum = 1; + if(accumulation.isText("MultipleBuffer")) + globalAccum = 2; + if(accumulation.isText("MultipleBufferSingleKernel")) + globalAccum = 3; + if(accumulation.isText("PartialsBuffer")) + globalAccum = 4; + auto pgr = int64_t(d["PrefetchGlobalRead"].number()); + + double tile; + if(d.truthy("EnableMatrixInstruction")) + { + auto miWaveTile = d["MIWaveTile"].list(); + if(miWaveTile.size() < 2) + d["MIWaveTile"].fail("a list of 2 values"); + tile = miWaveTile[0].number() * miWaveTile[1].number(); + } + else + tile = d["ThreadTile0"].number() * d["ThreadTile1"].number(); + double elements = !d["NumElementsPerBatchStore"].equals(0) + ? std::ceil(quotient(d["NumElementsPerThread"], + d["NumElementsPerBatchStore"])) + : d["NumElementsPerThread"].number(); + double waves = std::ceil(quotient(d["NumThreads"], d["WavefrontSize"])); + auto synchronizerSizePerWG = std::ceil(tile * elements * waves); + + auto workGroupMapping = [&]() -> int { + if(d["SpaceFillingAlgo"].list().empty()) + return int(d.integer("WorkGroupMapping")); + // convertSFCWGMListToHex: [[M1, N1], [M2, N2]] packed from the + // low byte up as M1, N1, M2, N2, read back as a signed int. + uint64_t output = 0; + auto levels = d["SFCWGM"].list(); + for(size_t idx = 0; idx < levels.size(); idx++) + { + auto dims = levels[idx].list(); + if(dims.size() < 2) + levels[idx].fail("a list of 2 values"); + if(16 * idx < 32) + output |= uint64_t(dims[0].integer()) << (16 * idx); + if(8 + 16 * idx < 32) + output |= uint64_t(dims[1].integer()) << (8 + 16 * idx); + } + return static_cast(static_cast(output & 0xFFFFFFFF)); + }; + + bool dtva = d.truthy("DirectToVgprA"); + bool dtvb = d.truthy("DirectToVgprB"); + bool dtlA = d.truthy("DirectToLdsA"); + bool dtlB = d.truthy("DirectToLdsB"); + + s.waveNum = floorQuotient(d["NumThreads"], d["WavefrontSize"]); + assignSequence(d["WorkGroup"], s.workGroupSize); + s.macroTile = dim3(d.integer("MacroTile0"), d.integer("MacroTile1"), 1); + assignSequence(d["ThreadTile"], s.threadTile); + assignArray(d["MatrixInstruction"], s.matrixInstruction); + s.grvwA = d.integer("GlobalReadVectorWidthA"); + s.grvwB = d.integer("GlobalReadVectorWidthB"); + s.gwvwC = d.integer("StoreVectorWidth"); + s.gwvwD = d.integer("StoreVectorWidth"); + s.workGroupMapping = workGroupMapping(); + s.staggerU = d.integer("StaggerU", 0); + s.staggerUMapping = d.integer("StaggerUMapping", 0); + s.depthU = d.integer("DepthU"); + s.globalSplitUPGR = GlobalSplitUPGR; + s.globalSplitU = int16_t(d.integer("GlobalSplitU")); + s.staggerStrideShift = d.integer("_staggerStrideShift", 0); + s.packBatchDims = 0; + + s.tileProcessingStrategy = normalized.policy.strategy; + s.workAssignment = normalized.policy.assignment; + s.streamKAtomic = int(normalized.streamKAtomic); + s.prefetchAcrossPersistent = int(normalized.prefetchAcrossPersistent); + s.magicDivAlg = int(d.integer("MagicDivAlg", 1)); + s.sourceKernel = d["KernelLanguage"].isText("Source"); + s.globalAccumulation = globalAccum; + s.adaptiveGemmGSUA = int(d.integer("AdaptiveGemmGSUA", 0)); + s.workspaceSizePerElemC = d.integer("_WorkspaceSizePerElemC"); + s.workspaceSizePerElemBias = d.integer("_WorkspaceSizePerElemBias"); + s.activationFused = d.truthy("ActivationFused"); + s.workGroupMappingXCC = int(d.integer("WorkGroupMappingXCC")); + s.workGroupMappingXCCGroup = int(d.integer("WorkGroupMappingXCCGroup")); + s.globalSplitUCoalesced = d.truthy("GlobalSplitUCoalesced"); + s.globalSplitUWorkGroupMappingRoundRobin + = d.truthy("GlobalSplitUWorkGroupMappingRoundRobin"); + + s.CUOccupancy = int(d.integer("CUOccupancy")); + s.PrefetchGlobalRead = int(pgr); + s.MathClocksUnrolledLoop = int(d.integer("MathClocksUnrolledLoop")); + s.synchronizerSizePerWG = size_t(synchronizerSizePerWG); + s.nonTemporalA = int(d.integer("NonTemporalA")); + s.nonTemporalB = int(d.integer("NonTemporalB")); + s.temporalHintA = int(d.integer("TemporalHintA", 0)); + s.temporalHintB = int(d.integer("TemporalHintB", 0)); + s.hasTemporalHint = d.truthy("_HasTemporalHint", false); + s.adaptiveGemmNTAB = int(d.integer("AdaptiveGemmNTAB", 0)); + s.customMainLoopScheduling = int(d.integer("UseCustomMainLoopSchedule")); + s.useSubtileImpl = d.truthy("UseSubtileImpl", false); + + s.NonTemporalD = int(d.integer("NonTemporalD")); + s.WaveSeparateGlobalReadA = int(d.integer("WaveSeparateGlobalReadA")); + s.WaveSeparateGlobalReadB = int(d.integer("WaveSeparateGlobalReadB")); + s.UnrollLoopSwapGlobalReadOrder = int(d.integer("UnrollLoopSwapGlobalReadOrder")); + s.DirectToVgprA = dtva; + s.DirectToVgprB = dtvb; + s.NumLoadsCoalescedA = int(d.integer("NumLoadsCoalescedA")); + s.NumLoadsCoalescedB = int(d.integer("NumLoadsCoalescedB")); + assignArray(d["MIWaveGroup"], s.waveGroup); + s.VectorWidthA = int(d.integer("VectorWidthA")); + s.VectorWidthB = int(d.integer("VectorWidthB")); + s.LocalSplitU = int(d.integer("LocalSplitU")); + s.DirectToLdsA = dtlA; + s.DirectToLdsB = dtlB; + s.WavefrontSize = int(d.integer("WavefrontSize")); + s.OneLDSBuffer = d.integer("1LDSBuffer", 0) == 1; + s.SourceSwap = d.truthy("SourceSwap", false); + s.UnrollMajorLDSA = d.has("UnrollMajorLDSA") ? int(d.truthy("UnrollMajorLDSA")) : -1; + s.UnrollMajorLDSB = d.has("UnrollMajorLDSB") ? int(d.truthy("UnrollMajorLDSB")) : -1; + s.enableLDSTrA = d.truthy("enableLDSTrA", false); + s.enableLDSTrB = d.truthy("enableLDSTrB", false); + s.BufferLoad = !d.has("BufferLoad") || d.truthy("BufferLoad"); + s.BufferStore = !d.has("BufferStore") || d.truthy("BufferStore"); + s.expertSchedulingMode = int(d.integer("ExpertSchedulingMode", 0)); + assignSequence(d["ClusterDim"], s.clusterDim); + s.validateExecutionPolicy(); + return s; + } + + // Contractions.InternalArgsSupport.FromOriginalState. + ContractionSolution::InternalArgsSupport internalArgsSupportFromState(Dict const& d) + { + auto isp = d.dict("InternalSupportParams"); + bool useSFC = isp.truthy("UseSFC") + || (d.has("SpaceFillingAlgo") && !d["SpaceFillingAlgo"].list().empty()); + ContractionSolution::InternalArgsSupport rv; + rv.version = isp.integer("KernArgsVersion"); + rv.persistentLoopArgsVersion = isp.integer("PersistentLoopArgsVersion", 0); + rv.gsu = isp.truthy("SupportUserGSU"); + rv.wgm = isp.truthy("SupportCustomWGM"); + rv.staggerU = isp.truthy("SupportCustomStaggerU"); + rv.perTileExtraIters = isp.truthy("SupportStreamKPerTileExtraIters", false); + rv.occupancyProbe = isp.truthy("SupportOccupancyProbe", false); + rv.useUniversalArgs = isp.truthy("UseUniversalArgs"); + rv.useSFC = useSFC; + return rv; + } + + // Contractions.CustomKernel.FromOriginalState, read as the library + // reads the record's customKernel. + CustomKernel customKernelFromState(Dict const& d) + { + CustomKernel rv; + rv.name = d.text("name"); + for(auto const& argValue : d["args"].list()) + { + Dict arg(argValue); + CustomArgDefinition definition; + definition.type = enumByName( + arg["type"], + CustomArgType_Count, + [](CustomArgType t) { return toString(t); }, + "a custom argument type"); + definition.semantic = enumByName( + arg["semantic"], + static_cast(CustomArgSemantic::COUNT), + [](CustomArgSemantic t) { return toString(t); }, + "a custom argument semantic"); + definition.padding = arg.integer("padding", 0); + definition.index = arg.integer("index", 0); + rv.args.push_back(definition); + } + assignSequence(d["macrotile"], rv.macrotile); + assignSequence(d["threads"], rv.threads); + auto grid = d["grid"].list(); + if(grid.size() > 3) + d["grid"].fail("a list of at most 3 values"); + CustomGridSize* gridSizes[] = {&rv.grid.x, &rv.grid.y, &rv.grid.z}; + for(size_t i = 0; i < grid.size(); i++) + *gridSizes[i] = enumByName( + grid[i], + CustomGridSize_Count, + [](CustomGridSize t) { return toString(t); }, + "a custom grid size"); + if(d.has("workspaceType")) + rv.workspaceType = enumByName( + d["workspaceType"], + CustomWorkspaceType_Count, + [](CustomWorkspaceType t) { return toString(t); }, + "a custom workspace type"); + rv.workspaceSizePerElemC = d.integer("workspaceSizePerElemC", 0); + rv.workspaceSizePerElemBias = d.integer("workspaceSizePerElemBias", 0); + rv.generated = d.truthy("generated", false); + rv.uniformSummationOrder = d.truthy("uniformSummationOrder", false); + return rv; + } + } // namespace + + std::vector supportedBuildChipIds() + { + std::vector chipIds; + chipIds.reserve(SupportedBuildChipIds.size()); + for(auto chip : SupportedBuildChipIds) + chipIds.emplace_back(chip); + return chipIds; + } + + std::vector supportedDataTypeNames() + { + std::vector names; + names.reserve(std::size(DataTypes)); + for(auto const& type : DataTypes) + names.push_back(type.name); + return names; + } + + std::shared_ptr solutionFromState(msgpack::object const& state) + { + Dict d{Value(state, "")}; + auto rv = std::make_shared(); + + if(d.has("SolutionNameMin")) + rv->solutionName = d.text("SolutionNameMin"); + if(d.has("KernelNameMin")) + rv->kernelName = d.text("KernelNameMin"); + + auto problemType = problemTypeFromState(d.dict("ProblemType")); + rv->problemType = problemType.record; + + int mxScaleFormat = 0; + if(auto const* value = d.find("MXScaleFormat")) + { + if(value->isText("HostPreSwizzle")) + mxScaleFormat = 1; + else if(value->isText("InMemorySwizzle")) + mxScaleFormat = 2; + } + rv->problemType.mxScaleFormat = mxScaleFormat; + + rv->problemPredicate = problemPredicateFromState(d, problemType); + rv->taskPredicate = taskPredicateFromState(d); + + if(d.has("DebugKernel")) + rv->debugKernel = d.truthy("DebugKernel"); + + if(d.has("SolutionIndex")) + rv->index = d.integer("SolutionIndex"); + + rv->libraryLogicIndex = d.integer("SolutionIndex", -1); + + rv->sizeMapping = sizeMappingFromState(d, rv->sizeMapping); + if(d.has("CustomKernel")) + rv->customKernel = customKernelFromState(d.dict("CustomKernel")); + + rv->internalArgsSupport = internalArgsSupportFromState(d); + rv->validatePersistentLoopArgs(); + + if(auto const* ideals = d.find("Ideals")) + { + if(ideals->object().type != object_type::MAP) + ideals->fail("a map"); + auto const& map = ideals->object().via.map; + for(uint32_t i = 0; i < map.size; i++) + { + Value key(map.ptr[i].key, "Ideals key"); + Value value(map.ptr[i].val, "Ideals value"); + int index = 0; + try + { + index = key.object().type == object_type::STR ? std::stoi(key.text()) + : int(key.integer()); + } + catch(std::invalid_argument const&) + { + throw std::runtime_error("Ideals key is not an integer: " + key.text()); + } + catch(std::out_of_range const&) + { + throw std::runtime_error("Ideals key is out of range: " + key.text()); + } + rv->ideals[index] = value.number(); + } + } + + if(auto const* linearModel = d.find("LinearModel")) + { + Dict model(*linearModel); + if(auto const* value = model.find("slope")) + rv->linearModel.slope = value->number(); + if(auto const* value = model.find("intercept")) + rv->linearModel.intercept = value->number(); + if(auto const* value = model.find("max")) + rv->linearModel.max = value->number(); + } + + rv->hardwarePredicate + = hardwarePredicate(d["ISA"], d.find("CUCount"), d.find("DeviceNames")); + + return rv; + } +} // namespace TensileLite diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/derive_solution_records.py b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/derive_solution_records.py new file mode 100644 index 000000000000..353914bfd875 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/derive_solution_records.py @@ -0,0 +1,209 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Writes the derived states in test_data/solution_records/.yaml. + +This is not a test module. A derived state is the map the library record is +built from: what Solution.FromOriginalState reads and what solutionFromState +reads in C++. Each state comes from the Python pipeline, not from a hand-built +dict: a solution of a checked-in library logic file or of a benchmark config +runs through the same derivation the library build uses, and the state is +written the way LibraryIO.writeSolutions writes it. + + python tensilelite/Tests/unit/derive_solution_records.py [name ...] + +then regenerate the record, predicate and msgpack files: + + python tensilelite/Tests/unit/test_solution_state_record.py + +The three *-plain states are not made here; they are the custom.config blocks +of three generated kernels. + +A few fixtures change one field of a derived state, because no checked-in +logic file or config reaches that field on a real kernel (an MX free-block +count other than 1, the occupancy probe, ReuseAcrossPersistent). The edit is +listed next to the source, and test_solution_state_record.py builds both +records from the edited state, so the two builders see the same input. +""" + +import copy +import sys +from pathlib import Path + +import yaml + +_UNIT = Path(__file__).resolve().parent +_CODEGEN = _UNIT / "characterization" / "_codegen" +for _path in (_UNIT.parent, _UNIT, _CODEGEN): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +_LOGIC = ( + _UNIT.parents[3] + / "library" + / "src" + / "amd_detail" + / "rocblaslt" + / "src" + / "Tensile" + / "Logic" + / "asm_full" +) + + +def records_dir(): + return _UNIT / "test_data" / "solution_records" + + +def _serialize(solution): + """One solution as LibraryIO.writeSolutions writes it.""" + from tensilelite.SolutionStructs.Problem import problemTypeToEnum + + state = copy.copy(solution.getAttributes()) + state["ProblemType"] = copy.deepcopy(state["ProblemType"].state) + problemTypeToEnum(state["ProblemType"]) + isa = state["ISA"] + state["ISA"] = [isa[0], isa[1], isa[2]] + return copy.deepcopy(state) + + +def _logic_solution(relative, arch, pick): + """The first solution of a logic file, after derivation, that `pick` accepts.""" + from config_harness import _isolated_globals_with_isa, _toolchain_for + from tensilelite import LibraryIO + + path = _LOGIC / relative + data = yaml.load(path.read_text(), Loader=yaml.CSafeLoader) + if isinstance(data, list): + data = LibraryIO.parseLibraryLogicList(data, str(path)) + assembler, isa_info_map = _toolchain_for(arch) + with _isolated_globals_with_isa(isa_info_map): + logic = LibraryIO.parseLibraryLogicData( + data, str(path), assembler, False, False, False, isa_info_map, False + ) + matches = [solution for solution in logic.solutions if pick(solution.getAttributes())] + if not matches: + raise RuntimeError(f"no solution of {relative} matches") + return _serialize(matches[0]) + + +def _config_solution(config, arch, index): + from config_harness import solutions_from_config + + solutions = solutions_from_config(config, arch=arch, limit_solutions=index + 1) + return _serialize(solutions[index]) + + +def _tile(mt0, mt1, depth_u, grvw_a, grvw_b): + return lambda s: ( + s.get("MacroTile0"), + s.get("MacroTile1"), + s.get("DepthU"), + s.get("GlobalReadVectorWidthA"), + s.get("GlobalReadVectorWidthB"), + ) == (mt0, mt1, depth_u, grvw_a, grvw_b) + + +def _mx_free_blocks(state): + state["ProblemType"]["MXBlockFreeA"] = 7 + state["ProblemType"]["MXBlockFreeB"] = 4 + return state + + +def _rap_and_probe(state): + state["ReuseAcrossPersistent"] = 1 + state["_RAPNumResidentKTiles"] = 3 + state["InternalSupportParams"]["SupportOccupancyProbe"] = True + return state + + +# name -> (build state, edit applied to the state or None) +_SOURCES = { + # fp8 x bf8 StreamK on gfx942: a 256x224 tile, DepthU 32, GlobalReadVectorWidth 8 for A + # and 4 for B. A swap of the two operands or of M and N changes the predicates. + "gfx942-streamk-fp8": ( + lambda: _logic_solution( + "aquavanjaram/gfx942/StreamK/gfx942_Cijk_Alik_Bjlk_F8B8NSS_BH_UserArgs.yaml", + "gfx942", + _tile(256, 224, 32, 8, 4), + ), + None, + ), + # fp8 MX on gfx950: MXBlockA/B 32, HostPreSwizzle scale format. + "gfx950-mx": ( + lambda: _logic_solution( + "gfx950/gfx950/GridBased/gfx950_Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_Bias_GG_HA_S_SAB_SAV_UserArgs.yaml", + "gfx950", + lambda s: True, + ), + None, + ), + # The same kernel with MX free-block counts that differ per operand. + "gfx950-mx-free-blocks": ( + lambda: _logic_solution( + "gfx950/gfx950/GridBased/gfx950_Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_Bias_GG_HA_S_SAB_SAV_UserArgs.yaml", + "gfx950", + lambda s: True, + ), + _mx_free_blocks, + ), + # int4 A with three scale group sizes and the UnsignedBias8 encoding on gfx1151: + # a 64x160 tile, DepthU 64. + "gfx1151-int4": ( + lambda: _logic_solution( + "gfx1151/FreeSize/gfx1151_Cijk_Alik_Bljk_I4B_BBS_BH_SABBGU8_UserArgs.yaml", + "gfx1151", + lambda s: True, + ), + None, + ), + # fp8 with stochastic rounding on gfx942: a 16x16 tile, GlobalReadVectorWidth 4 and 8. + "gfx942-stochastic-rounding": ( + lambda: _logic_solution( + "aquavanjaram/gfx942/GridBased/aquavanjaram_Cijk_Ailk_Bljk_HH_F8NHS_BH_BiasSH_HAS_SAV_UserArgs.yaml", + "gfx942", + _tile(16, 16, 32, 4, 8), + ), + None, + ), + # StreamK with a spatial cluster on gfx1250: PersistentLoopArgsVersion 2. + "gfx1250-streamk-cluster": ( + lambda: _config_solution( + "tensilelite/Tests/common/streamk/gfx1250/static_streamk_cluster_alpha0.yaml", + "gfx1250", + 0, + ), + None, + ), + # The cluster kernel with ReuseAcrossPersistent and the occupancy probe set. + "gfx1250-streamk-cluster-rap-probe": ( + lambda: _config_solution( + "tensilelite/Tests/common/streamk/gfx1250/static_streamk_cluster_alpha0.yaml", + "gfx1250", + 0, + ), + _rap_and_probe, + ), +} + + +def derived_sources(): + return sorted(_SOURCES) + + +def derive(name): + build, edit = _SOURCES[name] + state = build() + return edit(state) if edit else state + + +def main(names): + for name in names or derived_sources(): + state = derive(name) + path = records_dir() / f"{name}.yaml" + path.write_text(yaml.safe_dump(state, sort_keys=True)) + print(name, len(state)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/catalog_chip_ids.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/catalog_chip_ids.txt new file mode 100644 index 000000000000..19049a6a3e01 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/catalog_chip_ids.txt @@ -0,0 +1,14 @@ +74a0 +74a1 +74a2 +74a3 +74a5 +74a9 +75a0 +75b0 +75a2 +75b2 +75a3 +75b3 +75a8 +75b8 diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/catalog_data_types.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/catalog_data_types.txt new file mode 100644 index 000000000000..a0fde116e413 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/catalog_data_types.txt @@ -0,0 +1,25 @@ +Float +Double +ComplexFloat +ComplexDouble +Half +Int8x4 +Int32 +BFloat16 +Int8 +Int64 +XFloat32 +Float8_fnuz +BFloat8_fnuz +Float8BFloat8_fnuz +BFloat8Float8_fnuz +Float8 +BFloat8 +Float8BFloat8 +BFloat8Float8 +Float6 +BFloat6 +Float4 +E8 +E5M3 +Int4 diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.msgpack new file mode 100644 index 000000000000..d9d6dcaedcb5 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.predicates.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.predicates.msgpack new file mode 100644 index 000000000000..9a732af40413 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.predicates.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.predicates.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.predicates.yaml new file mode 100644 index 000000000000..a189b3b83b2d --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.predicates.yaml @@ -0,0 +1,118 @@ +problem: + type: And + value: + - type: BoundSizeMultiple + index: -1 + value: 64 + - type: OperationIdentifierEqual + value: Contraction_l_Alik_Bljk_Cijk_Dijk + - type: BiasDataTypeWhiteList + value: [] + - type: GateResidualDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: + - 3 + - type: AmaxDCheck + value: false + - type: TypesEqual + value: + - Int4 + - BFloat16 + - BFloat16 + - BFloat16 + - BFloat16 + - BFloat16 + - type: HighPrecisionAccumulate + value: true + - type: Activation + value: None + - type: ActivationComputeType + value: Float + - type: ActivationNoGuard + value: false + - type: UseGradient + value: false + - type: UseBias + value: 0 + - type: UseE + value: false + - type: UseGateResidual + value: false + - type: DataTypeE + value: BFloat16 + - type: StridedBatched + value: true + - type: GroupedGemm + value: false + - type: UseScaleAB + value: '' + - type: Or + value: + - type: ScaleBlockSizeA + value: 32 + - type: ScaleBlockSizeA + value: 64 + - type: ScaleBlockSizeA + value: 128 + - type: ScaleZeroPointA + value: false + - type: Int4EncodingA + value: UnsignedBias8 + - type: UseScaleCD + value: false + - type: UseScaleAlphaVec + value: 0 + - type: Sparse + value: 0 + - type: F32XdlMathOp + value: Float + - type: SupportDeviceUserArguments + value: true + - type: SwizzleTensorA + value: false + - type: SwizzleTensorB + value: false + - type: FusedGemmA2A + value: false + - type: MXBlockA + value: 0 + - type: MXBlockB + value: 0 + - type: WorkgroupNumberCheck + index: 0 + value: + - 64 + - 160 + - 1 + - type: LeadingFree0SizesGreaterOrEqual + value: 1 + - type: LeadingFree1SizesGreaterOrEqual + value: 1 + - type: KernelLanguageCompatible + value: Assembly + - type: BufferLoadOffsetLimitCheck + value: + ShiftPtrElemB: 0 + ShiftPtrElemA: 0 + DUorMT1: 160 + DUorMT0: 64 + - type: BufferLoadOffsetLimitCheck_Beta + value: 160 + - type: BufferStoreOffsetLimitCheck + value: 160 + - type: GlobalSplitUCheckMinK + value: + - 32 + - 1 + - type: WorkgroupMappingXCCCheck + value: + - 1 + - -1 +task: + type: LaunchLimits +hardware: + type: AMDGPU + value: + type: Processor + value: gfx1151 diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.record.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.record.txt new file mode 100644 index 000000000000..429995aa0038 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.record.txt @@ -0,0 +1,139 @@ +solutionName="Cijk_Alik_Bljk_I4B_BBS_BH_SABB128U8_UserArgs_MT64x160x64_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM64_BL1_BS1_CD1_1_CLR0_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA1151_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB8_LPMXSA0_LPMXSB0_LPMn1_LRVW16_LWPMn1_MIAV1_MIWT2_5_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR2_PLR0_PKA0_SGROB0_SIA0_SS1_SU0_SUM0_SUS128_SPO0_SRVW0_SSO0_SVW1_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS32_WG32_4_1_WGM8_WGMXCC1_WGMXCCGn1" +kernelName="RuntimeGroup_Cijk_Alik_Bljk_I4B_BBS_BH_SABBGU8_UserArgs_MT64x160x64_MI16x16x1_gfx1151" +sizeMapping.waveNum=4 +sizeMapping.WavefrontSize=32 +sizeMapping.workGroup=[32,4,1] +sizeMapping.macroTile=[64,160,1] +sizeMapping.threadTile=[1,1] +sizeMapping.matrixInstruction=[16,16,16,1] +sizeMapping.grvwA=1 +sizeMapping.grvwB=1 +sizeMapping.gwvwC=1 +sizeMapping.gwvwD=1 +sizeMapping.depthU=64 +sizeMapping.staggerU=0 +sizeMapping.staggerUMapping=0 +sizeMapping.globalSplitUPGR=16 +sizeMapping.globalSplitU=1 +sizeMapping.staggerStrideShift=0 +sizeMapping.workGroupMapping=8 +sizeMapping.packBatchDims=0 +sizeMapping.magicDivAlg=2 +sizeMapping.tileProcessingStrategy="None" +sizeMapping.workAssignment="StaticGrid" +sizeMapping.streamKAtomic=0 +sizeMapping.prefetchAcrossPersistent=0 +sizeMapping.sourceKernel=false +sizeMapping.globalAccumulation=2 +sizeMapping.adaptiveGemmGSUA=0 +sizeMapping.workspaceSizePerElemC=0 +sizeMapping.workspaceSizePerElemBias=0 +sizeMapping.activationFused=true +sizeMapping.workGroupMappingXCC=1 +sizeMapping.workGroupMappingXCCGroup=-1 +sizeMapping.globalSplitUCoalesced=false +sizeMapping.globalSplitUWorkGroupMappingRoundRobin=false +sizeMapping.CUOccupancy=-1 +sizeMapping.PrefetchGlobalRead=2 +sizeMapping.MathClocksUnrolledLoop=0 +sizeMapping.synchronizerSizePerWG=3200 +sizeMapping.nonTemporalA=0 +sizeMapping.nonTemporalB=0 +sizeMapping.temporalHintA=0 +sizeMapping.temporalHintB=0 +sizeMapping.hasTemporalHint=false +sizeMapping.adaptiveGemmNTAB=0 +sizeMapping.customMainLoopScheduling=0 +sizeMapping.useSubtileImpl=false +sizeMapping.NonTemporalD=0 +sizeMapping.WaveSeparateGlobalReadA=0 +sizeMapping.WaveSeparateGlobalReadB=0 +sizeMapping.UnrollLoopSwapGlobalReadOrder=0 +sizeMapping.DirectToVgprA=false +sizeMapping.DirectToVgprB=false +sizeMapping.NumLoadsCoalescedA=1 +sizeMapping.NumLoadsCoalescedB=1 +sizeMapping.WaveGroup=[2,2] +sizeMapping.VectorWidthA=1 +sizeMapping.VectorWidthB=1 +sizeMapping.LocalSplitU=1 +sizeMapping.DirectToLdsA=false +sizeMapping.DirectToLdsB=false +sizeMapping.OneLDSBuffer=false +sizeMapping.SourceSwap=true +sizeMapping.UnrollMajorLDSA=1 +sizeMapping.UnrollMajorLDSB=1 +sizeMapping.enableLDSTrA=false +sizeMapping.enableLDSTrB=false +sizeMapping.ExpertSchedulingMode=0 +sizeMapping.clusterDim=[1,1] +sizeMapping.BufferLoad=true +sizeMapping.BufferStore=true +internalArgsSupport.version=3 +internalArgsSupport.persistentLoopArgsVersion=0 +internalArgsSupport.gsu=true +internalArgsSupport.wgm=true +internalArgsSupport.staggerU=true +internalArgsSupport.perTileExtraIters=false +internalArgsSupport.useUniversalArgs=true +internalArgsSupport.useSFC=false +problemType.operationIdentifier="Contraction_l_Alik_Bljk_Cijk_Dijk" +problemType.transA=true +problemType.transB=false +problemType.computeInputTypeA="BFloat16" +problemType.computeInputTypeB="BFloat16" +problemType.aType="Int4" +problemType.bType="BFloat16" +problemType.cType="BFloat16" +problemType.dType="BFloat16" +problemType.eType="BFloat16" +problemType.computeType="Float" +problemType.useBeta=true +problemType.useBias=0 +problemType.biasSrcWhiteList=[3] +problemType.useE=false +problemType.useGateResidual=false +problemType.gateResidualDataTypeWhiteList=[] +problemType.useScaleAB="" +problemType.useScaleCD=false +problemType.useScaleAlphaVec=0 +problemType.biasDataTypeWhiteList=[] +problemType.highPrecisionAccumulate=true +problemType.useInitialStridesAB=false +problemType.useInitialStridesCD=false +problemType.stridedBatched=true +problemType.groupedGemm=false +problemType.useGradient=false +problemType.activationType="None" +problemType.activationArgLength=0 +problemType.activationComputeDataType="Float" +problemType.activationNoGuard=false +problemType.sparse=0 +problemType.f32XdlMathOp="Float" +problemType.supportDeviceUserArguments=true +problemType.outputAmaxD=false +problemType.swizzleTensorA=false +problemType.swizzleTensorB=false +problemType.metadataLayout=0 +problemType.mxBlockA=0 +problemType.mxBlockB=0 +problemType.mxBlockFreeA=1 +problemType.mxBlockFreeB=1 +problemType.mxTypeA="E8" +problemType.mxTypeB="E8" +problemType.mxScaleFormat=0 +problemType.fusedGemmA2A=false +problemType.stochasticRounding=false +problemPredicate=["And","BoundSizeMultiple","OperationIdentifierEqual","BiasDataTypeWhiteList","GateResidualDataTypeWhiteList","BiasSrcWhiteList","AmaxDCheck","TypesEqual","HighPrecisionAccumulate","Activation","ActivationComputeType","ActivationNoGuard","UseGradient","UseBias","UseE","UseGateResidual","DataTypeE","StridedBatched","GroupedGemm","UseScaleAB","Or","ScaleBlockSizeA","ScaleBlockSizeA","ScaleBlockSizeA","ScaleZeroPointA","Int4EncodingA","UseScaleCD","UseScaleAlphaVec","Sparse","F32XdlMathOp","SupportDeviceUserArguments","SwizzleTensorA","SwizzleTensorB","FusedGemmA2A","MXBlockA","MXBlockB","WorkgroupNumberCheck","LeadingFree0SizesGreaterOrEqual","LeadingFree1SizesGreaterOrEqual","KernelLanguageCompatible","BufferLoadOffsetLimitCheck","BufferLoadOffsetLimitCheck_Beta","BufferStoreOffsetLimitCheck","GlobalSplitUCheckMinK","WorkgroupMappingXCCCheck"] +taskPredicate=["LaunchLimits"] +hardwarePredicate=["AMDGPU","Processor"] +customKernel.name="RuntimeGroup_Cijk_Alik_Bljk_I4B_BBS_BH_SABBGU8_UserArgs_MT64x160x64_MI16x16x1_gfx1151" +customKernel.args=[{"semantic":"GemmInfo","type":"uint32"},{"semantic":"InternalArgs","type":"uint32"},{"semantic":"InternalArgs1","type":"uint32"},{"semantic":"NumWorkGroups","type":"uint32"},{"semantic":"SizeFree0","type":"uint32"},{"semantic":"SizeFree1","type":"uint32"},{"semantic":"SizeFree2","type":"uint32"},{"semantic":"SizeSum","type":"uint32"},{"semantic":"AddressA","type":"address"},{"semantic":"AddressB","type":"address"},{"semantic":"StrideA0","type":"uint32"},{"semantic":"StrideA1","type":"uint32"},{"semantic":"StrideB0","type":"uint32"},{"semantic":"StrideB1","type":"uint32"},{"semantic":"Alpha","type":"uint32"},{"semantic":"Beta","type":"uint32"},{"semantic":"AddressD","type":"address"},{"semantic":"AddressC","type":"address"},{"semantic":"StrideD0","type":"uint32"},{"semantic":"StrideD1","type":"uint32"},{"semantic":"StrideC0","type":"uint32"},{"semantic":"StrideC1","type":"uint32"},{"semantic":"AddressScaleA","type":"address"},{"semantic":"AddressScaleB","type":"address"},{"semantic":"BatchOffsetD","type":"uint64"},{"semantic":"BatchOffsetC","type":"uint64"},{"semantic":"BatchOffsetA","type":"uint64"},{"semantic":"BatchOffsetB","type":"uint64"},{"padding":4,"semantic":"ScaleBlockSizeA","type":"uint32"}] +customKernel.macrotile=[64,160,64] +customKernel.threads=[128,1,1] +customKernel.grid=["TilesXYBatchGSU","One","One"] +customKernel.workspaceType="None" +customKernel.workspaceSizePerElemC=0 +customKernel.workspaceSizePerElemBias=0 +customKernel.generated=false +customKernel.uniformSummationOrder=false diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.yaml new file mode 100644 index 000000000000..3a57cbf4305b --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1151-int4.yaml @@ -0,0 +1,584 @@ +1LDSBuffer: 0 +ActivationAlt: false +ActivationFuncCall: false +ActivationFused: true +AdaptiveGemm: 0 +AdaptiveGemmGSUA: 0 +AdaptiveGemmNTAB: 0 +AssertAIGreaterThanEqual: -1 +AssertAILessThanEqual: -1 +AssertFree0ElementMultiple: 1 +AssertFree1ElementMultiple: 1 +AssertSummationElementMultiple: 64 +AssignedDerivedParameters: false +AssignedProblemIndependentDerivedParameters: false +BaseName: RuntimeGroup_Cijk_Alik_Bljk_I4B_BBS_BH_SABBGU8_UserArgs_MT64x160x64_MI16x16x1_gfx1151 +BufferLoad: true +BufferStore: true +CUCount: null +CUOccupancy: -1 +ClusterAIDMapping: -1 +ClusterBarrier: false +ClusterDim: +- 1 +- 1 +ClusterLocalRead: 0 +CodeObjectVersion: '4' +CompactLoopStore: false +ConvertAfterDS: false +CustomKernel: + args: + - semantic: GemmInfo + type: uint32 + - semantic: InternalArgs + type: uint32 + - semantic: InternalArgs1 + type: uint32 + - semantic: NumWorkGroups + type: uint32 + - semantic: SizeFree0 + type: uint32 + - semantic: SizeFree1 + type: uint32 + - semantic: SizeFree2 + type: uint32 + - semantic: SizeSum + type: uint32 + - semantic: AddressA + type: address + - semantic: AddressB + type: address + - semantic: StrideA0 + type: uint32 + - semantic: StrideA1 + type: uint32 + - semantic: StrideB0 + type: uint32 + - semantic: StrideB1 + type: uint32 + - semantic: Alpha + type: uint32 + - semantic: Beta + type: uint32 + - semantic: AddressD + type: address + - semantic: AddressC + type: address + - semantic: StrideD0 + type: uint32 + - semantic: StrideD1 + type: uint32 + - semantic: StrideC0 + type: uint32 + - semantic: StrideC1 + type: uint32 + - semantic: AddressScaleA + type: address + - semantic: AddressScaleB + type: address + - semantic: BatchOffsetD + type: uint64 + - semantic: BatchOffsetC + type: uint64 + - semantic: BatchOffsetA + type: uint64 + - semantic: BatchOffsetB + type: uint64 + - padding: 4 + semantic: ScaleBlockSizeA + type: uint32 + grid: + - TilesXYBatchGSU + - One + - One + macrotile: + - 64 + - 160 + - 64 + name: RuntimeGroup_Cijk_Alik_Bljk_I4B_BBS_BH_SABBGU8_UserArgs_MT64x160x64_MI16x16x1_gfx1151 + threads: + - 128 + - 1 + - 1 + workspaceSizePerElemBias: 0 + workspaceSizePerElemC: 0 + workspaceType: None +CustomKernelName: RuntimeGroup_Cijk_Alik_Bljk_I4B_BBS_BH_SABBGU8_UserArgs_MT64x160x64_MI16x16x1_gfx1151 +DebugPersistentKernelLoopForever: false +DebugStreamK: 0 +DepthU: 64 +DeviceNames: +- Device 1536 +DirectToLds: 0 +DirectToLdsA: false +DirectToLdsB: false +DirectToLdsMetadata: 0 +DirectToVgprA: false +DirectToVgprB: false +DirectToVgprMXSA: false +DirectToVgprMXSB: false +DirectToVgprSparseMetadata: false +DtlPlusLdsBuf: 0 +ESMRuntimeGate: false +EdgeType: ShiftPtr +EnableESM2TrackValuVsrc: false +EnableF32XdlMathOp: false +EnableMatrixInstruction: true +EnableStinkyTofuESM2: false +ExpandPointerSwap: false +ExpertSchedulingMode: 0 +ExtraLatencyForLR: 0 +ExtraMiLatencyLeft: -1 +ForceDisableShadowInit: false +ForceUnrollSubIter: false +GlobalReadPerMfma: 1.0 +GlobalReadVectorWidthA: 1 +GlobalReadVectorWidthB: 1 +GlobalSplitU: 1 +GlobalSplitUAlgorithm: MultipleBuffer +GlobalSplitUCoalesced: false +GlobalSplitUWorkGroupMappingRoundRobin: false +GlobalWriteVectorWidth: 1 +GroupLoadStore: false +GuaranteeNoPartialA: true +GuaranteeNoPartialB: true +GuaranteeNoPartialMetadata: true +HalfPLR: 0 +HalfPLRA: false +HalfPLRB: false +ISA: +- 11 +- 5 +- 1 +InitCIterWmma: 0 +InnerUnroll: 1 +InterleaveAlpha: 0 +InternalSupportParams: + KernArgsVersion: 3 + PersistentLoopArgsVersion: 0 + SupportCustomStaggerU: true + SupportCustomWGM: true + SupportOccupancyProbe: false + SupportStreamKPerTileExtraIters: false + SupportUserGSU: true + UseSFC: false + UseUniversalArgs: true +Kernel: true +KernelLanguage: Assembly +KernelNameMin: RuntimeGroup_Cijk_Alik_Bljk_I4B_BBS_BH_SABBGU8_UserArgs_MT64x160x64_MI16x16x1_gfx1151 +LDSSegInterleaveOffsets: null +LDSSegmentBRotate: 0 +LDSSegmentInterleave: 0 +LDSTrInst: false +LSCA: 64 +LSCB: 64 +LSPA: 16 +LSPB: 16 +LVCA: 8 +LVCB: 8 +LVPA: 2 +LVPB: 2 +LdsBlockSizePerPadA: 128 +LdsBlockSizePerPadB: 128 +LdsBlockSizePerPadMXSA: 0 +LdsBlockSizePerPadMXSB: 0 +LdsBlockSizePerPadMetadata: 0 +LdsBytesNoAmax: 65024 +LdsInitCVgprs: false +LdsNumBytes: 65024 +LdsNumElementsAlignedA: 9216 +LdsNumElementsAlignedB: 23040 +LdsNumElementsAlignedMXSA: 0 +LdsNumElementsAlignedMXSB: 0 +LdsNumElementsAlignedMetadata: 0 +LdsOffsetA: 0 +LdsOffsetA_Blk: 32768 +LdsOffsetB: 9216 +LdsOffsetB_Blk: 41984 +LdsOffsetBias: 0 +LdsOffsetBiasGSU: 0 +LdsOffsetBiasNonGSU: 0 +LdsOffsetMXSA: 9216 +LdsOffsetMXSA_Blk: 41984 +LdsOffsetMXSB: 9216 +LdsOffsetMXSB_Blk: 41984 +LdsOffsetMetadata: 9216 +LdsOffsetMetadata_Blk: 41984 +LdsPadA: 8 +LdsPadB: 8 +LdsPadMXSA: 0 +LdsPadMXSB: 0 +LdsPadMetadata: -1 +LocalReadVectorWidth: 16 +LocalReadVectorWidthA: 16 +LocalReadVectorWidthB: 16 +LocalSplitU: 1 +LocalSplitUReuseLDS: 1 +LocalWritePerMfma: -1 +LocalWriteUseSgprA: false +LocalWriteUseSgprB: false +LocalWriteUseSgprMetadata: false +LoopIters: 4 +LoopUnroll: 64 +MFMA_BF16_1K: 0 +MIArchVgpr: true +MIBlock: +- 16 +- 16 +- 16 +- 1 +- 1 +- 1 +MIInputPerThread: 16 +MIInputPerThreadA: 16 +MIInputPerThreadB: 16 +MIInputPerThreadMetadata: 16 +MIOutputVectorWidth: 1 +MIRegPerOut: 1 +MIWaveGroup: +- 2 +- 2 +MIWaveTile: +- 2 +- 5 +MIWaveTileA: 2 +MIWaveTileB: 5 +MIWaveTileMetadata: 0 +MXLoadInst: BufferLoad +MXScaleFormat: NoSwizzle +MacroTile0: 64 +MacroTile1: 160 +MacroTileA: 64 +MacroTileB: 160 +MagicDivAlg: 2 +MathClocksUnrolledLoop: 0 +MatrixInstB: 1 +MatrixInstBM: 1 +MatrixInstBN: 1 +MatrixInstK: 16 +MatrixInstM: 16 +MatrixInstN: 16 +MatrixInstruction: +- 16 +- 16 +- 16 +- 1 +MaxLDS: 65536 +MaxOccupancy: 64 +MbskPrefetchMethod: 0 +MfmaInitCVgprs: false +MinGRIncPerMfma: 1 +Multicast: false +NoLdsWriteCode: false +NoReject: false +NoTailLoop: true +NonDTLTailLoopA: false +NonDTLTailLoopB: false +NonDTLTailLoopMetadata: false +NonTemporal: -1 +NonTemporalA: 0 +NonTemporalB: 0 +NonTemporalC: 0 +NonTemporalD: 0 +NonTemporalE: 0 +NonTemporalGate: 0 +NonTemporalMXSA: 0 +NonTemporalMXSB: 0 +NonTemporalMetadata: 0 +NonTemporalWS: 0 +NonVolatile: -1 +NonVolatileA: 0 +NonVolatileB: 0 +NonVolatileC: 0 +NonVolatileD: 0 +NonVolatileE: 0 +NonVolatileGate: 0 +NonVolatileMXSA: 0 +NonVolatileMXSB: 0 +NonVolatileMetadata: 0 +NonVolatileWS: 0 +NumElementsPerBatchStore: 0 +NumElementsPerThread: 80 +NumGlobalWriteVectorsPerThread: 80 +NumLdsBlk: 2 +NumLoadsA: 4 +NumLoadsB: 10 +NumLoadsCoalescedA: 1 +NumLoadsCoalescedB: 1 +NumLoadsPerpendicularA: 4 +NumLoadsPerpendicularB: 10 +NumThreads: 128 +NumTotalPackedLoadsA: -1 +NumTotalPackedLoadsB: -1 +NumWaveSplitK: 1 +NumWaves: 4 +OptNoLoadLoop: 0 +PackedC0IdxChars: +- I +PackedC0IndicesX: [] +PackedC1IdxChars: +- J +PackedC1IndicesX: +- 1 +PersistentXCCMapping: 0 +PostLoopStoreInNll: true +PrefetchAcrossPersistent: 0 +PrefetchGL2: 0 +PrefetchGlobalRead: 2 +PrefetchLocalRead: 0 +PreloadKernArgs: false +ProblemType: + Activation: false + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: none + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: false + BiasDataTypeList: [] + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 7 + DataTypeA: 24 + DataTypeAmaxD: 0 + DataTypeB: 7 + DataTypeE: 7 + DataTypeMXSA: 22 + DataTypeMXSB: 22 + DestDataType: 7 + F32XdlMathOp: 0 + FusedGemmA2A: false + GateResidualDataTypeList: [] + Gradient: false + GroupedGemm: false + HighPrecisionAccumulate: true + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: + - 3 + - 0 + - 2 + IndexAssignmentsB: + - 3 + - 1 + - 2 + IndexAssignmentsLD: + - 4 + - 5 + - 6 + - 7 + IndexAssignmentsMetadata: + - 3 + - 0 + - 2 + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndexUnrollM: 0 + IndicesBatch: + - 2 + IndicesFree: + - 0 + - 1 + IndicesSummation: + - 3 + Int4EncodingA: UnsignedBias8 + MXBlockA: 0 + MXBlockB: 0 + MXBlockFreeA: 1 + MXBlockFreeB: 1 + MacDataTypeA: 7 + MacDataTypeB: 7 + MetadataLayout: 0 + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMXSA: [] + MirrorDimsMXSB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + ScaleBlockSizesA: + - 32 + - 64 + - 128 + ScaleZeroPointA: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SetConstStrideGate: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: false + StridedBatched: true + SupportUserArgs: true + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseBias: 0 + UseE: false + UseGateResidual: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: '' + UseScaleAlphaVec: 0 + UseScaleCD: false + isMixMode: false +ReuseAcrossPersistent: 0 +SFCWGM: +- - 1 + - 1 +- - 1 + - 1 +ScheduleGROverBarrier: 0 +ScheduleGlobalRead: 1 +ScheduleIterAlg: 0 +ScheduleLocalWrite: 1 +SolutionIndex: 0 +SolutionNameMin: Cijk_Alik_Bljk_I4B_BBS_BH_SABB128U8_UserArgs_MT64x160x64_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM64_BL1_BS1_CD1_1_CLR0_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA1151_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB8_LPMXSA0_LPMXSB0_LPMn1_LRVW16_LWPMn1_MIAV1_MIWT2_5_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR2_PLR0_PKA0_SGROB0_SIA0_SS1_SU0_SUM0_SUS128_SPO0_SRVW0_SSO0_SVW1_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS32_WG32_4_1_WGM8_WGMXCC1_WGMXCCGn1 +SourceSwap: true +SpaceFillingAlgo: [] +StaggerU: 0 +StaggerUMapping: 0 +StaggerUStride: 128 +StinkyTofuRegisterAllocation: 0 +StorePriorityOpt: false +StoreRemapVectorWidth: 0 +StoreSwapAddr: false +StoreSyncOpt: 0 +StoreVectorWidth: 1 +StreamKAtomic: 0 +StreamKFixupTreeReduction: 0 +SubGroup0: 4 +SubGroup1: 32 +SubGroupA: 4 +SubGroupB: 32 +SuppressNoLoadLoop: false +SwInstructionPrefetch: 1 +SwapGlobalReadOrder: 0 +TDMInst: 0 +TDMIterateMode: 0 +TDMLoadWaveSync: false +TDMPlusLdsBuf: 0 +TDMSplit: false +TailloopInNll: false +TemporalHint: -1 +TemporalHintA: 0 +TemporalHintB: 0 +TemporalHintC: 0 +TemporalHintD: 0 +TemporalHintE: 0 +TemporalHintGate: 0 +TemporalHintMXSA: 0 +TemporalHintMXSB: 0 +TemporalHintMetadata: 0 +TemporalHintWS: 0 +ThreadTile: +- 1 +- 1 +ThreadTile0: 0 +ThreadTile1: 0 +ThreadTileA: 16 +ThreadTileB: 5 +TileProcessingStrategy: None +TransposeLDS: 1 +TransposeLDSMetadata: -1 +ULSGRODoubleG2L: 0 +UnrollLoopSwapGlobalReadOrder: 0 +UnrollMajorLDSA: true +UnrollMajorLDSB: true +Use64bShadowLimit: true +Use64bShadowLimitMX: false +UseCustomMainLoopSchedule: 0 +UseDirect32XEmulation: false +UseDot2F32XEmulation: false +UseDotInstruction: false +UseDualFMAC: false +UseF32XEmulation: false +UseGeneralizedNLCOneA: false +UseGeneralizedNLCOneB: false +UseGeneralizedNLCOneMXSA: false +UseGeneralizedNLCOneMXSB: false +UseGeneralizedNLCOneMetadata: false +UseInstOffsetForGRO: 0 +UseMFMAF32XEmulation: false +UsePLRPack: 0 +UseSgprForGRO: -1 +UseSubtileImpl: false +Valid: true +VectorStore: -1 +VectorWidthA: 1 +VectorWidthB: 1 +WaveSeparateGlobalReadA: 0 +WaveSeparateGlobalReadB: 0 +WaveSeparateGlobalReadMetadata: 0 +WaveSplitK: false +WavefrontSize: 32 +WorkAssignment: StaticGrid +WorkGroup: +- 32 +- 4 +- 1 +WorkGroupMapping: 8 +WorkGroupMappingXCC: 1 +WorkGroupMappingXCCGroup: -1 +WorkGroupReduction: false +WorkQueueStealing: 0 +WorkspaceCheck: +- 4 +- 0 +- 1 +_DepthU: 64 +_DepthUA: 64 +_DepthUB: 64 +_DepthUMetadata: 64 +_GlobalAccumulation: MultipleBuffer +_PersistentLoop: false +_ScheduleIterAlg: 0 +_StinkyTofuOptLevel: 0 +_UseSgprForGRO: 0 +_VectorStore: 1 +_WorkspaceSizePerElemBias: 0 +_WorkspaceSizePerElemC: 0 +_inputVWA_was_auto: false +_inputVWB_was_auto: false +_staggerStrideShift: 0 +enableGLTrA: false +enableGLTrB: false +enableLDSTrA: false +enableLDSTrB: false +enableLDSTrMXSA: false +enableLDSTrMXSB: false +enableLDSTrMetadata: 0 +enableTDMA: false +enableTDMB: false +enableTDMMetadata: false +ldsNumBytesA: 9216 +ldsNumBytesB: 23040 +ldsNumBytesMXSA: 0 +ldsNumBytesMXSB: 0 +ldsNumBytesMetadata: 0 +numSubTiles: 1 +reorderGRInstForDTVA: false +reorderGRInstForDTVB: false +tailLoopOptA: false +tailLoopOptB: false diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.msgpack new file mode 100644 index 000000000000..351876088167 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.predicates.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.predicates.msgpack new file mode 100644 index 000000000000..8f96ab7a2b56 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.predicates.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.predicates.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.predicates.yaml new file mode 100644 index 000000000000..c46ed38289ce --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.predicates.yaml @@ -0,0 +1,99 @@ +problem: + type: And + value: + - type: OperationIdentifierEqual + value: Contraction_l_Alik_Bljk_Cijk_Dijk + - type: BiasDataTypeWhiteList + value: [] + - type: GateResidualDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: + - 3 + - type: AmaxDCheck + value: false + - type: TypesEqual + value: + - BFloat16 + - BFloat16 + - Float + - Float + - BFloat16 + - BFloat16 + - type: HighPrecisionAccumulate + value: true + - type: Activation + value: None + - type: ActivationComputeType + value: Float + - type: ActivationNoGuard + value: false + - type: UseGradient + value: false + - type: UseBias + value: 0 + - type: UseE + value: false + - type: UseGateResidual + value: false + - type: DataTypeE + value: Float + - type: StridedBatched + value: true + - type: GroupedGemm + value: false + - type: UseScaleAB + value: '' + - type: UseScaleCD + value: false + - type: UseScaleAlphaVec + value: 0 + - type: Sparse + value: 0 + - type: F32XdlMathOp + value: Float + - type: SupportDeviceUserArguments + value: true + - type: SwizzleTensorA + value: false + - type: SwizzleTensorB + value: false + - type: FusedGemmA2A + value: false + - type: MXBlockA + value: 0 + - type: MXBlockB + value: 0 + - type: LeadingFree0SizesGreaterOrEqual + value: 1 + - type: LeadingFree1SizesGreaterOrEqual + value: 1 + - type: KernelLanguageCompatible + value: Assembly + - type: BufferLoadOffsetLimitCheck + value: + ShiftPtrElemB: 0 + ShiftPtrElemA: 0 + DUorMT1: 32 + DUorMT0: 32 + - type: BufferLoadOffsetLimitCheck_Beta + value: 32 + - type: BufferStoreOffsetLimitCheck + value: 32 + - type: WorkgroupMappingXCCCheck + value: + - 1 + - -1 + - type: SizeGreaterThan + index: 3 + value: 0 + - type: SizeLessThan + index: 3 + value: 193 +task: + type: WorkspaceCheck +hardware: + type: AMDGPU + value: + type: Processor + value: gfx1250 diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.record.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.record.txt new file mode 100644 index 000000000000..5b29e402b965 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.record.txt @@ -0,0 +1,130 @@ +solutionName="" +kernelName="" +sizeMapping.waveNum=4 +sizeMapping.WavefrontSize=32 +sizeMapping.workGroup=[32,4,1] +sizeMapping.macroTile=[32,32,1] +sizeMapping.threadTile=[1,1] +sizeMapping.matrixInstruction=[16,16,32,1] +sizeMapping.grvwA=1 +sizeMapping.grvwB=1 +sizeMapping.gwvwC=8 +sizeMapping.gwvwD=8 +sizeMapping.depthU=64 +sizeMapping.staggerU=0 +sizeMapping.staggerUMapping=0 +sizeMapping.globalSplitUPGR=16 +sizeMapping.globalSplitU=0 +sizeMapping.staggerStrideShift=0 +sizeMapping.workGroupMapping=1 +sizeMapping.packBatchDims=0 +sizeMapping.magicDivAlg=2 +sizeMapping.tileProcessingStrategy="StreamK" +sizeMapping.workAssignment="StaticGrid" +sizeMapping.streamKAtomic=0 +sizeMapping.prefetchAcrossPersistent=0 +sizeMapping.sourceKernel=false +sizeMapping.globalAccumulation=4 +sizeMapping.adaptiveGemmGSUA=0 +sizeMapping.workspaceSizePerElemC=4 +sizeMapping.workspaceSizePerElemBias=0 +sizeMapping.activationFused=true +sizeMapping.workGroupMappingXCC=1 +sizeMapping.workGroupMappingXCCGroup=-1 +sizeMapping.globalSplitUCoalesced=false +sizeMapping.globalSplitUWorkGroupMappingRoundRobin=false +sizeMapping.CUOccupancy=-1 +sizeMapping.PrefetchGlobalRead=1 +sizeMapping.MathClocksUnrolledLoop=0 +sizeMapping.synchronizerSizePerWG=32 +sizeMapping.nonTemporalA=0 +sizeMapping.nonTemporalB=0 +sizeMapping.temporalHintA=0 +sizeMapping.temporalHintB=0 +sizeMapping.hasTemporalHint=true +sizeMapping.adaptiveGemmNTAB=0 +sizeMapping.customMainLoopScheduling=0 +sizeMapping.useSubtileImpl=false +sizeMapping.NonTemporalD=0 +sizeMapping.WaveSeparateGlobalReadA=0 +sizeMapping.WaveSeparateGlobalReadB=0 +sizeMapping.UnrollLoopSwapGlobalReadOrder=0 +sizeMapping.DirectToVgprA=false +sizeMapping.DirectToVgprB=false +sizeMapping.NumLoadsCoalescedA=1 +sizeMapping.NumLoadsCoalescedB=1 +sizeMapping.WaveGroup=[2,2] +sizeMapping.VectorWidthA=1 +sizeMapping.VectorWidthB=1 +sizeMapping.LocalSplitU=1 +sizeMapping.DirectToLdsA=false +sizeMapping.DirectToLdsB=false +sizeMapping.OneLDSBuffer=false +sizeMapping.SourceSwap=false +sizeMapping.UnrollMajorLDSA=1 +sizeMapping.UnrollMajorLDSB=1 +sizeMapping.enableLDSTrA=false +sizeMapping.enableLDSTrB=false +sizeMapping.ExpertSchedulingMode=0 +sizeMapping.clusterDim=[2,1] +sizeMapping.BufferLoad=true +sizeMapping.BufferStore=true +internalArgsSupport.version=3 +internalArgsSupport.persistentLoopArgsVersion=2 +internalArgsSupport.gsu=false +internalArgsSupport.wgm=true +internalArgsSupport.staggerU=false +internalArgsSupport.perTileExtraIters=true +internalArgsSupport.useUniversalArgs=true +internalArgsSupport.useSFC=false +internalArgsSupport.occupancyProbe=true +problemType.operationIdentifier="Contraction_l_Alik_Bljk_Cijk_Dijk" +problemType.transA=true +problemType.transB=false +problemType.computeInputTypeA="BFloat16" +problemType.computeInputTypeB="BFloat16" +problemType.aType="BFloat16" +problemType.bType="BFloat16" +problemType.cType="Float" +problemType.dType="Float" +problemType.eType="Float" +problemType.computeType="Float" +problemType.useBeta=true +problemType.useBias=0 +problemType.biasSrcWhiteList=[3] +problemType.useE=false +problemType.useGateResidual=false +problemType.gateResidualDataTypeWhiteList=[] +problemType.useScaleAB="" +problemType.useScaleCD=false +problemType.useScaleAlphaVec=0 +problemType.biasDataTypeWhiteList=[] +problemType.highPrecisionAccumulate=true +problemType.useInitialStridesAB=false +problemType.useInitialStridesCD=false +problemType.stridedBatched=true +problemType.groupedGemm=false +problemType.useGradient=false +problemType.activationType="None" +problemType.activationArgLength=0 +problemType.activationComputeDataType="Float" +problemType.activationNoGuard=false +problemType.sparse=0 +problemType.f32XdlMathOp="Float" +problemType.supportDeviceUserArguments=true +problemType.outputAmaxD=false +problemType.swizzleTensorA=false +problemType.swizzleTensorB=false +problemType.metadataLayout=0 +problemType.mxBlockA=0 +problemType.mxBlockB=0 +problemType.mxBlockFreeA=1 +problemType.mxBlockFreeB=1 +problemType.mxTypeA="E8" +problemType.mxTypeB="E8" +problemType.mxScaleFormat=2 +problemType.fusedGemmA2A=false +problemType.stochasticRounding=false +problemPredicate=["And","OperationIdentifierEqual","BiasDataTypeWhiteList","GateResidualDataTypeWhiteList","BiasSrcWhiteList","AmaxDCheck","TypesEqual","HighPrecisionAccumulate","Activation","ActivationComputeType","ActivationNoGuard","UseGradient","UseBias","UseE","UseGateResidual","DataTypeE","StridedBatched","GroupedGemm","UseScaleAB","UseScaleCD","UseScaleAlphaVec","Sparse","F32XdlMathOp","SupportDeviceUserArguments","SwizzleTensorA","SwizzleTensorB","FusedGemmA2A","MXBlockA","MXBlockB","LeadingFree0SizesGreaterOrEqual","LeadingFree1SizesGreaterOrEqual","KernelLanguageCompatible","BufferLoadOffsetLimitCheck","BufferLoadOffsetLimitCheck_Beta","BufferStoreOffsetLimitCheck","WorkgroupMappingXCCCheck","SizeGreaterThan","SizeLessThan"] +taskPredicate=["WorkspaceCheck"] +hardwarePredicate=["AMDGPU","Processor"] diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.yaml new file mode 100644 index 000000000000..7dc6f511b5f6 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster-rap-probe.yaml @@ -0,0 +1,507 @@ +1LDSBuffer: 0 +ActivationAlt: false +ActivationFuncCall: false +ActivationFused: true +AdaptiveGemm: 0 +AdaptiveGemmGSUA: 0 +AdaptiveGemmNTAB: 0 +AssertAIGreaterThanEqual: -1 +AssertAILessThanEqual: -1 +AssertFree0ElementMultiple: 1 +AssertFree1ElementMultiple: 1 +AssertSummationElementMultiple: 1 +AssignedDerivedParameters: true +AssignedProblemIndependentDerivedParameters: true +BufferLoad: true +BufferStore: true +CUOccupancy: -1 +ClusterAIDMapping: -1 +ClusterBarrier: true +ClusterDim: +- 2 +- 1 +ClusterLocalRead: 1 +CodeObjectVersion: default +CompactLoopStore: false +ConvertAfterDS: false +DebugPersistentKernelLoopForever: false +DebugStreamK: 0 +DepthU: 64 +DirectToLds: 0 +DirectToLdsA: false +DirectToLdsB: false +DirectToLdsMetadata: 0 +DirectToVgprA: false +DirectToVgprB: false +DirectToVgprMXSA: false +DirectToVgprMXSB: false +DirectToVgprSparseMetadata: false +DisableXdlArbStall: false +DtlPlusLdsBuf: 0 +ESMRuntimeGate: false +EdgeType: None +EnableESM2TrackValuVsrc: true +EnableF32XdlMathOp: false +EnableMatrixInstruction: true +EnableStinkyTofuESM2: true +ExpandPointerSwap: false +ExpertSchedulingMode: 0 +ExtraLatencyForLR: 0 +ExtraMiLatencyLeft: -1 +ForceDisableShadowInit: false +ForceUnrollSubIter: false +GlobalReadPerMfma: 1.0 +GlobalReadVectorWidthA: 1 +GlobalReadVectorWidthB: 1 +GlobalSplitU: 0 +GlobalSplitUAlgorithm: MultipleBuffer +GlobalSplitUCoalesced: false +GlobalSplitUWorkGroupMappingRoundRobin: false +GlobalWriteVectorWidth: 1 +GroupLoadStore: false +GuaranteeNoPartialA: true +GuaranteeNoPartialB: true +GuaranteeNoPartialMetadata: true +HalfPLR: 0 +HalfPLRA: false +HalfPLRB: false +ISA: +- 12 +- 5 +- 0 +InitCIterWmma: 1 +InnerUnroll: 1 +InterleaveAlpha: 0 +InternalSupportParams: + KernArgsVersion: 3 + PersistentLoopArgsVersion: 2 + SupportCustomStaggerU: false + SupportCustomWGM: true + SupportOccupancyProbe: true + SupportStreamKPerTileExtraIters: true + SupportUserGSU: false + UseSFC: false + UseUniversalArgs: true +KernelLanguage: Assembly +LDSSegInterleaveOffsets: {} +LDSSegmentBRotate: 0 +LDSSegmentInterleave: 0 +LDSTrInst: false +LSCA: 64 +LSCB: 64 +LSPA: 32 +LSPB: 32 +LVCA: 64 +LVCB: 64 +LVPA: 32 +LVPB: 32 +LdsBlockSizePerPadA: 256 +LdsBlockSizePerPadB: 256 +LdsBlockSizePerPadMXSA: 0 +LdsBlockSizePerPadMXSB: 0 +LdsBlockSizePerPadMetadata: 0 +LdsBytesNoAmax: 25088 +LdsInitCVgprs: false +LdsNumBytes: 25088 +LdsNumElementsAlignedA: 4352 +LdsNumElementsAlignedB: 4352 +LdsNumElementsAlignedMXSA: 0 +LdsNumElementsAlignedMXSB: 0 +LdsNumElementsAlignedMetadata: 0 +LdsOffsetA: 0 +LdsOffsetA_Blk: 16384 +LdsOffsetB: 4352 +LdsOffsetB_Blk: 20736 +LdsOffsetBias: 0 +LdsOffsetBiasGSU: 0 +LdsOffsetBiasNonGSU: 0 +LdsOffsetMXSA: 4352 +LdsOffsetMXSA_Blk: 20736 +LdsOffsetMXSB: 4352 +LdsOffsetMXSB_Blk: 20736 +LdsOffsetMetadata: 4352 +LdsOffsetMetadata_Blk: 20736 +LdsPadA: 8 +LdsPadB: 8 +LdsPadMXSA: 0 +LdsPadMXSB: 0 +LdsPadMetadata: -1 +LocalReadVectorWidth: -1 +LocalReadVectorWidthA: 8 +LocalReadVectorWidthB: 8 +LocalSplitU: 1 +LocalSplitUReuseLDS: 1 +LocalWritePerMfma: -1 +LocalWriteUseSgprA: false +LocalWriteUseSgprB: false +LocalWriteUseSgprMetadata: false +LoopIters: 2 +LoopUnroll: 64 +MFMA_BF16_1K: 0 +MIArchVgpr: true +MIBlock: +- 16 +- 16 +- 32 +- 1 +- 1 +- 1 +MIInputPerThread: 16 +MIInputPerThreadA: 16 +MIInputPerThreadB: 16 +MIInputPerThreadMetadata: 16 +MIOutputVectorWidth: 8 +MIRegPerOut: 1 +MIWaveGroup: +- 2 +- 2 +MIWaveTile: +- 1 +- 1 +MIWaveTileA: 1 +MIWaveTileB: 1 +MIWaveTileMetadata: 0 +MXLoadInst: TDM +MXScaleFormat: InMemorySwizzle +MacroTile0: 32 +MacroTile1: 32 +MacroTileA: 32 +MacroTileB: 32 +MagicDivAlg: 2 +MathClocksUnrolledLoop: 0 +MatrixInstB: 1 +MatrixInstBM: 1 +MatrixInstBN: 1 +MatrixInstK: 32 +MatrixInstM: 16 +MatrixInstN: 16 +MatrixInstruction: +- 16 +- 16 +- 32 +- 1 +MaxLDS: 327680 +MaxOccupancy: 64 +MbskPrefetchMethod: 0 +MfmaInitCVgprs: false +MinGRIncPerMfma: 1 +Multicast: true +NoLdsWriteCode: true +NoReject: false +NoTailLoop: false +NonDTLTailLoopA: false +NonDTLTailLoopB: false +NonDTLTailLoopMetadata: false +NonTemporal: -1 +NonTemporalA: 0 +NonTemporalB: 0 +NonTemporalC: 0 +NonTemporalD: 0 +NonTemporalE: 0 +NonTemporalGate: 0 +NonTemporalMXSA: 0 +NonTemporalMXSB: 0 +NonTemporalMetadata: 0 +NonTemporalWS: 0 +NonVolatile: -1 +NonVolatileA: 0 +NonVolatileB: 0 +NonVolatileC: 0 +NonVolatileD: 0 +NonVolatileE: 0 +NonVolatileGate: 0 +NonVolatileMXSA: 0 +NonVolatileMXSB: 0 +NonVolatileMetadata: 0 +NonVolatileWS: 0 +NumElementsPerBatchStore: 0 +NumElementsPerThread: 8 +NumGlobalWriteVectorsPerThread: 8 +NumLdsBlk: 2 +NumLoadsA: 1 +NumLoadsB: 1 +NumLoadsCoalescedA: 1 +NumLoadsCoalescedB: 1 +NumLoadsPerpendicularA: 1 +NumLoadsPerpendicularB: 1 +NumThreads: 128 +NumTotalPackedLoadsA: -1 +NumTotalPackedLoadsB: -1 +NumWaveSplitK: 1 +NumWaves: 4 +OptNoLoadLoop: 1 +PackedC0IdxChars: +- I +PackedC0IndicesX: +- 0 +PackedC1IdxChars: +- J +PackedC1IndicesX: +- 1 +PersistentXCCMapping: 0 +PostLoopStoreInNll: false +PrefetchAcrossPersistent: 0 +PrefetchGL2: 0 +PrefetchGlobalRead: 1 +PrefetchLocalRead: 1 +PreloadKernArgs: true +ProblemType: + Activation: false + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: none + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: false + BiasDataTypeList: [] + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 7 + DataTypeA: 7 + DataTypeAmaxD: 0 + DataTypeB: 7 + DataTypeE: 0 + DataTypeMXSA: 22 + DataTypeMXSB: 22 + DestDataType: 0 + F32XdlMathOp: 0 + FusedGemmA2A: false + GateResidualDataTypeList: [] + Gradient: false + GroupedGemm: false + HighPrecisionAccumulate: true + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: + - 3 + - 0 + - 2 + IndexAssignmentsB: + - 3 + - 1 + - 2 + IndexAssignmentsLD: + - 4 + - 5 + - 6 + - 7 + IndexAssignmentsMetadata: + - 3 + - 0 + - 2 + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndexUnrollM: 0 + IndicesBatch: + - 2 + IndicesFree: + - 0 + - 1 + IndicesSummation: + - 3 + Int4EncodingA: Signed + MXBlockA: 0 + MXBlockB: 0 + MXBlockFreeA: 1 + MXBlockFreeB: 1 + MacDataTypeA: 7 + MacDataTypeB: 7 + MetadataLayout: 0 + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMXSA: [] + MirrorDimsMXSB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + ScaleBlockSizesA: [] + ScaleZeroPointA: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SetConstStrideGate: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: false + StridedBatched: true + SupportUserArgs: true + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseBias: 0 + UseE: false + UseGateResidual: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: '' + UseScaleAlphaVec: 0 + UseScaleCD: false + isMixMode: false +ReuseAcrossPersistent: 1 +SFCWGM: +- - 1 + - 1 +- - 1 + - 1 +ScheduleGROverBarrier: 0 +ScheduleGlobalRead: 1 +ScheduleIterAlg: 4 +ScheduleLocalWrite: 1 +SourceSwap: false +SpaceFillingAlgo: [] +StaggerU: 0 +StaggerUMapping: 0 +StaggerUStride: 0 +StinkyTofuRegisterAllocation: 0 +StorePriorityOpt: false +StoreRemapVectorWidth: 0 +StoreSwapAddr: false +StoreSyncOpt: 0 +StoreVectorWidth: 8 +StreamKAtomic: 0 +StreamKFixupTreeReduction: 0 +SubGroup0: 4 +SubGroup1: 32 +SubGroupA: 4 +SubGroupB: 32 +SuppressNoLoadLoop: false +SwInstructionPrefetch: 1 +SwapGlobalReadOrder: 0 +TDMInst: 3 +TDMIterateMode: 0 +TDMLoadWaveSync: false +TDMPlusLdsBuf: 0 +TDMSplit: false +TailloopInNll: false +TemporalHint: -1 +TemporalHintA: 0 +TemporalHintB: 0 +TemporalHintC: 0 +TemporalHintD: 0 +TemporalHintE: 0 +TemporalHintGate: 0 +TemporalHintMXSA: 0 +TemporalHintMXSB: 0 +TemporalHintMetadata: 0 +TemporalHintWS: 0 +ThreadTile: +- 1 +- 1 +ThreadTile0: 8 +ThreadTile1: 1 +ThreadTileA: 8 +ThreadTileB: 1 +TileProcessingStrategy: StreamK +TransposeLDS: 1 +TransposeLDSMetadata: -1 +ULSGRODoubleG2L: 0 +UnrollLoopSwapGlobalReadOrder: 0 +UnrollMajorLDSA: true +UnrollMajorLDSB: true +Use64bShadowLimit: true +Use64bShadowLimitMX: false +UseCustomMainLoopSchedule: 0 +UseDirect32XEmulation: false +UseDot2F32XEmulation: false +UseDotInstruction: false +UseDualFMAC: false +UseF32XEmulation: false +UseGeneralizedNLCOneA: false +UseGeneralizedNLCOneB: false +UseGeneralizedNLCOneMXSA: false +UseGeneralizedNLCOneMXSB: false +UseGeneralizedNLCOneMetadata: false +UseInstOffsetForGRO: 0 +UseMFMAF32XEmulation: false +UsePLRPack: 0 +UseSgprForGRO: -1 +UseSubtileImpl: false +Valid: true +VectorStore: -1 +VectorWidthA: 1 +VectorWidthB: 1 +WaveSeparateGlobalReadA: 0 +WaveSeparateGlobalReadB: 0 +WaveSeparateGlobalReadMetadata: 0 +WaveSplitK: false +WavefrontSize: 32 +WorkAssignment: StaticGrid +WorkGroup: +- 32 +- 4 +- 1 +WorkGroupMapping: 1 +WorkGroupMappingXCC: 1 +WorkGroupMappingXCCGroup: -1 +WorkGroupReduction: false +WorkQueueStealing: 0 +WorkspaceCheck: +- 4 +- 0 +- 0 +_DepthU: 64 +_DepthUA: 64 +_DepthUB: 64 +_DepthUMetadata: 64 +_DualPeriodMXFP8: false +_GlobalAccumulation: PartialsBuffer +_HasTemporalHint: 1 +_PersistentLoop: true +_PersistentVectorEpilogueLds: false +_PrefetchAcrossPersistentEnabled: false +_RAPNumResidentKTiles: 3 +_ScalePeriodUA: 64 +_ScalePeriodUB: 64 +_ScaleReuseA: 1 +_ScaleReuseB: 1 +_ScheduleIterAlg: 0 +_SeparateEpilogueLds: false +_StinkyTofuOptLevel: 3 +_UseSgprForGRO: 1 +_VectorStore: 1 +_WorkspaceSizePerElemBias: 0 +_WorkspaceSizePerElemC: 4 +_inputVWA_was_auto: true +_inputVWB_was_auto: true +_staggerStrideShift: 0 +enableGLTrA: false +enableGLTrB: false +enableLDSTrA: false +enableLDSTrB: false +enableLDSTrMXSA: false +enableLDSTrMXSB: false +enableLDSTrMetadata: 0 +enableTDMA: true +enableTDMB: true +enableTDMMetadata: false +ldsNumBytesA: 4352 +ldsNumBytesB: 4352 +ldsNumBytesMXSA: 0 +ldsNumBytesMXSB: 0 +ldsNumBytesMetadata: 0 +numSubTiles: 1 +reorderGRInstForDTVA: false +reorderGRInstForDTVB: false +tailLoopOptA: false +tailLoopOptB: false diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.msgpack new file mode 100644 index 000000000000..2a4f1d8f16b4 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.predicates.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.predicates.msgpack new file mode 100644 index 000000000000..729bdb147a33 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.predicates.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.predicates.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.predicates.yaml new file mode 100644 index 000000000000..e4457cebfa2b --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.predicates.yaml @@ -0,0 +1,93 @@ +problem: + type: And + value: + - type: OperationIdentifierEqual + value: Contraction_l_Alik_Bljk_Cijk_Dijk + - type: BiasDataTypeWhiteList + value: [] + - type: GateResidualDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: + - 3 + - type: AmaxDCheck + value: false + - type: TypesEqual + value: + - BFloat16 + - BFloat16 + - Float + - Float + - BFloat16 + - BFloat16 + - type: HighPrecisionAccumulate + value: true + - type: Activation + value: None + - type: ActivationComputeType + value: Float + - type: ActivationNoGuard + value: false + - type: UseGradient + value: false + - type: UseBias + value: 0 + - type: UseE + value: false + - type: UseGateResidual + value: false + - type: DataTypeE + value: Float + - type: StridedBatched + value: true + - type: GroupedGemm + value: false + - type: UseScaleAB + value: '' + - type: UseScaleCD + value: false + - type: UseScaleAlphaVec + value: 0 + - type: Sparse + value: 0 + - type: F32XdlMathOp + value: Float + - type: SupportDeviceUserArguments + value: true + - type: SwizzleTensorA + value: false + - type: SwizzleTensorB + value: false + - type: FusedGemmA2A + value: false + - type: MXBlockA + value: 0 + - type: MXBlockB + value: 0 + - type: LeadingFree0SizesGreaterOrEqual + value: 1 + - type: LeadingFree1SizesGreaterOrEqual + value: 1 + - type: KernelLanguageCompatible + value: Assembly + - type: BufferLoadOffsetLimitCheck + value: + ShiftPtrElemB: 0 + ShiftPtrElemA: 0 + DUorMT1: 32 + DUorMT0: 32 + - type: BufferLoadOffsetLimitCheck_Beta + value: 32 + - type: BufferStoreOffsetLimitCheck + value: 32 + - type: WorkgroupMappingXCCCheck + value: + - 1 + - -1 +task: + type: WorkspaceCheck +hardware: + type: AMDGPU + value: + type: Processor + value: gfx1250 diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.record.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.record.txt new file mode 100644 index 000000000000..a2c159986c89 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.record.txt @@ -0,0 +1,129 @@ +solutionName="" +kernelName="" +sizeMapping.waveNum=4 +sizeMapping.WavefrontSize=32 +sizeMapping.workGroup=[32,4,1] +sizeMapping.macroTile=[32,32,1] +sizeMapping.threadTile=[1,1] +sizeMapping.matrixInstruction=[16,16,32,1] +sizeMapping.grvwA=1 +sizeMapping.grvwB=1 +sizeMapping.gwvwC=8 +sizeMapping.gwvwD=8 +sizeMapping.depthU=64 +sizeMapping.staggerU=0 +sizeMapping.staggerUMapping=0 +sizeMapping.globalSplitUPGR=16 +sizeMapping.globalSplitU=0 +sizeMapping.staggerStrideShift=0 +sizeMapping.workGroupMapping=1 +sizeMapping.packBatchDims=0 +sizeMapping.magicDivAlg=2 +sizeMapping.tileProcessingStrategy="StreamK" +sizeMapping.workAssignment="StaticGrid" +sizeMapping.streamKAtomic=0 +sizeMapping.prefetchAcrossPersistent=0 +sizeMapping.sourceKernel=false +sizeMapping.globalAccumulation=4 +sizeMapping.adaptiveGemmGSUA=0 +sizeMapping.workspaceSizePerElemC=4 +sizeMapping.workspaceSizePerElemBias=0 +sizeMapping.activationFused=true +sizeMapping.workGroupMappingXCC=1 +sizeMapping.workGroupMappingXCCGroup=-1 +sizeMapping.globalSplitUCoalesced=false +sizeMapping.globalSplitUWorkGroupMappingRoundRobin=false +sizeMapping.CUOccupancy=-1 +sizeMapping.PrefetchGlobalRead=1 +sizeMapping.MathClocksUnrolledLoop=0 +sizeMapping.synchronizerSizePerWG=32 +sizeMapping.nonTemporalA=0 +sizeMapping.nonTemporalB=0 +sizeMapping.temporalHintA=0 +sizeMapping.temporalHintB=0 +sizeMapping.hasTemporalHint=true +sizeMapping.adaptiveGemmNTAB=0 +sizeMapping.customMainLoopScheduling=0 +sizeMapping.useSubtileImpl=false +sizeMapping.NonTemporalD=0 +sizeMapping.WaveSeparateGlobalReadA=0 +sizeMapping.WaveSeparateGlobalReadB=0 +sizeMapping.UnrollLoopSwapGlobalReadOrder=0 +sizeMapping.DirectToVgprA=false +sizeMapping.DirectToVgprB=false +sizeMapping.NumLoadsCoalescedA=1 +sizeMapping.NumLoadsCoalescedB=1 +sizeMapping.WaveGroup=[2,2] +sizeMapping.VectorWidthA=1 +sizeMapping.VectorWidthB=1 +sizeMapping.LocalSplitU=1 +sizeMapping.DirectToLdsA=false +sizeMapping.DirectToLdsB=false +sizeMapping.OneLDSBuffer=false +sizeMapping.SourceSwap=false +sizeMapping.UnrollMajorLDSA=1 +sizeMapping.UnrollMajorLDSB=1 +sizeMapping.enableLDSTrA=false +sizeMapping.enableLDSTrB=false +sizeMapping.ExpertSchedulingMode=0 +sizeMapping.clusterDim=[2,1] +sizeMapping.BufferLoad=true +sizeMapping.BufferStore=true +internalArgsSupport.version=3 +internalArgsSupport.persistentLoopArgsVersion=2 +internalArgsSupport.gsu=false +internalArgsSupport.wgm=true +internalArgsSupport.staggerU=false +internalArgsSupport.perTileExtraIters=true +internalArgsSupport.useUniversalArgs=true +internalArgsSupport.useSFC=false +problemType.operationIdentifier="Contraction_l_Alik_Bljk_Cijk_Dijk" +problemType.transA=true +problemType.transB=false +problemType.computeInputTypeA="BFloat16" +problemType.computeInputTypeB="BFloat16" +problemType.aType="BFloat16" +problemType.bType="BFloat16" +problemType.cType="Float" +problemType.dType="Float" +problemType.eType="Float" +problemType.computeType="Float" +problemType.useBeta=true +problemType.useBias=0 +problemType.biasSrcWhiteList=[3] +problemType.useE=false +problemType.useGateResidual=false +problemType.gateResidualDataTypeWhiteList=[] +problemType.useScaleAB="" +problemType.useScaleCD=false +problemType.useScaleAlphaVec=0 +problemType.biasDataTypeWhiteList=[] +problemType.highPrecisionAccumulate=true +problemType.useInitialStridesAB=false +problemType.useInitialStridesCD=false +problemType.stridedBatched=true +problemType.groupedGemm=false +problemType.useGradient=false +problemType.activationType="None" +problemType.activationArgLength=0 +problemType.activationComputeDataType="Float" +problemType.activationNoGuard=false +problemType.sparse=0 +problemType.f32XdlMathOp="Float" +problemType.supportDeviceUserArguments=true +problemType.outputAmaxD=false +problemType.swizzleTensorA=false +problemType.swizzleTensorB=false +problemType.metadataLayout=0 +problemType.mxBlockA=0 +problemType.mxBlockB=0 +problemType.mxBlockFreeA=1 +problemType.mxBlockFreeB=1 +problemType.mxTypeA="E8" +problemType.mxTypeB="E8" +problemType.mxScaleFormat=2 +problemType.fusedGemmA2A=false +problemType.stochasticRounding=false +problemPredicate=["And","OperationIdentifierEqual","BiasDataTypeWhiteList","GateResidualDataTypeWhiteList","BiasSrcWhiteList","AmaxDCheck","TypesEqual","HighPrecisionAccumulate","Activation","ActivationComputeType","ActivationNoGuard","UseGradient","UseBias","UseE","UseGateResidual","DataTypeE","StridedBatched","GroupedGemm","UseScaleAB","UseScaleCD","UseScaleAlphaVec","Sparse","F32XdlMathOp","SupportDeviceUserArguments","SwizzleTensorA","SwizzleTensorB","FusedGemmA2A","MXBlockA","MXBlockB","LeadingFree0SizesGreaterOrEqual","LeadingFree1SizesGreaterOrEqual","KernelLanguageCompatible","BufferLoadOffsetLimitCheck","BufferLoadOffsetLimitCheck_Beta","BufferStoreOffsetLimitCheck","WorkgroupMappingXCCCheck"] +taskPredicate=["WorkspaceCheck"] +hardwarePredicate=["AMDGPU","Processor"] diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.yaml new file mode 100644 index 000000000000..bb9468c3b9ea --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx1250-streamk-cluster.yaml @@ -0,0 +1,506 @@ +1LDSBuffer: 0 +ActivationAlt: false +ActivationFuncCall: false +ActivationFused: true +AdaptiveGemm: 0 +AdaptiveGemmGSUA: 0 +AdaptiveGemmNTAB: 0 +AssertAIGreaterThanEqual: -1 +AssertAILessThanEqual: -1 +AssertFree0ElementMultiple: 1 +AssertFree1ElementMultiple: 1 +AssertSummationElementMultiple: 1 +AssignedDerivedParameters: true +AssignedProblemIndependentDerivedParameters: true +BufferLoad: true +BufferStore: true +CUOccupancy: -1 +ClusterAIDMapping: -1 +ClusterBarrier: true +ClusterDim: +- 2 +- 1 +ClusterLocalRead: 1 +CodeObjectVersion: default +CompactLoopStore: false +ConvertAfterDS: false +DebugPersistentKernelLoopForever: false +DebugStreamK: 0 +DepthU: 64 +DirectToLds: 0 +DirectToLdsA: false +DirectToLdsB: false +DirectToLdsMetadata: 0 +DirectToVgprA: false +DirectToVgprB: false +DirectToVgprMXSA: false +DirectToVgprMXSB: false +DirectToVgprSparseMetadata: false +DisableXdlArbStall: false +DtlPlusLdsBuf: 0 +ESMRuntimeGate: false +EdgeType: None +EnableESM2TrackValuVsrc: true +EnableF32XdlMathOp: false +EnableMatrixInstruction: true +EnableStinkyTofuESM2: true +ExpandPointerSwap: false +ExpertSchedulingMode: 0 +ExtraLatencyForLR: 0 +ExtraMiLatencyLeft: -1 +ForceDisableShadowInit: false +ForceUnrollSubIter: false +GlobalReadPerMfma: 1.0 +GlobalReadVectorWidthA: 1 +GlobalReadVectorWidthB: 1 +GlobalSplitU: 0 +GlobalSplitUAlgorithm: MultipleBuffer +GlobalSplitUCoalesced: false +GlobalSplitUWorkGroupMappingRoundRobin: false +GlobalWriteVectorWidth: 1 +GroupLoadStore: false +GuaranteeNoPartialA: true +GuaranteeNoPartialB: true +GuaranteeNoPartialMetadata: true +HalfPLR: 0 +HalfPLRA: false +HalfPLRB: false +ISA: +- 12 +- 5 +- 0 +InitCIterWmma: 1 +InnerUnroll: 1 +InterleaveAlpha: 0 +InternalSupportParams: + KernArgsVersion: 3 + PersistentLoopArgsVersion: 2 + SupportCustomStaggerU: false + SupportCustomWGM: true + SupportOccupancyProbe: false + SupportStreamKPerTileExtraIters: true + SupportUserGSU: false + UseSFC: false + UseUniversalArgs: true +KernelLanguage: Assembly +LDSSegInterleaveOffsets: {} +LDSSegmentBRotate: 0 +LDSSegmentInterleave: 0 +LDSTrInst: false +LSCA: 64 +LSCB: 64 +LSPA: 32 +LSPB: 32 +LVCA: 64 +LVCB: 64 +LVPA: 32 +LVPB: 32 +LdsBlockSizePerPadA: 256 +LdsBlockSizePerPadB: 256 +LdsBlockSizePerPadMXSA: 0 +LdsBlockSizePerPadMXSB: 0 +LdsBlockSizePerPadMetadata: 0 +LdsBytesNoAmax: 25088 +LdsInitCVgprs: false +LdsNumBytes: 25088 +LdsNumElementsAlignedA: 4352 +LdsNumElementsAlignedB: 4352 +LdsNumElementsAlignedMXSA: 0 +LdsNumElementsAlignedMXSB: 0 +LdsNumElementsAlignedMetadata: 0 +LdsOffsetA: 0 +LdsOffsetA_Blk: 16384 +LdsOffsetB: 4352 +LdsOffsetB_Blk: 20736 +LdsOffsetBias: 0 +LdsOffsetBiasGSU: 0 +LdsOffsetBiasNonGSU: 0 +LdsOffsetMXSA: 4352 +LdsOffsetMXSA_Blk: 20736 +LdsOffsetMXSB: 4352 +LdsOffsetMXSB_Blk: 20736 +LdsOffsetMetadata: 4352 +LdsOffsetMetadata_Blk: 20736 +LdsPadA: 8 +LdsPadB: 8 +LdsPadMXSA: 0 +LdsPadMXSB: 0 +LdsPadMetadata: -1 +LocalReadVectorWidth: -1 +LocalReadVectorWidthA: 8 +LocalReadVectorWidthB: 8 +LocalSplitU: 1 +LocalSplitUReuseLDS: 1 +LocalWritePerMfma: -1 +LocalWriteUseSgprA: false +LocalWriteUseSgprB: false +LocalWriteUseSgprMetadata: false +LoopIters: 2 +LoopUnroll: 64 +MFMA_BF16_1K: 0 +MIArchVgpr: true +MIBlock: +- 16 +- 16 +- 32 +- 1 +- 1 +- 1 +MIInputPerThread: 16 +MIInputPerThreadA: 16 +MIInputPerThreadB: 16 +MIInputPerThreadMetadata: 16 +MIOutputVectorWidth: 8 +MIRegPerOut: 1 +MIWaveGroup: +- 2 +- 2 +MIWaveTile: +- 1 +- 1 +MIWaveTileA: 1 +MIWaveTileB: 1 +MIWaveTileMetadata: 0 +MXLoadInst: TDM +MXScaleFormat: InMemorySwizzle +MacroTile0: 32 +MacroTile1: 32 +MacroTileA: 32 +MacroTileB: 32 +MagicDivAlg: 2 +MathClocksUnrolledLoop: 0 +MatrixInstB: 1 +MatrixInstBM: 1 +MatrixInstBN: 1 +MatrixInstK: 32 +MatrixInstM: 16 +MatrixInstN: 16 +MatrixInstruction: +- 16 +- 16 +- 32 +- 1 +MaxLDS: 327680 +MaxOccupancy: 64 +MbskPrefetchMethod: 0 +MfmaInitCVgprs: false +MinGRIncPerMfma: 1 +Multicast: true +NoLdsWriteCode: true +NoReject: false +NoTailLoop: false +NonDTLTailLoopA: false +NonDTLTailLoopB: false +NonDTLTailLoopMetadata: false +NonTemporal: -1 +NonTemporalA: 0 +NonTemporalB: 0 +NonTemporalC: 0 +NonTemporalD: 0 +NonTemporalE: 0 +NonTemporalGate: 0 +NonTemporalMXSA: 0 +NonTemporalMXSB: 0 +NonTemporalMetadata: 0 +NonTemporalWS: 0 +NonVolatile: -1 +NonVolatileA: 0 +NonVolatileB: 0 +NonVolatileC: 0 +NonVolatileD: 0 +NonVolatileE: 0 +NonVolatileGate: 0 +NonVolatileMXSA: 0 +NonVolatileMXSB: 0 +NonVolatileMetadata: 0 +NonVolatileWS: 0 +NumElementsPerBatchStore: 0 +NumElementsPerThread: 8 +NumGlobalWriteVectorsPerThread: 8 +NumLdsBlk: 2 +NumLoadsA: 1 +NumLoadsB: 1 +NumLoadsCoalescedA: 1 +NumLoadsCoalescedB: 1 +NumLoadsPerpendicularA: 1 +NumLoadsPerpendicularB: 1 +NumThreads: 128 +NumTotalPackedLoadsA: -1 +NumTotalPackedLoadsB: -1 +NumWaveSplitK: 1 +NumWaves: 4 +OptNoLoadLoop: 1 +PackedC0IdxChars: +- I +PackedC0IndicesX: +- 0 +PackedC1IdxChars: +- J +PackedC1IndicesX: +- 1 +PersistentXCCMapping: 0 +PostLoopStoreInNll: false +PrefetchAcrossPersistent: 0 +PrefetchGL2: 0 +PrefetchGlobalRead: 1 +PrefetchLocalRead: 1 +PreloadKernArgs: true +ProblemType: + Activation: false + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: none + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: false + BiasDataTypeList: [] + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 7 + DataTypeA: 7 + DataTypeAmaxD: 0 + DataTypeB: 7 + DataTypeE: 0 + DataTypeMXSA: 22 + DataTypeMXSB: 22 + DestDataType: 0 + F32XdlMathOp: 0 + FusedGemmA2A: false + GateResidualDataTypeList: [] + Gradient: false + GroupedGemm: false + HighPrecisionAccumulate: true + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: + - 3 + - 0 + - 2 + IndexAssignmentsB: + - 3 + - 1 + - 2 + IndexAssignmentsLD: + - 4 + - 5 + - 6 + - 7 + IndexAssignmentsMetadata: + - 3 + - 0 + - 2 + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndexUnrollM: 0 + IndicesBatch: + - 2 + IndicesFree: + - 0 + - 1 + IndicesSummation: + - 3 + Int4EncodingA: Signed + MXBlockA: 0 + MXBlockB: 0 + MXBlockFreeA: 1 + MXBlockFreeB: 1 + MacDataTypeA: 7 + MacDataTypeB: 7 + MetadataLayout: 0 + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMXSA: [] + MirrorDimsMXSB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + ScaleBlockSizesA: [] + ScaleZeroPointA: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SetConstStrideGate: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: false + StridedBatched: true + SupportUserArgs: true + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: false + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseBias: 0 + UseE: false + UseGateResidual: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: '' + UseScaleAlphaVec: 0 + UseScaleCD: false + isMixMode: false +ReuseAcrossPersistent: 0 +SFCWGM: +- - 1 + - 1 +- - 1 + - 1 +ScheduleGROverBarrier: 0 +ScheduleGlobalRead: 1 +ScheduleIterAlg: 4 +ScheduleLocalWrite: 1 +SourceSwap: false +SpaceFillingAlgo: [] +StaggerU: 0 +StaggerUMapping: 0 +StaggerUStride: 0 +StinkyTofuRegisterAllocation: 0 +StorePriorityOpt: false +StoreRemapVectorWidth: 0 +StoreSwapAddr: false +StoreSyncOpt: 0 +StoreVectorWidth: 8 +StreamKAtomic: 0 +StreamKFixupTreeReduction: 0 +SubGroup0: 4 +SubGroup1: 32 +SubGroupA: 4 +SubGroupB: 32 +SuppressNoLoadLoop: false +SwInstructionPrefetch: 1 +SwapGlobalReadOrder: 0 +TDMInst: 3 +TDMIterateMode: 0 +TDMLoadWaveSync: false +TDMPlusLdsBuf: 0 +TDMSplit: false +TailloopInNll: false +TemporalHint: -1 +TemporalHintA: 0 +TemporalHintB: 0 +TemporalHintC: 0 +TemporalHintD: 0 +TemporalHintE: 0 +TemporalHintGate: 0 +TemporalHintMXSA: 0 +TemporalHintMXSB: 0 +TemporalHintMetadata: 0 +TemporalHintWS: 0 +ThreadTile: +- 1 +- 1 +ThreadTile0: 8 +ThreadTile1: 1 +ThreadTileA: 8 +ThreadTileB: 1 +TileProcessingStrategy: StreamK +TransposeLDS: 1 +TransposeLDSMetadata: -1 +ULSGRODoubleG2L: 0 +UnrollLoopSwapGlobalReadOrder: 0 +UnrollMajorLDSA: true +UnrollMajorLDSB: true +Use64bShadowLimit: true +Use64bShadowLimitMX: false +UseCustomMainLoopSchedule: 0 +UseDirect32XEmulation: false +UseDot2F32XEmulation: false +UseDotInstruction: false +UseDualFMAC: false +UseF32XEmulation: false +UseGeneralizedNLCOneA: false +UseGeneralizedNLCOneB: false +UseGeneralizedNLCOneMXSA: false +UseGeneralizedNLCOneMXSB: false +UseGeneralizedNLCOneMetadata: false +UseInstOffsetForGRO: 0 +UseMFMAF32XEmulation: false +UsePLRPack: 0 +UseSgprForGRO: -1 +UseSubtileImpl: false +Valid: true +VectorStore: -1 +VectorWidthA: 1 +VectorWidthB: 1 +WaveSeparateGlobalReadA: 0 +WaveSeparateGlobalReadB: 0 +WaveSeparateGlobalReadMetadata: 0 +WaveSplitK: false +WavefrontSize: 32 +WorkAssignment: StaticGrid +WorkGroup: +- 32 +- 4 +- 1 +WorkGroupMapping: 1 +WorkGroupMappingXCC: 1 +WorkGroupMappingXCCGroup: -1 +WorkGroupReduction: false +WorkQueueStealing: 0 +WorkspaceCheck: +- 4 +- 0 +- 0 +_DepthU: 64 +_DepthUA: 64 +_DepthUB: 64 +_DepthUMetadata: 64 +_DualPeriodMXFP8: false +_GlobalAccumulation: PartialsBuffer +_HasTemporalHint: 1 +_PersistentLoop: true +_PersistentVectorEpilogueLds: false +_PrefetchAcrossPersistentEnabled: false +_ScalePeriodUA: 64 +_ScalePeriodUB: 64 +_ScaleReuseA: 1 +_ScaleReuseB: 1 +_ScheduleIterAlg: 0 +_SeparateEpilogueLds: false +_StinkyTofuOptLevel: 3 +_UseSgprForGRO: 1 +_VectorStore: 1 +_WorkspaceSizePerElemBias: 0 +_WorkspaceSizePerElemC: 4 +_inputVWA_was_auto: true +_inputVWB_was_auto: true +_staggerStrideShift: 0 +enableGLTrA: false +enableGLTrB: false +enableLDSTrA: false +enableLDSTrB: false +enableLDSTrMXSA: false +enableLDSTrMXSB: false +enableLDSTrMetadata: 0 +enableTDMA: true +enableTDMB: true +enableTDMMetadata: false +ldsNumBytesA: 4352 +ldsNumBytesB: 4352 +ldsNumBytesMXSA: 0 +ldsNumBytesMXSB: 0 +ldsNumBytesMetadata: 0 +numSubTiles: 1 +reorderGRInstForDTVA: false +reorderGRInstForDTVB: false +tailLoopOptA: false +tailLoopOptB: false diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.msgpack new file mode 100644 index 000000000000..91963e127f63 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.predicates.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.predicates.msgpack new file mode 100644 index 000000000000..b5ec3fe4bf62 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.predicates.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.predicates.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.predicates.yaml new file mode 100644 index 000000000000..f0796f1a424d --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.predicates.yaml @@ -0,0 +1,117 @@ +problem: + type: And + value: + - type: OperationIdentifierEqual + value: Contraction_l_Ailk_Bljk_Cijk_Dijk + - type: BiasDataTypeWhiteList + value: + - 0 + - 4 + - type: GateResidualDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: + - 3 + - type: AmaxDCheck + value: false + - type: TypesEqual + value: + - Half + - Half + - Half + - Half + - Half + - Half + - type: HighPrecisionAccumulate + value: true + - type: Activation + value: None + - type: ActivationComputeType + value: Float + - type: ActivationNoGuard + value: false + - type: UseGradient + value: false + - type: UseBias + value: 0 + - type: UseE + value: false + - type: UseGateResidual + value: false + - type: DataTypeE + value: Half + - type: StridedBatched + value: true + - type: GroupedGemm + value: false + - type: UseScaleAB + value: '' + - type: UseScaleCD + value: false + - type: UseScaleAlphaVec + value: 0 + - type: Sparse + value: 0 + - type: F32XdlMathOp + value: Float + - type: SupportDeviceUserArguments + value: false + - type: SwizzleTensorA + value: false + - type: SwizzleTensorB + value: false + - type: FusedGemmA2A + value: false + - type: MXBlockA + value: 0 + - type: MXBlockB + value: 0 + - type: SynchronizerSizeCheck + index: 0 + value: + - 16 + - 16 + - 1 + - 4 + - 1 + - 1 + - type: WorkgroupNumberCheck + index: 0 + value: + - 16 + - 16 + - 1 + - type: LeadingFree0SizesGreaterOrEqual + value: 4 + - type: LeadingFree1SizesGreaterOrEqual + value: 1 + - type: KernelLanguageCompatible + value: Assembly + - type: BufferLoadOffsetLimitCheck + value: + ShiftPtrElemB: 0 + ShiftPtrElemA: 4 + DUorMT1: 16 + DUorMT0: 16 + - type: BufferLoadOffsetLimitCheck_Beta + value: 16 + - type: BufferStoreOffsetLimitCheck + value: 16 + - type: GlobalSplitUCheckMinK + value: + - 32 + - 1 + - type: WorkgroupMappingXCCCheck + value: + - 1 + - -1 +task: + type: And + value: + - type: WorkspaceCheck + - type: LaunchLimits +hardware: + type: AMDGPU + value: + type: Processor + value: gfx90a diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.record.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.record.txt new file mode 100644 index 000000000000..169d82b88c78 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.record.txt @@ -0,0 +1,139 @@ +solutionName="Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSU1_GSUAMBSK_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA90a_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGM8_WGMXCC1_WGMXCCGn1" +kernelName="Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA90a_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1" +sizeMapping.waveNum=1 +sizeMapping.WavefrontSize=64 +sizeMapping.workGroup=[16,4,1] +sizeMapping.macroTile=[16,16,1] +sizeMapping.threadTile=[1,1] +sizeMapping.matrixInstruction=[16,16,16,1] +sizeMapping.grvwA=4 +sizeMapping.grvwB=4 +sizeMapping.gwvwC=4 +sizeMapping.gwvwD=4 +sizeMapping.depthU=16 +sizeMapping.staggerU=32 +sizeMapping.staggerUMapping=0 +sizeMapping.globalSplitUPGR=16 +sizeMapping.globalSplitU=1 +sizeMapping.staggerStrideShift=3 +sizeMapping.workGroupMapping=8 +sizeMapping.packBatchDims=0 +sizeMapping.magicDivAlg=2 +sizeMapping.tileProcessingStrategy="None" +sizeMapping.workAssignment="StaticGrid" +sizeMapping.streamKAtomic=0 +sizeMapping.prefetchAcrossPersistent=0 +sizeMapping.sourceKernel=false +sizeMapping.globalAccumulation=3 +sizeMapping.adaptiveGemmGSUA=0 +sizeMapping.workspaceSizePerElemC=4 +sizeMapping.workspaceSizePerElemBias=0 +sizeMapping.activationFused=true +sizeMapping.workGroupMappingXCC=1 +sizeMapping.workGroupMappingXCCGroup=-1 +sizeMapping.globalSplitUCoalesced=false +sizeMapping.globalSplitUWorkGroupMappingRoundRobin=false +sizeMapping.CUOccupancy=5 +sizeMapping.PrefetchGlobalRead=0 +sizeMapping.MathClocksUnrolledLoop=0 +sizeMapping.synchronizerSizePerWG=4 +sizeMapping.nonTemporalA=0 +sizeMapping.nonTemporalB=0 +sizeMapping.temporalHintA=0 +sizeMapping.temporalHintB=0 +sizeMapping.hasTemporalHint=false +sizeMapping.adaptiveGemmNTAB=0 +sizeMapping.customMainLoopScheduling=0 +sizeMapping.useSubtileImpl=false +sizeMapping.NonTemporalD=0 +sizeMapping.WaveSeparateGlobalReadA=0 +sizeMapping.WaveSeparateGlobalReadB=0 +sizeMapping.UnrollLoopSwapGlobalReadOrder=0 +sizeMapping.DirectToVgprA=false +sizeMapping.DirectToVgprB=false +sizeMapping.NumLoadsCoalescedA=1 +sizeMapping.NumLoadsCoalescedB=1 +sizeMapping.WaveGroup=[1,1] +sizeMapping.VectorWidthA=1 +sizeMapping.VectorWidthB=1 +sizeMapping.LocalSplitU=1 +sizeMapping.DirectToLdsA=false +sizeMapping.DirectToLdsB=false +sizeMapping.OneLDSBuffer=false +sizeMapping.SourceSwap=false +sizeMapping.UnrollMajorLDSA=0 +sizeMapping.UnrollMajorLDSB=0 +sizeMapping.enableLDSTrA=false +sizeMapping.enableLDSTrB=false +sizeMapping.ExpertSchedulingMode=0 +sizeMapping.clusterDim=[1,1] +sizeMapping.BufferLoad=true +sizeMapping.BufferStore=true +internalArgsSupport.version=3 +internalArgsSupport.persistentLoopArgsVersion=0 +internalArgsSupport.gsu=true +internalArgsSupport.wgm=true +internalArgsSupport.staggerU=true +internalArgsSupport.perTileExtraIters=false +internalArgsSupport.useUniversalArgs=true +internalArgsSupport.useSFC=false +problemType.operationIdentifier="Contraction_l_Ailk_Bljk_Cijk_Dijk" +problemType.transA=false +problemType.transB=false +problemType.computeInputTypeA="Half" +problemType.computeInputTypeB="Half" +problemType.aType="Half" +problemType.bType="Half" +problemType.cType="Half" +problemType.dType="Half" +problemType.eType="Half" +problemType.computeType="Float" +problemType.useBeta=true +problemType.useBias=0 +problemType.biasSrcWhiteList=[3] +problemType.useE=false +problemType.useGateResidual=false +problemType.gateResidualDataTypeWhiteList=[] +problemType.useScaleAB="" +problemType.useScaleCD=false +problemType.useScaleAlphaVec=0 +problemType.biasDataTypeWhiteList=[0,4] +problemType.highPrecisionAccumulate=true +problemType.useInitialStridesAB=false +problemType.useInitialStridesCD=false +problemType.stridedBatched=true +problemType.groupedGemm=false +problemType.useGradient=false +problemType.activationType="None" +problemType.activationArgLength=0 +problemType.activationComputeDataType="Float" +problemType.activationNoGuard=false +problemType.sparse=0 +problemType.f32XdlMathOp="Float" +problemType.supportDeviceUserArguments=false +problemType.outputAmaxD=false +problemType.swizzleTensorA=false +problemType.swizzleTensorB=false +problemType.metadataLayout=0 +problemType.mxBlockA=0 +problemType.mxBlockB=0 +problemType.mxBlockFreeA=1 +problemType.mxBlockFreeB=1 +problemType.mxTypeA="E8" +problemType.mxTypeB="E8" +problemType.mxScaleFormat=0 +problemType.fusedGemmA2A=false +problemType.stochasticRounding=false +problemPredicate=["And","OperationIdentifierEqual","BiasDataTypeWhiteList","GateResidualDataTypeWhiteList","BiasSrcWhiteList","AmaxDCheck","TypesEqual","HighPrecisionAccumulate","Activation","ActivationComputeType","ActivationNoGuard","UseGradient","UseBias","UseE","UseGateResidual","DataTypeE","StridedBatched","GroupedGemm","UseScaleAB","UseScaleCD","UseScaleAlphaVec","Sparse","F32XdlMathOp","SupportDeviceUserArguments","SwizzleTensorA","SwizzleTensorB","FusedGemmA2A","MXBlockA","MXBlockB","SynchronizerSizeCheck","WorkgroupNumberCheck","LeadingFree0SizesGreaterOrEqual","LeadingFree1SizesGreaterOrEqual","KernelLanguageCompatible","BufferLoadOffsetLimitCheck","BufferLoadOffsetLimitCheck_Beta","BufferStoreOffsetLimitCheck","GlobalSplitUCheckMinK","WorkgroupMappingXCCCheck"] +taskPredicate=["And","WorkspaceCheck","LaunchLimits"] +hardwarePredicate=["AMDGPU","Processor"] +customKernel.name="Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA90a_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1" +customKernel.args=[{"semantic":"GemmInfo","type":"uint32"},{"semantic":"InternalArgs","type":"uint32"},{"semantic":"InternalArgs1","type":"uint32"},{"semantic":"NumWorkGroups","type":"uint32"},{"semantic":"SizeFree0","type":"uint32"},{"semantic":"SizeFree1","type":"uint32"},{"semantic":"SizeFree2","type":"uint32"},{"semantic":"SizeSum","type":"uint32"},{"semantic":"AddressD","type":"address"},{"semantic":"AddressC","type":"address"},{"semantic":"AddressA","type":"address"},{"semantic":"AddressB","type":"address"},{"semantic":"StrideD0","type":"uint32"},{"semantic":"StrideD1","type":"uint32"},{"semantic":"StrideC0","type":"uint32"},{"semantic":"StrideC1","type":"uint32"},{"semantic":"StrideA0","type":"uint32"},{"semantic":"StrideA1","type":"uint32"},{"semantic":"StrideB0","type":"uint32"},{"semantic":"StrideB1","type":"uint32"},{"semantic":"Alpha","type":"uint32"},{"semantic":"Beta","type":"uint32"},{"semantic":"AddressTD","type":"address"},{"semantic":"Synchronizer","type":"address"},{"semantic":"GSUSync","type":"uint32"}] +customKernel.macrotile=[16,16,16] +customKernel.threads=[64,1,1] +customKernel.grid=["TilesXYBatchGSU","One","One"] +customKernel.workspaceType="None" +customKernel.workspaceSizePerElemC=0 +customKernel.workspaceSizePerElemBias=0 +customKernel.generated=true +customKernel.uniformSummationOrder=false diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.yaml new file mode 100644 index 000000000000..d78920781a4d --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx90a-plain.yaml @@ -0,0 +1,568 @@ +1LDSBuffer: 0 +ActivationAlt: false +ActivationFuncCall: false +ActivationFused: true +AdaptiveGemm: 0 +AdaptiveGemmGSUA: 0 +AdaptiveGemmNTAB: 0 +AssertAIGreaterThanEqual: -1 +AssertAILessThanEqual: -1 +AssertFree0ElementMultiple: 1 +AssertFree1ElementMultiple: 1 +AssertSummationElementMultiple: 1 +AssignedDerivedParameters: true +AssignedProblemIndependentDerivedParameters: true +BaseName: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LD8o0J2kRSAfWwkXhY4TRGnZEgIpreJSG7l9ugobCrSio= +BufferLoad: true +BufferStore: true +CUOccupancy: 5 +ClusterBarrier: false +ClusterDim: +- 1 +- 1 +ClusterLocalRead: 1 +CodeObjectVersion: '4' +CompactLoopStore: false +ConvertAfterDS: false +CustomKernel: + args: + - semantic: GemmInfo + type: uint32 + - semantic: InternalArgs + type: uint32 + - semantic: InternalArgs1 + type: uint32 + - semantic: NumWorkGroups + type: uint32 + - semantic: SizeFree0 + type: uint32 + - semantic: SizeFree1 + type: uint32 + - semantic: SizeFree2 + type: uint32 + - semantic: SizeSum + type: uint32 + - semantic: AddressD + type: address + - semantic: AddressC + type: address + - semantic: AddressA + type: address + - semantic: AddressB + type: address + - semantic: StrideD0 + type: uint32 + - semantic: StrideD1 + type: uint32 + - semantic: StrideC0 + type: uint32 + - semantic: StrideC1 + type: uint32 + - semantic: StrideA0 + type: uint32 + - semantic: StrideA1 + type: uint32 + - semantic: StrideB0 + type: uint32 + - semantic: StrideB1 + type: uint32 + - semantic: Alpha + type: uint32 + - semantic: Beta + type: uint32 + - semantic: AddressTD + type: address + - semantic: Synchronizer + type: address + - semantic: GSUSync + type: uint32 + generated: true + grid: + - TilesXYBatchGSU + - One + - One + macrotile: + - 16 + - 16 + - 16 + name: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA90a_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1 + threads: + - 64 + - 1 + - 1 + workspaceSizePerElemBias: 0 + workspaceSizePerElemC: 0 + workspaceType: None +DebugPersistentKernelLoopForever: false +DebugStreamK: 0 +DepthU: 16 +DirectToLds: 0 +DirectToLdsA: false +DirectToLdsB: false +DirectToLdsMetadata: 0 +DirectToVgprA: false +DirectToVgprB: false +DirectToVgprMXSA: false +DirectToVgprMXSB: false +DirectToVgprSparseMetadata: false +DisableXdlArbStall: true +DtlPlusLdsBuf: 0 +ESMRuntimeGate: false +EdgeType: ShiftPtr +EnableESM2TrackValuVsrc: true +EnableF32XdlMathOp: false +EnableMatrixInstruction: true +EnableStinkyTofuESM2: false +ExpandPointerSwap: false +ExpertSchedulingMode: 0 +ExtraLatencyForLR: 0 +ExtraMiLatencyLeft: -1 +ForceDisableShadowInit: false +ForceUnrollSubIter: false +GlobalReadPerMfma: 1.0 +GlobalReadVectorWidthA: 4 +GlobalReadVectorWidthB: 4 +GlobalSplitU: 1 +GlobalSplitUAlgorithm: MultipleBufferSingleKernel +GlobalSplitUCoalesced: false +GlobalSplitUWorkGroupMappingRoundRobin: false +GlobalWriteVectorWidth: 1 +GroupLoadStore: false +GuaranteeNoPartialA: false +GuaranteeNoPartialB: true +GuaranteeNoPartialMetadata: true +HalfPLR: 0 +HalfPLRA: false +HalfPLRB: false +ISA: +- 9 +- 0 +- 10 +InitCIterWmma: 0 +InnerUnroll: 1 +InterleaveAlpha: 0 +InternalSupportParams: + KernArgsVersion: 3 + PersistentLoopArgsVersion: 0 + SupportCustomStaggerU: true + SupportCustomWGM: true + SupportStreamKPerTileExtraIters: false + SupportUserGSU: true + UseSFC: false + UseUniversalArgs: true +Kernel: true +KernelLanguage: Assembly +KernelNameMin: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA90a_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1 +LDSSegInterleaveOffsets: null +LDSSegmentInterleave: 0 +LDSTrInst: false +LSCA: 16 +LSCB: 16 +LSPA: 16 +LSPB: 16 +LVCA: 4 +LVCB: 4 +LVPA: 4 +LVPB: 4 +LdsBlockSizePerPadA: 128 +LdsBlockSizePerPadB: 128 +LdsBlockSizePerPadMXSA: 0 +LdsBlockSizePerPadMXSB: 0 +LdsBlockSizePerPadMetadata: 0 +LdsBytesNoAmax: 1280 +LdsInitCVgprs: false +LdsNumBytes: 1280 +LdsNumElementsAlignedA: 640 +LdsNumElementsAlignedB: 640 +LdsNumElementsAlignedMXSA: 0 +LdsNumElementsAlignedMXSB: 0 +LdsNumElementsAlignedMetadata: 0 +LdsOffsetA: 0 +LdsOffsetA_Blk: 0 +LdsOffsetB: 640 +LdsOffsetB_Blk: 0 +LdsOffsetBias: 0 +LdsOffsetBiasGSU: 0 +LdsOffsetBiasNonGSU: 0 +LdsOffsetMXSA: 640 +LdsOffsetMXSA_Blk: 0 +LdsOffsetMXSB: 640 +LdsOffsetMXSB_Blk: 0 +LdsOffsetMetadata: 640 +LdsOffsetMetadata_Blk: 0 +LdsPadA: 16 +LdsPadB: 16 +LdsPadMXSA: 0 +LdsPadMXSB: 0 +LdsPadMetadata: -1 +LocalReadVectorWidth: -1 +LocalReadVectorWidthA: 4 +LocalReadVectorWidthB: 4 +LocalSplitU: 1 +LocalSplitUReuseLDS: 1 +LocalWritePerMfma: -1 +LocalWriteUseSgprA: false +LocalWriteUseSgprB: false +LocalWriteUseSgprMetadata: false +LoopIters: 1 +LoopUnroll: 16 +MFMA_BF16_1K: false +MIArchVgpr: false +MIBlock: +- 16 +- 16 +- 16 +- 1 +- 1 +- 1 +MIInputPerThread: 4 +MIInputPerThreadA: 4 +MIInputPerThreadB: 4 +MIInputPerThreadMetadata: 4 +MIOutputVectorWidth: 4 +MIRegPerOut: 1 +MIWaveGroup: +- 1 +- 1 +MIWaveTile: +- 1 +- 1 +MIWaveTileA: 1 +MIWaveTileB: 1 +MIWaveTileMetadata: 0 +MXLoadInst: BufferLoad +MXScaleFormat: NoSwizzle +MacroTile0: 16 +MacroTile1: 16 +MacroTileA: 16 +MacroTileB: 16 +MagicDivAlg: 2 +MathClocksUnrolledLoop: 0 +MatrixInstB: 1 +MatrixInstBM: 1 +MatrixInstBN: 1 +MatrixInstK: 16 +MatrixInstM: 16 +MatrixInstN: 16 +MatrixInstruction: +- 16 +- 16 +- 16 +- 1 +MaxLDS: 65536 +MaxOccupancy: 64 +MbskPrefetchMethod: 0 +MfmaInitCVgprs: false +MinGRIncPerMfma: 1 +Multicast: false +NoLdsWriteCode: false +NoReject: false +NoTailLoop: false +NonDTLTailLoopA: false +NonDTLTailLoopB: false +NonDTLTailLoopMetadata: false +NonTemporal: -1 +NonTemporalA: 0 +NonTemporalB: 0 +NonTemporalC: 0 +NonTemporalD: 0 +NonTemporalE: 0 +NonTemporalGate: 0 +NonTemporalMXSA: 0 +NonTemporalMXSB: 0 +NonTemporalMetadata: 0 +NonTemporalWS: 0 +NonVolatile: -1 +NonVolatileA: 0 +NonVolatileB: 0 +NonVolatileC: 0 +NonVolatileD: 0 +NonVolatileE: 0 +NonVolatileGate: 0 +NonVolatileMXSA: 0 +NonVolatileMXSB: 0 +NonVolatileMetadata: 0 +NonVolatileWS: 0 +NumElementsPerBatchStore: 0 +NumElementsPerThread: 4 +NumGlobalWriteVectorsPerThread: 4 +NumLdsBlk: 2 +NumLoadsA: 1 +NumLoadsB: 1 +NumLoadsCoalescedA: 1 +NumLoadsCoalescedB: 1 +NumLoadsPerpendicularA: 1 +NumLoadsPerpendicularB: 1 +NumThreads: 64 +NumTotalPackedLoadsA: -1 +NumTotalPackedLoadsB: -1 +NumWaveSplitK: 1 +NumWaves: 1 +OptNoLoadLoop: 0 +PackedC0IdxChars: +- I +PackedC0IndicesX: +- 0 +PackedC1IdxChars: +- J +PackedC1IndicesX: +- 1 +PersistentXCCMapping: 0 +PrefetchAcrossPersistent: 0 +PrefetchGL2: 0 +PrefetchGlobalRead: 0 +PrefetchLocalRead: 0 +PreloadKernArgs: false +ProblemType: + Activation: false + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: none + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: false + BiasDataTypeList: + - 0 + - 4 + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 4 + DataTypeA: 4 + DataTypeAmaxD: 0 + DataTypeB: 4 + DataTypeE: 4 + DataTypeMXSA: 22 + DataTypeMXSB: 22 + DestDataType: 4 + F32XdlMathOp: 0 + FusedGemmA2A: false + GateResidualDataTypeList: [] + Gradient: false + GroupedGemm: false + HighPrecisionAccumulate: true + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: + - 0 + - 3 + - 2 + IndexAssignmentsB: + - 3 + - 1 + - 2 + IndexAssignmentsLD: + - 4 + - 5 + - 6 + - 7 + IndexAssignmentsMetadata: + - 3 + - 0 + - 2 + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndexUnrollM: 0 + IndicesBatch: + - 2 + IndicesFree: + - 0 + - 1 + IndicesSummation: + - 3 + MXBlockA: 0 + MXBlockB: 0 + MacDataTypeA: 4 + MacDataTypeB: 4 + MetadataLayout: 0 + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMXSA: [] + MirrorDimsMXSB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SetConstStrideGate: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: false + StridedBatched: true + SupportUserArgs: false + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseBias: 0 + UseE: false + UseGateResidual: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: '' + UseScaleAlphaVec: 0 + UseScaleCD: false + isMixMode: false +ReuseAcrossPersistent: 0 +SFCWGM: +- - 1 + - 1 +- - 1 + - 1 +ScheduleGROverBarrier: 0 +ScheduleGlobalRead: 1 +ScheduleIterAlg: 3 +ScheduleLocalWrite: 1 +SolutionIndex: 0 +SolutionNameMin: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSU1_GSUAMBSK_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA90a_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGM8_WGMXCC1_WGMXCCGn1 +SourceSwap: false +SpaceFillingAlgo: [] +StaggerU: 32 +StaggerUMapping: 0 +StaggerUStride: 256 +StorePriorityOpt: false +StoreRemapVectorWidth: 0 +StoreSwapAddr: false +StoreSyncOpt: 0 +StoreVectorWidth: 4 +StreamKAtomic: 0 +StreamKFixupTreeReduction: 0 +SubGroup0: 4 +SubGroup1: 16 +SubGroupA: 4 +SubGroupB: 16 +SuppressNoLoadLoop: false +SwInstructionPrefetch: 1 +SwapGlobalReadOrder: 0 +SynchronizerSizeCheck: 1 +TDMInst: 0 +TDMIterateMode: 0 +TDMLoadWaveSync: false +TDMPlusLdsBuf: 0 +TDMSplit: false +TailloopInNll: false +TemporalHint: -1 +TemporalHintA: 0 +TemporalHintB: 0 +TemporalHintC: 0 +TemporalHintD: 0 +TemporalHintE: 0 +TemporalHintGate: 0 +TemporalHintMXSA: 0 +TemporalHintMXSB: 0 +TemporalHintMetadata: 0 +TemporalHintWS: 0 +ThreadTile: +- 1 +- 1 +ThreadTile0: 4 +ThreadTile1: 1 +ThreadTileA: 4 +ThreadTileB: 1 +TileProcessingStrategy: None +TransposeLDS: 0 +TransposeLDSMetadata: -1 +ULSGRODoubleG2L: 0 +UnrollLoopSwapGlobalReadOrder: 0 +UnrollMajorLDSA: 0 +UnrollMajorLDSB: 0 +Use64bShadowLimit: true +Use64bShadowLimitMX: false +UseCustomMainLoopSchedule: 0 +UseDirect32XEmulation: false +UseDot2F32XEmulation: false +UseDotInstruction: false +UseDualFMAC: false +UseF32XEmulation: false +UseGeneralizedNLCOneA: false +UseGeneralizedNLCOneB: false +UseGeneralizedNLCOneMXSA: false +UseGeneralizedNLCOneMXSB: false +UseGeneralizedNLCOneMetadata: false +UseInstOffsetForGRO: 0 +UseMFMAF32XEmulation: false +UsePLRPack: 0 +UseSgprForGRO: -1 +UseSubtileImpl: false +Valid: true +VectorStore: -1 +VectorWidthA: 1 +VectorWidthB: 1 +WaveSeparateGlobalReadA: 0 +WaveSeparateGlobalReadB: 0 +WaveSeparateGlobalReadMetadata: 0 +WaveSplitK: false +WavefrontSize: 64 +WorkAssignment: StaticGrid +WorkGroup: +- 16 +- 4 +- 1 +WorkGroupMapping: 8 +WorkGroupMappingXCC: 1 +WorkGroupMappingXCCGroup: -1 +WorkGroupReduction: false +WorkQueueStealing: 0 +WorkspaceCheck: +- 4 +- 0 +- 1 +_DepthU: 16 +_DepthUA: 16 +_DepthUB: 16 +_DepthUMetadata: 16 +_GlobalAccumulation: MultipleBufferSingleKernel +_HasTemporalHint: 0 +_PersistentLoop: false +_PersistentVectorEpilogueLds: false +_PrefetchAcrossPersistentEnabled: false +_ScheduleIterAlg: 3 +_SeparateEpilogueLds: false +_StinkyTofuOptLevel: 0 +_UseSgprForGRO: false +_VectorStore: 1 +_WorkspaceSizePerElemBias: 0 +_WorkspaceSizePerElemC: 4 +_inputVWA_was_auto: true +_inputVWB_was_auto: true +_staggerStrideShift: 3 +enableGLTrA: false +enableGLTrB: false +enableLDSTrA: false +enableLDSTrB: false +enableLDSTrMXSA: false +enableLDSTrMXSB: false +enableLDSTrMetadata: 0 +enableTDMA: false +enableTDMB: false +enableTDMMetadata: false +ldsNumBytesA: 640 +ldsNumBytesB: 640 +ldsNumBytesMXSA: 0 +ldsNumBytesMXSB: 0 +ldsNumBytesMetadata: 0 +numSubTiles: 1 +reorderGRInstForDTVA: false +reorderGRInstForDTVB: false +tailLoopOptA: false +tailLoopOptB: false diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.msgpack new file mode 100644 index 000000000000..eefa95d13e9d Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.predicates.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.predicates.msgpack new file mode 100644 index 000000000000..3785a905482f Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.predicates.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.predicates.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.predicates.yaml new file mode 100644 index 000000000000..513e698dc49a --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.predicates.yaml @@ -0,0 +1,117 @@ +problem: + type: And + value: + - type: OperationIdentifierEqual + value: Contraction_l_Ailk_Bljk_Cijk_Dijk + - type: BiasDataTypeWhiteList + value: + - 0 + - 4 + - type: GateResidualDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: + - 3 + - type: AmaxDCheck + value: false + - type: TypesEqual + value: + - Half + - Half + - Half + - Half + - Half + - Half + - type: HighPrecisionAccumulate + value: true + - type: Activation + value: None + - type: ActivationComputeType + value: Float + - type: ActivationNoGuard + value: false + - type: UseGradient + value: false + - type: UseBias + value: 0 + - type: UseE + value: false + - type: UseGateResidual + value: false + - type: DataTypeE + value: Half + - type: StridedBatched + value: true + - type: GroupedGemm + value: false + - type: UseScaleAB + value: '' + - type: UseScaleCD + value: false + - type: UseScaleAlphaVec + value: 0 + - type: Sparse + value: 0 + - type: F32XdlMathOp + value: Float + - type: SupportDeviceUserArguments + value: false + - type: SwizzleTensorA + value: false + - type: SwizzleTensorB + value: false + - type: FusedGemmA2A + value: false + - type: MXBlockA + value: 0 + - type: MXBlockB + value: 0 + - type: SynchronizerSizeCheck + index: 0 + value: + - 16 + - 16 + - 1 + - 4 + - 1 + - 1 + - type: WorkgroupNumberCheck + index: 0 + value: + - 16 + - 16 + - 1 + - type: LeadingFree0SizesGreaterOrEqual + value: 4 + - type: LeadingFree1SizesGreaterOrEqual + value: 1 + - type: KernelLanguageCompatible + value: Assembly + - type: BufferLoadOffsetLimitCheck + value: + ShiftPtrElemB: 0 + ShiftPtrElemA: 4 + DUorMT1: 16 + DUorMT0: 16 + - type: BufferLoadOffsetLimitCheck_Beta + value: 16 + - type: BufferStoreOffsetLimitCheck + value: 16 + - type: GlobalSplitUCheckMinK + value: + - 32 + - 1 + - type: WorkgroupMappingXCCCheck + value: + - 1 + - -1 +task: + type: And + value: + - type: WorkspaceCheck + - type: LaunchLimits +hardware: + type: AMDGPU + value: + type: Processor + value: gfx942 diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.record.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.record.txt new file mode 100644 index 000000000000..a031d612592d --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.record.txt @@ -0,0 +1,139 @@ +solutionName="Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSU1_GSUAMBSK_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGM8_WGMXCC1_WGMXCCGn1" +kernelName="Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA942_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1" +sizeMapping.waveNum=1 +sizeMapping.WavefrontSize=64 +sizeMapping.workGroup=[16,4,1] +sizeMapping.macroTile=[16,16,1] +sizeMapping.threadTile=[1,1] +sizeMapping.matrixInstruction=[16,16,16,1] +sizeMapping.grvwA=4 +sizeMapping.grvwB=4 +sizeMapping.gwvwC=4 +sizeMapping.gwvwD=4 +sizeMapping.depthU=16 +sizeMapping.staggerU=32 +sizeMapping.staggerUMapping=0 +sizeMapping.globalSplitUPGR=16 +sizeMapping.globalSplitU=1 +sizeMapping.staggerStrideShift=3 +sizeMapping.workGroupMapping=8 +sizeMapping.packBatchDims=0 +sizeMapping.magicDivAlg=2 +sizeMapping.tileProcessingStrategy="None" +sizeMapping.workAssignment="StaticGrid" +sizeMapping.streamKAtomic=0 +sizeMapping.prefetchAcrossPersistent=0 +sizeMapping.sourceKernel=false +sizeMapping.globalAccumulation=3 +sizeMapping.adaptiveGemmGSUA=0 +sizeMapping.workspaceSizePerElemC=4 +sizeMapping.workspaceSizePerElemBias=0 +sizeMapping.activationFused=true +sizeMapping.workGroupMappingXCC=1 +sizeMapping.workGroupMappingXCCGroup=-1 +sizeMapping.globalSplitUCoalesced=false +sizeMapping.globalSplitUWorkGroupMappingRoundRobin=false +sizeMapping.CUOccupancy=5 +sizeMapping.PrefetchGlobalRead=0 +sizeMapping.MathClocksUnrolledLoop=256 +sizeMapping.synchronizerSizePerWG=4 +sizeMapping.nonTemporalA=0 +sizeMapping.nonTemporalB=0 +sizeMapping.temporalHintA=0 +sizeMapping.temporalHintB=0 +sizeMapping.hasTemporalHint=false +sizeMapping.adaptiveGemmNTAB=0 +sizeMapping.customMainLoopScheduling=0 +sizeMapping.useSubtileImpl=false +sizeMapping.NonTemporalD=0 +sizeMapping.WaveSeparateGlobalReadA=0 +sizeMapping.WaveSeparateGlobalReadB=0 +sizeMapping.UnrollLoopSwapGlobalReadOrder=0 +sizeMapping.DirectToVgprA=false +sizeMapping.DirectToVgprB=false +sizeMapping.NumLoadsCoalescedA=1 +sizeMapping.NumLoadsCoalescedB=1 +sizeMapping.WaveGroup=[1,1] +sizeMapping.VectorWidthA=1 +sizeMapping.VectorWidthB=1 +sizeMapping.LocalSplitU=1 +sizeMapping.DirectToLdsA=false +sizeMapping.DirectToLdsB=false +sizeMapping.OneLDSBuffer=false +sizeMapping.SourceSwap=false +sizeMapping.UnrollMajorLDSA=0 +sizeMapping.UnrollMajorLDSB=0 +sizeMapping.enableLDSTrA=false +sizeMapping.enableLDSTrB=false +sizeMapping.ExpertSchedulingMode=0 +sizeMapping.clusterDim=[1,1] +sizeMapping.BufferLoad=true +sizeMapping.BufferStore=true +internalArgsSupport.version=3 +internalArgsSupport.persistentLoopArgsVersion=0 +internalArgsSupport.gsu=true +internalArgsSupport.wgm=true +internalArgsSupport.staggerU=true +internalArgsSupport.perTileExtraIters=false +internalArgsSupport.useUniversalArgs=true +internalArgsSupport.useSFC=false +problemType.operationIdentifier="Contraction_l_Ailk_Bljk_Cijk_Dijk" +problemType.transA=false +problemType.transB=false +problemType.computeInputTypeA="Half" +problemType.computeInputTypeB="Half" +problemType.aType="Half" +problemType.bType="Half" +problemType.cType="Half" +problemType.dType="Half" +problemType.eType="Half" +problemType.computeType="Float" +problemType.useBeta=true +problemType.useBias=0 +problemType.biasSrcWhiteList=[3] +problemType.useE=false +problemType.useGateResidual=false +problemType.gateResidualDataTypeWhiteList=[] +problemType.useScaleAB="" +problemType.useScaleCD=false +problemType.useScaleAlphaVec=0 +problemType.biasDataTypeWhiteList=[0,4] +problemType.highPrecisionAccumulate=true +problemType.useInitialStridesAB=false +problemType.useInitialStridesCD=false +problemType.stridedBatched=true +problemType.groupedGemm=false +problemType.useGradient=false +problemType.activationType="None" +problemType.activationArgLength=0 +problemType.activationComputeDataType="Float" +problemType.activationNoGuard=false +problemType.sparse=0 +problemType.f32XdlMathOp="Float" +problemType.supportDeviceUserArguments=false +problemType.outputAmaxD=false +problemType.swizzleTensorA=false +problemType.swizzleTensorB=false +problemType.metadataLayout=0 +problemType.mxBlockA=0 +problemType.mxBlockB=0 +problemType.mxBlockFreeA=1 +problemType.mxBlockFreeB=1 +problemType.mxTypeA="E8" +problemType.mxTypeB="E8" +problemType.mxScaleFormat=0 +problemType.fusedGemmA2A=false +problemType.stochasticRounding=false +problemPredicate=["And","OperationIdentifierEqual","BiasDataTypeWhiteList","GateResidualDataTypeWhiteList","BiasSrcWhiteList","AmaxDCheck","TypesEqual","HighPrecisionAccumulate","Activation","ActivationComputeType","ActivationNoGuard","UseGradient","UseBias","UseE","UseGateResidual","DataTypeE","StridedBatched","GroupedGemm","UseScaleAB","UseScaleCD","UseScaleAlphaVec","Sparse","F32XdlMathOp","SupportDeviceUserArguments","SwizzleTensorA","SwizzleTensorB","FusedGemmA2A","MXBlockA","MXBlockB","SynchronizerSizeCheck","WorkgroupNumberCheck","LeadingFree0SizesGreaterOrEqual","LeadingFree1SizesGreaterOrEqual","KernelLanguageCompatible","BufferLoadOffsetLimitCheck","BufferLoadOffsetLimitCheck_Beta","BufferStoreOffsetLimitCheck","GlobalSplitUCheckMinK","WorkgroupMappingXCCCheck"] +taskPredicate=["And","WorkspaceCheck","LaunchLimits"] +hardwarePredicate=["AMDGPU","Processor"] +customKernel.name="Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA942_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1" +customKernel.args=[{"semantic":"GemmInfo","type":"uint32"},{"semantic":"InternalArgs","type":"uint32"},{"semantic":"InternalArgs1","type":"uint32"},{"semantic":"NumWorkGroups","type":"uint32"},{"semantic":"SizeFree0","type":"uint32"},{"semantic":"SizeFree1","type":"uint32"},{"semantic":"SizeFree2","type":"uint32"},{"semantic":"SizeSum","type":"uint32"},{"semantic":"AddressD","type":"address"},{"semantic":"AddressC","type":"address"},{"semantic":"AddressA","type":"address"},{"semantic":"AddressB","type":"address"},{"semantic":"StrideD0","type":"uint32"},{"semantic":"StrideD1","type":"uint32"},{"semantic":"StrideC0","type":"uint32"},{"semantic":"StrideC1","type":"uint32"},{"semantic":"StrideA0","type":"uint32"},{"semantic":"StrideA1","type":"uint32"},{"semantic":"StrideB0","type":"uint32"},{"semantic":"StrideB1","type":"uint32"},{"semantic":"Alpha","type":"uint32"},{"semantic":"Beta","type":"uint32"},{"semantic":"AddressTD","type":"address"},{"semantic":"Synchronizer","type":"address"},{"semantic":"GSUSync","type":"uint32"}] +customKernel.macrotile=[16,16,16] +customKernel.threads=[64,1,1] +customKernel.grid=["TilesXYBatchGSU","One","One"] +customKernel.workspaceType="None" +customKernel.workspaceSizePerElemC=0 +customKernel.workspaceSizePerElemBias=0 +customKernel.generated=true +customKernel.uniformSummationOrder=false diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.yaml new file mode 100644 index 000000000000..7b1f72793c6d --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-plain.yaml @@ -0,0 +1,568 @@ +1LDSBuffer: 0 +ActivationAlt: false +ActivationFuncCall: false +ActivationFused: true +AdaptiveGemm: 0 +AdaptiveGemmGSUA: 0 +AdaptiveGemmNTAB: 0 +AssertAIGreaterThanEqual: -1 +AssertAILessThanEqual: -1 +AssertFree0ElementMultiple: 1 +AssertFree1ElementMultiple: 1 +AssertSummationElementMultiple: 1 +AssignedDerivedParameters: true +AssignedProblemIndependentDerivedParameters: true +BaseName: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LD8FToh8dUfkUq7xkODH4qErpo3OUHGD4h3KSmJd4Hjb8= +BufferLoad: true +BufferStore: true +CUOccupancy: 5 +ClusterBarrier: false +ClusterDim: +- 1 +- 1 +ClusterLocalRead: 1 +CodeObjectVersion: '4' +CompactLoopStore: false +ConvertAfterDS: false +CustomKernel: + args: + - semantic: GemmInfo + type: uint32 + - semantic: InternalArgs + type: uint32 + - semantic: InternalArgs1 + type: uint32 + - semantic: NumWorkGroups + type: uint32 + - semantic: SizeFree0 + type: uint32 + - semantic: SizeFree1 + type: uint32 + - semantic: SizeFree2 + type: uint32 + - semantic: SizeSum + type: uint32 + - semantic: AddressD + type: address + - semantic: AddressC + type: address + - semantic: AddressA + type: address + - semantic: AddressB + type: address + - semantic: StrideD0 + type: uint32 + - semantic: StrideD1 + type: uint32 + - semantic: StrideC0 + type: uint32 + - semantic: StrideC1 + type: uint32 + - semantic: StrideA0 + type: uint32 + - semantic: StrideA1 + type: uint32 + - semantic: StrideB0 + type: uint32 + - semantic: StrideB1 + type: uint32 + - semantic: Alpha + type: uint32 + - semantic: Beta + type: uint32 + - semantic: AddressTD + type: address + - semantic: Synchronizer + type: address + - semantic: GSUSync + type: uint32 + generated: true + grid: + - TilesXYBatchGSU + - One + - One + macrotile: + - 16 + - 16 + - 16 + name: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA942_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1 + threads: + - 64 + - 1 + - 1 + workspaceSizePerElemBias: 0 + workspaceSizePerElemC: 0 + workspaceType: None +DebugPersistentKernelLoopForever: false +DebugStreamK: 0 +DepthU: 16 +DirectToLds: 0 +DirectToLdsA: false +DirectToLdsB: false +DirectToLdsMetadata: 0 +DirectToVgprA: false +DirectToVgprB: false +DirectToVgprMXSA: false +DirectToVgprMXSB: false +DirectToVgprSparseMetadata: false +DisableXdlArbStall: true +DtlPlusLdsBuf: 0 +ESMRuntimeGate: false +EdgeType: ShiftPtr +EnableESM2TrackValuVsrc: true +EnableF32XdlMathOp: false +EnableMatrixInstruction: true +EnableStinkyTofuESM2: false +ExpandPointerSwap: false +ExpertSchedulingMode: 0 +ExtraLatencyForLR: 0 +ExtraMiLatencyLeft: -1 +ForceDisableShadowInit: false +ForceUnrollSubIter: false +GlobalReadPerMfma: 1.0 +GlobalReadVectorWidthA: 4 +GlobalReadVectorWidthB: 4 +GlobalSplitU: 1 +GlobalSplitUAlgorithm: MultipleBufferSingleKernel +GlobalSplitUCoalesced: false +GlobalSplitUWorkGroupMappingRoundRobin: false +GlobalWriteVectorWidth: 1 +GroupLoadStore: false +GuaranteeNoPartialA: false +GuaranteeNoPartialB: true +GuaranteeNoPartialMetadata: true +HalfPLR: 0 +HalfPLRA: false +HalfPLRB: false +ISA: +- 9 +- 4 +- 2 +InitCIterWmma: 0 +InnerUnroll: 1 +InterleaveAlpha: 0 +InternalSupportParams: + KernArgsVersion: 3 + PersistentLoopArgsVersion: 0 + SupportCustomStaggerU: true + SupportCustomWGM: true + SupportStreamKPerTileExtraIters: false + SupportUserGSU: true + UseSFC: false + UseUniversalArgs: true +Kernel: true +KernelLanguage: Assembly +KernelNameMin: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA942_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1 +LDSSegInterleaveOffsets: null +LDSSegmentInterleave: 0 +LDSTrInst: false +LSCA: 16 +LSCB: 16 +LSPA: 16 +LSPB: 16 +LVCA: 4 +LVCB: 4 +LVPA: 4 +LVPB: 4 +LdsBlockSizePerPadA: 128 +LdsBlockSizePerPadB: 128 +LdsBlockSizePerPadMXSA: 0 +LdsBlockSizePerPadMXSB: 0 +LdsBlockSizePerPadMetadata: 0 +LdsBytesNoAmax: 1280 +LdsInitCVgprs: false +LdsNumBytes: 1280 +LdsNumElementsAlignedA: 640 +LdsNumElementsAlignedB: 640 +LdsNumElementsAlignedMXSA: 0 +LdsNumElementsAlignedMXSB: 0 +LdsNumElementsAlignedMetadata: 0 +LdsOffsetA: 0 +LdsOffsetA_Blk: 0 +LdsOffsetB: 640 +LdsOffsetB_Blk: 0 +LdsOffsetBias: 0 +LdsOffsetBiasGSU: 0 +LdsOffsetBiasNonGSU: 0 +LdsOffsetMXSA: 640 +LdsOffsetMXSA_Blk: 0 +LdsOffsetMXSB: 640 +LdsOffsetMXSB_Blk: 0 +LdsOffsetMetadata: 640 +LdsOffsetMetadata_Blk: 0 +LdsPadA: 16 +LdsPadB: 16 +LdsPadMXSA: 0 +LdsPadMXSB: 0 +LdsPadMetadata: -1 +LocalReadVectorWidth: -1 +LocalReadVectorWidthA: 4 +LocalReadVectorWidthB: 4 +LocalSplitU: 1 +LocalSplitUReuseLDS: 1 +LocalWritePerMfma: -1 +LocalWriteUseSgprA: false +LocalWriteUseSgprB: false +LocalWriteUseSgprMetadata: false +LoopIters: 1 +LoopUnroll: 16 +MFMA_BF16_1K: false +MIArchVgpr: false +MIBlock: +- 16 +- 16 +- 16 +- 1 +- 1 +- 1 +MIInputPerThread: 4 +MIInputPerThreadA: 4 +MIInputPerThreadB: 4 +MIInputPerThreadMetadata: 4 +MIOutputVectorWidth: 4 +MIRegPerOut: 1 +MIWaveGroup: +- 1 +- 1 +MIWaveTile: +- 1 +- 1 +MIWaveTileA: 1 +MIWaveTileB: 1 +MIWaveTileMetadata: 0 +MXLoadInst: BufferLoad +MXScaleFormat: NoSwizzle +MacroTile0: 16 +MacroTile1: 16 +MacroTileA: 16 +MacroTileB: 16 +MagicDivAlg: 2 +MathClocksUnrolledLoop: 256 +MatrixInstB: 1 +MatrixInstBM: 1 +MatrixInstBN: 1 +MatrixInstK: 16 +MatrixInstM: 16 +MatrixInstN: 16 +MatrixInstruction: +- 16 +- 16 +- 16 +- 1 +MaxLDS: 65536 +MaxOccupancy: 64 +MbskPrefetchMethod: 0 +MfmaInitCVgprs: false +MinGRIncPerMfma: 1 +Multicast: false +NoLdsWriteCode: false +NoReject: false +NoTailLoop: false +NonDTLTailLoopA: false +NonDTLTailLoopB: false +NonDTLTailLoopMetadata: false +NonTemporal: -1 +NonTemporalA: 0 +NonTemporalB: 0 +NonTemporalC: 0 +NonTemporalD: 0 +NonTemporalE: 0 +NonTemporalGate: 0 +NonTemporalMXSA: 0 +NonTemporalMXSB: 0 +NonTemporalMetadata: 0 +NonTemporalWS: 0 +NonVolatile: -1 +NonVolatileA: 0 +NonVolatileB: 0 +NonVolatileC: 0 +NonVolatileD: 0 +NonVolatileE: 0 +NonVolatileGate: 0 +NonVolatileMXSA: 0 +NonVolatileMXSB: 0 +NonVolatileMetadata: 0 +NonVolatileWS: 0 +NumElementsPerBatchStore: 0 +NumElementsPerThread: 4 +NumGlobalWriteVectorsPerThread: 4 +NumLdsBlk: 2 +NumLoadsA: 1 +NumLoadsB: 1 +NumLoadsCoalescedA: 1 +NumLoadsCoalescedB: 1 +NumLoadsPerpendicularA: 1 +NumLoadsPerpendicularB: 1 +NumThreads: 64 +NumTotalPackedLoadsA: -1 +NumTotalPackedLoadsB: -1 +NumWaveSplitK: 1 +NumWaves: 1 +OptNoLoadLoop: 0 +PackedC0IdxChars: +- I +PackedC0IndicesX: +- 0 +PackedC1IdxChars: +- J +PackedC1IndicesX: +- 1 +PersistentXCCMapping: 0 +PrefetchAcrossPersistent: 0 +PrefetchGL2: 0 +PrefetchGlobalRead: 0 +PrefetchLocalRead: 0 +PreloadKernArgs: false +ProblemType: + Activation: false + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: none + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: false + BiasDataTypeList: + - 0 + - 4 + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 4 + DataTypeA: 4 + DataTypeAmaxD: 0 + DataTypeB: 4 + DataTypeE: 4 + DataTypeMXSA: 22 + DataTypeMXSB: 22 + DestDataType: 4 + F32XdlMathOp: 0 + FusedGemmA2A: false + GateResidualDataTypeList: [] + Gradient: false + GroupedGemm: false + HighPrecisionAccumulate: true + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: + - 0 + - 3 + - 2 + IndexAssignmentsB: + - 3 + - 1 + - 2 + IndexAssignmentsLD: + - 4 + - 5 + - 6 + - 7 + IndexAssignmentsMetadata: + - 3 + - 0 + - 2 + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndexUnrollM: 0 + IndicesBatch: + - 2 + IndicesFree: + - 0 + - 1 + IndicesSummation: + - 3 + MXBlockA: 0 + MXBlockB: 0 + MacDataTypeA: 4 + MacDataTypeB: 4 + MetadataLayout: 0 + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMXSA: [] + MirrorDimsMXSB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SetConstStrideGate: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: false + StridedBatched: true + SupportUserArgs: false + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseBias: 0 + UseE: false + UseGateResidual: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: '' + UseScaleAlphaVec: 0 + UseScaleCD: false + isMixMode: false +ReuseAcrossPersistent: 0 +SFCWGM: +- - 1 + - 1 +- - 1 + - 1 +ScheduleGROverBarrier: 0 +ScheduleGlobalRead: 1 +ScheduleIterAlg: 3 +ScheduleLocalWrite: 1 +SolutionIndex: 0 +SolutionNameMin: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSU1_GSUAMBSK_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA942_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGM8_WGMXCC1_WGMXCCGn1 +SourceSwap: false +SpaceFillingAlgo: [] +StaggerU: 32 +StaggerUMapping: 0 +StaggerUStride: 256 +StorePriorityOpt: false +StoreRemapVectorWidth: 0 +StoreSwapAddr: false +StoreSyncOpt: 0 +StoreVectorWidth: 4 +StreamKAtomic: 0 +StreamKFixupTreeReduction: 0 +SubGroup0: 4 +SubGroup1: 16 +SubGroupA: 4 +SubGroupB: 16 +SuppressNoLoadLoop: false +SwInstructionPrefetch: 1 +SwapGlobalReadOrder: 0 +SynchronizerSizeCheck: 1 +TDMInst: 0 +TDMIterateMode: 0 +TDMLoadWaveSync: false +TDMPlusLdsBuf: 0 +TDMSplit: false +TailloopInNll: false +TemporalHint: -1 +TemporalHintA: 0 +TemporalHintB: 0 +TemporalHintC: 0 +TemporalHintD: 0 +TemporalHintE: 0 +TemporalHintGate: 0 +TemporalHintMXSA: 0 +TemporalHintMXSB: 0 +TemporalHintMetadata: 0 +TemporalHintWS: 0 +ThreadTile: +- 1 +- 1 +ThreadTile0: 4 +ThreadTile1: 1 +ThreadTileA: 4 +ThreadTileB: 1 +TileProcessingStrategy: None +TransposeLDS: 0 +TransposeLDSMetadata: -1 +ULSGRODoubleG2L: 0 +UnrollLoopSwapGlobalReadOrder: 0 +UnrollMajorLDSA: 0 +UnrollMajorLDSB: 0 +Use64bShadowLimit: true +Use64bShadowLimitMX: false +UseCustomMainLoopSchedule: 0 +UseDirect32XEmulation: false +UseDot2F32XEmulation: false +UseDotInstruction: false +UseDualFMAC: false +UseF32XEmulation: false +UseGeneralizedNLCOneA: false +UseGeneralizedNLCOneB: false +UseGeneralizedNLCOneMXSA: false +UseGeneralizedNLCOneMXSB: false +UseGeneralizedNLCOneMetadata: false +UseInstOffsetForGRO: 0 +UseMFMAF32XEmulation: false +UsePLRPack: 0 +UseSgprForGRO: -1 +UseSubtileImpl: false +Valid: true +VectorStore: -1 +VectorWidthA: 1 +VectorWidthB: 1 +WaveSeparateGlobalReadA: 0 +WaveSeparateGlobalReadB: 0 +WaveSeparateGlobalReadMetadata: 0 +WaveSplitK: false +WavefrontSize: 64 +WorkAssignment: StaticGrid +WorkGroup: +- 16 +- 4 +- 1 +WorkGroupMapping: 8 +WorkGroupMappingXCC: 1 +WorkGroupMappingXCCGroup: -1 +WorkGroupReduction: false +WorkQueueStealing: 0 +WorkspaceCheck: +- 4 +- 0 +- 1 +_DepthU: 16 +_DepthUA: 16 +_DepthUB: 16 +_DepthUMetadata: 16 +_GlobalAccumulation: MultipleBufferSingleKernel +_HasTemporalHint: 0 +_PersistentLoop: false +_PersistentVectorEpilogueLds: false +_PrefetchAcrossPersistentEnabled: false +_ScheduleIterAlg: 3 +_SeparateEpilogueLds: false +_StinkyTofuOptLevel: 0 +_UseSgprForGRO: false +_VectorStore: 1 +_WorkspaceSizePerElemBias: 0 +_WorkspaceSizePerElemC: 4 +_inputVWA_was_auto: true +_inputVWB_was_auto: true +_staggerStrideShift: 3 +enableGLTrA: false +enableGLTrB: false +enableLDSTrA: false +enableLDSTrB: false +enableLDSTrMXSA: false +enableLDSTrMXSB: false +enableLDSTrMetadata: 0 +enableTDMA: false +enableTDMB: false +enableTDMMetadata: false +ldsNumBytesA: 640 +ldsNumBytesB: 640 +ldsNumBytesMXSA: 0 +ldsNumBytesMXSB: 0 +ldsNumBytesMetadata: 0 +numSubTiles: 1 +reorderGRInstForDTVA: false +reorderGRInstForDTVB: false +tailLoopOptA: true +tailLoopOptB: true diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.msgpack new file mode 100644 index 000000000000..5fd872680a09 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.predicates.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.predicates.msgpack new file mode 100644 index 000000000000..1ddfc5ddd883 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.predicates.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.predicates.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.predicates.yaml new file mode 100644 index 000000000000..3700a51a627b --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.predicates.yaml @@ -0,0 +1,116 @@ +problem: + type: And + value: + - type: OperationIdentifierEqual + value: Contraction_l_Ailk_Bljk_Cijk_Dijk + - type: BiasDataTypeWhiteList + value: + - 0 + - 4 + - type: GateResidualDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: + - 3 + - type: AmaxDCheck + value: false + - type: ActivationEnumWhiteList + value: + - None + - Gelu + - Relu + - Sigmoid + - Silu + - Clamp + - type: TypesEqual + value: + - Half + - Half + - Half + - Half + - Float8_fnuz + - Float8_fnuz + - type: HighPrecisionAccumulate + value: true + - type: Activation + value: Hipblaslt_all + - type: ActivationComputeType + value: Float + - type: ActivationNoGuard + value: false + - type: UseGradient + value: false + - type: UseBias + value: 1 + - type: UseE + value: false + - type: UseGateResidual + value: false + - type: DataTypeE + value: Half + - type: StridedBatched + value: true + - type: GroupedGemm + value: false + - type: UseScaleAB + value: '' + - type: UseScaleCD + value: false + - type: UseScaleAlphaVec + value: 1 + - type: Sparse + value: 0 + - type: F32XdlMathOp + value: Float + - type: SupportDeviceUserArguments + value: true + - type: SwizzleTensorA + value: false + - type: SwizzleTensorB + value: false + - type: FusedGemmA2A + value: false + - type: MXBlockA + value: 0 + - type: MXBlockB + value: 0 + - type: WorkgroupNumberCheck + index: 0 + value: + - 16 + - 16 + - 1 + - type: LeadingFree0SizesGreaterOrEqual + value: 4 + - type: LeadingFree1SizesGreaterOrEqual + value: 1 + - type: KernelLanguageCompatible + value: Assembly + - type: BufferLoadOffsetLimitCheck + value: + ShiftPtrElemB: 0 + ShiftPtrElemA: 4 + DUorMT1: 16 + DUorMT0: 32 + - type: BufferLoadOffsetLimitCheck_Beta + value: 16 + - type: BufferStoreOffsetLimitCheck + value: 16 + - type: GlobalSplitUCheckMinK + value: + - 32 + - 1 + - type: WorkgroupMappingXCCCheck + value: + - 8 + - -1 +task: + type: And + value: + - type: WorkspaceCheck + - type: LaunchLimits +hardware: + type: AMDGPU + value: + type: Processor + value: gfx942 diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.record.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.record.txt new file mode 100644 index 000000000000..492e5ae4cb5c --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.record.txt @@ -0,0 +1,129 @@ +solutionName="Cijk_Ailk_Bljk_HH_F8NHS_BH_BiasSH_HAS_SAV_UserArgs_MT16x16x32_MI16x16x1_SN_GRVWA4_GRVWB8_GSU1_MIWT1_1_SU32_SUM1_SUS256_WGM8" +kernelName="Cijk_Ailk_Bljk_HH_F8NHS_BH_BiasSH_HAS_SAV_UserArgs_MT16x16x32_MI16x16x1_SN_GRVWA4_GRVWB8_MIWT1_1" +sizeMapping.waveNum=1 +sizeMapping.WavefrontSize=64 +sizeMapping.workGroup=[16,4,1] +sizeMapping.macroTile=[16,16,1] +sizeMapping.threadTile=[1,1] +sizeMapping.matrixInstruction=[16,16,32,1] +sizeMapping.grvwA=4 +sizeMapping.grvwB=8 +sizeMapping.gwvwC=4 +sizeMapping.gwvwD=4 +sizeMapping.depthU=32 +sizeMapping.staggerU=32 +sizeMapping.staggerUMapping=1 +sizeMapping.globalSplitUPGR=16 +sizeMapping.globalSplitU=1 +sizeMapping.staggerStrideShift=3 +sizeMapping.workGroupMapping=8 +sizeMapping.packBatchDims=0 +sizeMapping.magicDivAlg=2 +sizeMapping.tileProcessingStrategy="None" +sizeMapping.workAssignment="StaticGrid" +sizeMapping.streamKAtomic=0 +sizeMapping.prefetchAcrossPersistent=0 +sizeMapping.sourceKernel=false +sizeMapping.globalAccumulation=2 +sizeMapping.adaptiveGemmGSUA=0 +sizeMapping.workspaceSizePerElemC=4 +sizeMapping.workspaceSizePerElemBias=0 +sizeMapping.activationFused=true +sizeMapping.workGroupMappingXCC=8 +sizeMapping.workGroupMappingXCCGroup=-1 +sizeMapping.globalSplitUCoalesced=false +sizeMapping.globalSplitUWorkGroupMappingRoundRobin=false +sizeMapping.CUOccupancy=-1 +sizeMapping.PrefetchGlobalRead=1 +sizeMapping.MathClocksUnrolledLoop=0 +sizeMapping.synchronizerSizePerWG=4 +sizeMapping.nonTemporalA=0 +sizeMapping.nonTemporalB=0 +sizeMapping.temporalHintA=0 +sizeMapping.temporalHintB=0 +sizeMapping.hasTemporalHint=false +sizeMapping.adaptiveGemmNTAB=0 +sizeMapping.customMainLoopScheduling=0 +sizeMapping.useSubtileImpl=false +sizeMapping.NonTemporalD=4 +sizeMapping.WaveSeparateGlobalReadA=0 +sizeMapping.WaveSeparateGlobalReadB=0 +sizeMapping.UnrollLoopSwapGlobalReadOrder=0 +sizeMapping.DirectToVgprA=false +sizeMapping.DirectToVgprB=false +sizeMapping.NumLoadsCoalescedA=1 +sizeMapping.NumLoadsCoalescedB=1 +sizeMapping.WaveGroup=[1,1] +sizeMapping.VectorWidthA=1 +sizeMapping.VectorWidthB=1 +sizeMapping.LocalSplitU=1 +sizeMapping.DirectToLdsA=false +sizeMapping.DirectToLdsB=false +sizeMapping.OneLDSBuffer=false +sizeMapping.SourceSwap=false +sizeMapping.UnrollMajorLDSA=0 +sizeMapping.UnrollMajorLDSB=1 +sizeMapping.enableLDSTrA=false +sizeMapping.enableLDSTrB=false +sizeMapping.ExpertSchedulingMode=0 +sizeMapping.clusterDim=[1,1] +sizeMapping.BufferLoad=true +sizeMapping.BufferStore=true +internalArgsSupport.version=3 +internalArgsSupport.persistentLoopArgsVersion=0 +internalArgsSupport.gsu=true +internalArgsSupport.wgm=true +internalArgsSupport.staggerU=true +internalArgsSupport.perTileExtraIters=false +internalArgsSupport.useUniversalArgs=true +internalArgsSupport.useSFC=false +problemType.operationIdentifier="Contraction_l_Ailk_Bljk_Cijk_Dijk" +problemType.transA=false +problemType.transB=false +problemType.computeInputTypeA="Float8_fnuz" +problemType.computeInputTypeB="Float8_fnuz" +problemType.aType="Half" +problemType.bType="Half" +problemType.cType="Half" +problemType.dType="Half" +problemType.eType="Half" +problemType.computeType="Float" +problemType.useBeta=true +problemType.useBias=1 +problemType.biasSrcWhiteList=[3] +problemType.useE=false +problemType.useGateResidual=false +problemType.gateResidualDataTypeWhiteList=[] +problemType.useScaleAB="" +problemType.useScaleCD=false +problemType.useScaleAlphaVec=1 +problemType.biasDataTypeWhiteList=[0,4] +problemType.highPrecisionAccumulate=true +problemType.useInitialStridesAB=false +problemType.useInitialStridesCD=false +problemType.stridedBatched=true +problemType.groupedGemm=false +problemType.useGradient=false +problemType.activationType="Hipblaslt_all" +problemType.activationArgLength=2 +problemType.activationComputeDataType="Float" +problemType.activationNoGuard=false +problemType.sparse=0 +problemType.f32XdlMathOp="Float" +problemType.supportDeviceUserArguments=true +problemType.outputAmaxD=false +problemType.swizzleTensorA=false +problemType.swizzleTensorB=false +problemType.metadataLayout=0 +problemType.mxBlockA=0 +problemType.mxBlockB=0 +problemType.mxBlockFreeA=1 +problemType.mxBlockFreeB=1 +problemType.mxTypeA="E8" +problemType.mxTypeB="E8" +problemType.mxScaleFormat=0 +problemType.fusedGemmA2A=false +problemType.stochasticRounding=true +problemPredicate=["And","OperationIdentifierEqual","BiasDataTypeWhiteList","GateResidualDataTypeWhiteList","BiasSrcWhiteList","AmaxDCheck","ActivationEnumWhiteList","TypesEqual","HighPrecisionAccumulate","Activation","ActivationComputeType","ActivationNoGuard","UseGradient","UseBias","UseE","UseGateResidual","DataTypeE","StridedBatched","GroupedGemm","UseScaleAB","UseScaleCD","UseScaleAlphaVec","Sparse","F32XdlMathOp","SupportDeviceUserArguments","SwizzleTensorA","SwizzleTensorB","FusedGemmA2A","MXBlockA","MXBlockB","WorkgroupNumberCheck","LeadingFree0SizesGreaterOrEqual","LeadingFree1SizesGreaterOrEqual","KernelLanguageCompatible","BufferLoadOffsetLimitCheck","BufferLoadOffsetLimitCheck_Beta","BufferStoreOffsetLimitCheck","GlobalSplitUCheckMinK","WorkgroupMappingXCCCheck"] +taskPredicate=["And","WorkspaceCheck","LaunchLimits"] +hardwarePredicate=["AMDGPU","Processor"] diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.yaml new file mode 100644 index 000000000000..637978e15145 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-stochastic-rounding.yaml @@ -0,0 +1,519 @@ +1LDSBuffer: 0 +ActivationAlt: false +ActivationFuncCall: true +ActivationFused: true +AdaptiveGemm: 0 +AdaptiveGemmGSUA: 0 +AdaptiveGemmNTAB: 0 +AssertAIGreaterThanEqual: -1 +AssertAILessThanEqual: -1 +AssertFree0ElementMultiple: 1 +AssertFree1ElementMultiple: 1 +AssertSummationElementMultiple: 1 +AssignedDerivedParameters: true +AssignedProblemIndependentDerivedParameters: true +BufferLoad: true +BufferStore: true +CUCount: null +CUOccupancy: -1 +ClusterAIDMapping: -1 +ClusterBarrier: false +ClusterDim: +- 1 +- 1 +ClusterLocalRead: 0 +CodeObjectVersion: default +CompactLoopStore: false +ConvertAfterDS: false +CustomKernelName: '' +DebugPersistentKernelLoopForever: false +DebugStreamK: 0 +DepthU: 32 +DeviceNames: +- Device 0049 +- Device 0050 +DirectToLds: 0 +DirectToLdsA: false +DirectToLdsB: false +DirectToLdsMetadata: 0 +DirectToVgprA: false +DirectToVgprB: false +DirectToVgprMXSA: false +DirectToVgprMXSB: false +DirectToVgprSparseMetadata: false +DisableXdlArbStall: true +DtlPlusLdsBuf: 0 +ESMRuntimeGate: false +EdgeType: ShiftPtr +EnableESM2TrackValuVsrc: true +EnableF32XdlMathOp: false +EnableMatrixInstruction: true +EnableStinkyTofuESM2: false +ExpandPointerSwap: true +ExpertSchedulingMode: 0 +ExtraLatencyForLR: 0 +ExtraMiLatencyLeft: -1 +ForceDisableShadowInit: false +ForceUnrollSubIter: false +GlobalReadPerMfma: 1.0 +GlobalReadVectorWidthA: 4 +GlobalReadVectorWidthB: 8 +GlobalSplitU: 1 +GlobalSplitUAlgorithm: MultipleBuffer +GlobalSplitUCoalesced: false +GlobalSplitUWorkGroupMappingRoundRobin: false +GlobalWriteVectorWidth: 1 +GroupLoadStore: false +GuaranteeNoPartialA: false +GuaranteeNoPartialB: true +GuaranteeNoPartialMetadata: true +HalfPLR: 0 +HalfPLRA: false +HalfPLRB: false +ISA: +- 9 +- 4 +- 2 +InitCIterWmma: 0 +InnerUnroll: 1 +InterleaveAlpha: 0 +InternalSupportParams: + KernArgsVersion: 3 + PersistentLoopArgsVersion: 0 + SupportCustomStaggerU: true + SupportCustomWGM: true + SupportOccupancyProbe: false + SupportStreamKPerTileExtraIters: false + SupportUserGSU: true + UseSFC: false + UseUniversalArgs: true +KernelLanguage: Assembly +KernelNameMin: Cijk_Ailk_Bljk_HH_F8NHS_BH_BiasSH_HAS_SAV_UserArgs_MT16x16x32_MI16x16x1_SN_GRVWA4_GRVWB8_MIWT1_1 +LDSSegInterleaveOffsets: {} +LDSSegmentBRotate: 0 +LDSSegmentInterleave: 0 +LDSTrInst: false +LSCA: 16 +LSCB: 32 +LSPA: 16 +LSPB: 16 +LVCA: 4 +LVCB: 4 +LVPA: 4 +LVPB: 2 +LdsBlockSizePerPadA: 128 +LdsBlockSizePerPadB: 128 +LdsBlockSizePerPadMXSA: 0 +LdsBlockSizePerPadMXSB: 0 +LdsBlockSizePerPadMetadata: 0 +LdsBytesNoAmax: 3360 +LdsInitCVgprs: false +LdsNumBytes: 3360 +LdsNumElements: 3360 +LdsNumElementsAlignedA: 768 +LdsNumElementsAlignedB: 768 +LdsNumElementsAlignedMXSA: 0 +LdsNumElementsAlignedMXSB: 0 +LdsNumElementsAlignedMetadata: 0 +LdsOffsetA: 0 +LdsOffsetA_Blk: 2048 +LdsOffsetB: 768 +LdsOffsetB_Blk: 2816 +LdsOffsetBias: 0 +LdsOffsetBiasGSU: 0 +LdsOffsetBiasNonGSU: 0 +LdsOffsetMXSA: 768 +LdsOffsetMXSA_Blk: 2816 +LdsOffsetMXSB: 768 +LdsOffsetMXSB_Blk: 2816 +LdsOffsetMetadata: 768 +LdsOffsetMetadata_Blk: 2816 +LdsPadA: 16 +LdsPadB: 8 +LdsPadMXSA: 0 +LdsPadMXSB: 0 +LdsPadMetadata: -1 +LocalReadVectorWidth: 8 +LocalReadVectorWidthA: 8 +LocalReadVectorWidthB: 8 +LocalSplitU: 1 +LocalSplitUReuseLDS: 1 +LocalWritePerMfma: -1 +LocalWriteUseSgprA: false +LocalWriteUseSgprB: false +LocalWriteUseSgprMetadata: false +LoopIters: 1 +LoopUnroll: 32 +MFMA_BF16_1K: false +MIArchVgpr: false +MIBlock: +- 16 +- 16 +- 32 +- 1 +- 1 +- 1 +MIInputPerThread: 8 +MIInputPerThreadA: 8 +MIInputPerThreadB: 8 +MIInputPerThreadMetadata: 8 +MIOutputVectorWidth: 4 +MIRegPerOut: 1 +MIWaveGroup: +- 1 +- 1 +MIWaveTile: +- 1 +- 1 +MIWaveTileA: 1 +MIWaveTileB: 1 +MIWaveTileMetadata: 0 +MXLoadInst: BufferLoad +MXScaleFormat: NoSwizzle +MacroTile0: 16 +MacroTile1: 16 +MacroTileA: 16 +MacroTileB: 16 +MagicDivAlg: 2 +MathClocksUnrolledLoop: 0 +MatrixInstB: 1 +MatrixInstBM: 1 +MatrixInstBN: 1 +MatrixInstK: 32 +MatrixInstM: 16 +MatrixInstN: 16 +MatrixInstruction: +- 16 +- 16 +- 32 +- 1 +MaxLDS: 65536 +MaxOccupancy: 40 +MbskPrefetchMethod: 0 +MfmaInitCVgprs: false +MinGRIncPerMfma: 1 +Multicast: false +NoLdsWriteCode: false +NoReject: false +NoTailLoop: false +NonDTLTailLoopA: false +NonDTLTailLoopB: false +NonDTLTailLoopMetadata: false +NonTemporal: -1 +NonTemporalA: 0 +NonTemporalB: 0 +NonTemporalC: 0 +NonTemporalD: 4 +NonTemporalE: 0 +NonTemporalGate: 0 +NonTemporalMXSA: 0 +NonTemporalMXSB: 0 +NonTemporalMetadata: 0 +NonTemporalWS: 0 +NonVolatile: -1 +NonVolatileA: 0 +NonVolatileB: 0 +NonVolatileC: 0 +NonVolatileD: 0 +NonVolatileE: 0 +NonVolatileGate: 0 +NonVolatileMXSA: 0 +NonVolatileMXSB: 0 +NonVolatileMetadata: 0 +NonVolatileWS: 0 +NumElementsPerBatchStore: 0 +NumElementsPerThread: 4 +NumGlobalWriteVectorsPerThread: 4 +NumLdsBlk: 2 +NumLoadsA: 2 +NumLoadsB: 1 +NumLoadsCoalescedA: 1 +NumLoadsCoalescedB: 1 +NumLoadsPerpendicularA: 2 +NumLoadsPerpendicularB: 1 +NumThreads: 64 +NumTotalPackedLoadsA: -1 +NumTotalPackedLoadsB: -1 +NumWaveSplitK: 1 +NumWaves: 1 +OptNoLoadLoop: 1 +PackedC0IdxChars: +- I +PackedC0IndicesX: +- 0 +PackedC1IdxChars: +- J +PackedC1IndicesX: +- 1 +PersistentXCCMapping: 0 +PostLoopStoreInNll: false +PrefetchAcrossPersistent: 0 +PrefetchGL2: 0 +PrefetchGlobalRead: 1 +PrefetchLocalRead: 0 +PreloadKernArgs: true +ProblemType: + Activation: true + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: hipblaslt_all + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: false + BiasDataTypeList: + - 0 + - 4 + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 11 + DataTypeA: 4 + DataTypeAmaxD: 0 + DataTypeB: 4 + DataTypeE: 4 + DataTypeMXSA: 22 + DataTypeMXSB: 22 + DestDataType: 4 + F32XdlMathOp: 0 + FusedGemmA2A: false + GateResidualDataTypeList: [] + Gradient: false + GroupedGemm: false + HighPrecisionAccumulate: true + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: + - 0 + - 3 + - 2 + IndexAssignmentsB: + - 3 + - 1 + - 2 + IndexAssignmentsLD: + - 4 + - 5 + - 6 + - 7 + IndexAssignmentsMetadata: + - 3 + - 0 + - 2 + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndexUnrollM: 0 + IndicesBatch: + - 2 + IndicesFree: + - 0 + - 1 + IndicesSummation: + - 3 + Int4EncodingA: Signed + MXBlockA: 0 + MXBlockB: 0 + MXBlockFreeA: 1 + MXBlockFreeB: 1 + MacDataTypeA: 11 + MacDataTypeB: 11 + MetadataLayout: 0 + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMXSA: [] + MirrorDimsMXSB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + ScaleBlockSizesA: [] + ScaleZeroPointA: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SetConstStrideGate: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: true + StridedBatched: true + SupportUserArgs: true + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseBias: 1 + UseE: false + UseGateResidual: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: '' + UseScaleAlphaVec: 1 + UseScaleCD: false + isMixMode: false +ReuseAcrossPersistent: 0 +SFCWGM: +- - 1 + - 1 +- - 1 + - 1 +ScheduleGROverBarrier: 0 +ScheduleGlobalRead: 1 +ScheduleIterAlg: 3 +ScheduleLocalWrite: 1 +SolutionIndex: 5 +SolutionNameMin: Cijk_Ailk_Bljk_HH_F8NHS_BH_BiasSH_HAS_SAV_UserArgs_MT16x16x32_MI16x16x1_SN_GRVWA4_GRVWB8_GSU1_MIWT1_1_SU32_SUM1_SUS256_WGM8 +SolutionUID: 0uR2w0CUPgbG +SourceSwap: false +SpaceFillingAlgo: [] +StaggerU: 32 +StaggerUMapping: 1 +StaggerUStride: 256 +StinkyTofuRegisterAllocation: 0 +StorePriorityOpt: false +StoreRemapVectorWidth: 0 +StoreSwapAddr: false +StoreSyncOpt: 0 +StoreVectorWidth: 4 +StreamKAtomic: 0 +StreamKFixupTreeReduction: 0 +SubGroup0: 4 +SubGroup1: 16 +SubGroupA: 4 +SubGroupB: 16 +SuppressNoLoadLoop: false +SwInstructionPrefetch: 1 +SwapGlobalReadOrder: 0 +TDMInst: 0 +TDMIterateMode: 0 +TDMLoadWaveSync: false +TDMPlusLdsBuf: 0 +TDMSplit: false +TailloopInNll: false +TemporalHint: -1 +TemporalHintA: 0 +TemporalHintB: 0 +TemporalHintC: 0 +TemporalHintD: 0 +TemporalHintE: 0 +TemporalHintGate: 0 +TemporalHintMXSA: 0 +TemporalHintMXSB: 0 +TemporalHintMetadata: 0 +TemporalHintWS: 0 +ThreadTile: +- 1 +- 1 +ThreadTile0: 4 +ThreadTile1: 1 +ThreadTileA: 4 +ThreadTileB: 1 +TileProcessingStrategy: None +TransposeLDS: 1 +TransposeLDSMetadata: 1 +ULSGRODoubleG2L: 0 +UnrollLoopSwapGlobalReadOrder: 0 +UnrollMajorLDSA: false +UnrollMajorLDSB: true +UnrollMajorLDSMetadata: true +Use64bShadowLimit: true +Use64bShadowLimitMX: false +UseCustomMainLoopSchedule: 0 +UseDirect32XEmulation: false +UseDot2F32XEmulation: false +UseDotInstruction: false +UseDualFMAC: false +UseF32XEmulation: false +UseGeneralizedNLCOneA: false +UseGeneralizedNLCOneB: false +UseGeneralizedNLCOneMXSA: false +UseGeneralizedNLCOneMXSB: false +UseGeneralizedNLCOneMetadata: false +UseInstOffsetForGRO: 0 +UseMFMAF32XEmulation: false +UsePLRPack: 0 +UseSgprForGRO: -1 +UseSubtileImpl: false +Valid: true +VectorStore: -1 +VectorWidthA: 1 +VectorWidthB: 1 +WaveSeparateGlobalReadA: 0 +WaveSeparateGlobalReadB: 0 +WaveSeparateGlobalReadMetadata: 0 +WaveSplitK: false +WavefrontSize: 64 +WorkAssignment: StaticGrid +WorkGroup: +- 16 +- 4 +- 1 +WorkGroupMapping: 8 +WorkGroupMappingXCC: 8 +WorkGroupMappingXCCGroup: -1 +WorkGroupReduction: false +WorkQueueStealing: 0 +WorkspaceCheck: +- 4 +- 0 +- 1 +_DepthU: 32 +_DepthUA: 32 +_DepthUB: 32 +_DepthUMetadata: 32 +_DualPeriodMXFP8: false +_GlobalAccumulation: MultipleBuffer +_HasTemporalHint: 0 +_PersistentLoop: false +_PersistentVectorEpilogueLds: false +_PrefetchAcrossPersistentEnabled: false +_ScalePeriodUA: 32 +_ScalePeriodUB: 32 +_ScaleReuseA: 1 +_ScaleReuseB: 1 +_ScheduleIterAlg: 3 +_SeparateEpilogueLds: false +_StinkyTofuOptLevel: 0 +_UseSgprForGRO: false +_VectorStore: 1 +_WorkspaceSizePerElemBias: 0 +_WorkspaceSizePerElemC: 4 +_inputVWA_was_auto: false +_inputVWB_was_auto: false +_staggerStrideShift: 3 +enableGLTrA: false +enableGLTrB: false +enableLDSTrA: false +enableLDSTrB: false +enableLDSTrMXSA: false +enableLDSTrMXSB: false +enableLDSTrMetadata: 0 +enableTDMA: false +enableTDMB: false +enableTDMMetadata: false +ldsNumBytesA: 576 +ldsNumBytesB: 544 +ldsNumBytesMXSA: 0 +ldsNumBytesMXSB: 0 +ldsNumBytesMetadata: 0 +numSubTiles: 1 +reorderGRInstForDTVA: false +reorderGRInstForDTVB: false +tailLoopOptA: true +tailLoopOptB: true diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.msgpack new file mode 100644 index 000000000000..36459c7b63ee Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.predicates.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.predicates.msgpack new file mode 100644 index 000000000000..9f4419078198 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.predicates.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.predicates.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.predicates.yaml new file mode 100644 index 000000000000..48b45cbe2698 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.predicates.yaml @@ -0,0 +1,93 @@ +problem: + type: And + value: + - type: OperationIdentifierEqual + value: Contraction_l_Alik_Bjlk_Cijk_Dijk + - type: BiasDataTypeWhiteList + value: [] + - type: GateResidualDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: + - 3 + - type: AmaxDCheck + value: false + - type: TypesEqual + value: + - Float8_fnuz + - BFloat8_fnuz + - Float + - Float + - Float8_fnuz + - BFloat8_fnuz + - type: HighPrecisionAccumulate + value: true + - type: Activation + value: None + - type: ActivationComputeType + value: Float + - type: ActivationNoGuard + value: false + - type: UseGradient + value: false + - type: UseBias + value: 0 + - type: UseE + value: false + - type: UseGateResidual + value: false + - type: DataTypeE + value: Float + - type: StridedBatched + value: true + - type: GroupedGemm + value: false + - type: UseScaleAB + value: '' + - type: UseScaleCD + value: false + - type: UseScaleAlphaVec + value: 0 + - type: Sparse + value: 0 + - type: F32XdlMathOp + value: Float + - type: SupportDeviceUserArguments + value: true + - type: SwizzleTensorA + value: false + - type: SwizzleTensorB + value: false + - type: FusedGemmA2A + value: false + - type: MXBlockA + value: 0 + - type: MXBlockB + value: 0 + - type: LeadingFree0SizesGreaterOrEqual + value: 1 + - type: LeadingFree1SizesGreaterOrEqual + value: 4 + - type: KernelLanguageCompatible + value: Assembly + - type: BufferLoadOffsetLimitCheck + value: + ShiftPtrElemB: 4 + ShiftPtrElemA: 0 + DUorMT1: 32 + DUorMT0: 256 + - type: BufferLoadOffsetLimitCheck_Beta + value: 224 + - type: BufferStoreOffsetLimitCheck + value: 224 + - type: WorkgroupMappingXCCCheck + value: + - -1 + - -1 +task: + type: WorkspaceCheck +hardware: + type: AMDGPU + value: + type: Processor + value: gfx942 diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.record.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.record.txt new file mode 100644 index 000000000000..fba10d0ed328 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.record.txt @@ -0,0 +1,129 @@ +solutionName="Cijk_Alik_Bjlk_F8NSS_BH_UserArgs_MT256x224x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS0_FDSI0_GRPM1_GRVWA8_GRVWB4_GSU0_GSUAMB_GSUC0_GSUWGMRR0_GLS0_ISA942_IU1_K1_LBSPPA256_LBSPPB1792_LBSPPM0_LPA8_LPB16_LPM0_LRVW8_LWPMn1_MIAV0_MIWT8_7_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB7_ONLL1_PGR2_PLR0_PKA1_SIA3_SS1_SU0_SUM1_SUS32_SPO0_SRVW0_SSO0_SVW8_SK3_SKXCCM8_TLDS1_ULSGRO0_USL1_UIOFGRO0_USFGRO0_VSn1_VWA8_VWB1_WSGRA0_WSGRB0_WS64_WG32_8_1_WGM6_WGMXCC1_WGMXCCGn1" +kernelName="Cijk_Alik_Bjlk_F8NSS_BH_UserArgs_MT256x224x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS0_FDSI0_GRPM1_GRVWA8_GRVWB4_GSU0_GSUAMB_GLS0_ISA942_IU1_K1_LBSPPA256_LBSPPB1792_LBSPPM0_LPA8_LPB16_LPM0_LRVW8_LWPMn1_MIAV0_MIWT8_7_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB7_ONLL1_PGR2_PLR0_PKA1_SIA3_SS1_SPO0_SRVW0_SSO0_SVW8_SK3_SKXCCM8_TLDS1_ULSGRO0_USL1_UIOFGRO0_USFGRO0_VSn1_VWA8_VWB1_WSGRA0_WSGRB0_WS64_WG32_8_1" +sizeMapping.waveNum=4 +sizeMapping.WavefrontSize=64 +sizeMapping.workGroup=[32,8,1] +sizeMapping.macroTile=[256,224,1] +sizeMapping.threadTile=[1,1] +sizeMapping.matrixInstruction=[16,16,32,1] +sizeMapping.grvwA=8 +sizeMapping.grvwB=4 +sizeMapping.gwvwC=8 +sizeMapping.gwvwD=8 +sizeMapping.depthU=32 +sizeMapping.staggerU=0 +sizeMapping.staggerUMapping=0 +sizeMapping.globalSplitUPGR=16 +sizeMapping.globalSplitU=0 +sizeMapping.staggerStrideShift=0 +sizeMapping.workGroupMapping=0 +sizeMapping.packBatchDims=0 +sizeMapping.magicDivAlg=2 +sizeMapping.tileProcessingStrategy="StreamK" +sizeMapping.workAssignment="StaticGrid" +sizeMapping.streamKAtomic=0 +sizeMapping.prefetchAcrossPersistent=0 +sizeMapping.sourceKernel=false +sizeMapping.globalAccumulation=4 +sizeMapping.adaptiveGemmGSUA=0 +sizeMapping.workspaceSizePerElemC=4 +sizeMapping.workspaceSizePerElemBias=0 +sizeMapping.activationFused=true +sizeMapping.workGroupMappingXCC=-1 +sizeMapping.workGroupMappingXCCGroup=-1 +sizeMapping.globalSplitUCoalesced=false +sizeMapping.globalSplitUWorkGroupMappingRoundRobin=false +sizeMapping.CUOccupancy=-1 +sizeMapping.PrefetchGlobalRead=2 +sizeMapping.MathClocksUnrolledLoop=0 +sizeMapping.synchronizerSizePerWG=50176 +sizeMapping.nonTemporalA=0 +sizeMapping.nonTemporalB=0 +sizeMapping.temporalHintA=0 +sizeMapping.temporalHintB=0 +sizeMapping.hasTemporalHint=false +sizeMapping.adaptiveGemmNTAB=0 +sizeMapping.customMainLoopScheduling=0 +sizeMapping.useSubtileImpl=false +sizeMapping.NonTemporalD=0 +sizeMapping.WaveSeparateGlobalReadA=0 +sizeMapping.WaveSeparateGlobalReadB=0 +sizeMapping.UnrollLoopSwapGlobalReadOrder=0 +sizeMapping.DirectToVgprA=false +sizeMapping.DirectToVgprB=false +sizeMapping.NumLoadsCoalescedA=1 +sizeMapping.NumLoadsCoalescedB=7 +sizeMapping.WaveGroup=[2,2] +sizeMapping.VectorWidthA=8 +sizeMapping.VectorWidthB=1 +sizeMapping.LocalSplitU=1 +sizeMapping.DirectToLdsA=false +sizeMapping.DirectToLdsB=false +sizeMapping.OneLDSBuffer=false +sizeMapping.SourceSwap=true +sizeMapping.UnrollMajorLDSA=1 +sizeMapping.UnrollMajorLDSB=0 +sizeMapping.enableLDSTrA=false +sizeMapping.enableLDSTrB=false +sizeMapping.ExpertSchedulingMode=0 +sizeMapping.clusterDim=[1,1] +sizeMapping.BufferLoad=true +sizeMapping.BufferStore=true +internalArgsSupport.version=3 +internalArgsSupport.persistentLoopArgsVersion=0 +internalArgsSupport.gsu=false +internalArgsSupport.wgm=true +internalArgsSupport.staggerU=true +internalArgsSupport.perTileExtraIters=true +internalArgsSupport.useUniversalArgs=true +internalArgsSupport.useSFC=false +problemType.operationIdentifier="Contraction_l_Alik_Bjlk_Cijk_Dijk" +problemType.transA=true +problemType.transB=true +problemType.computeInputTypeA="Float8_fnuz" +problemType.computeInputTypeB="BFloat8_fnuz" +problemType.aType="Float8_fnuz" +problemType.bType="BFloat8_fnuz" +problemType.cType="Float" +problemType.dType="Float" +problemType.eType="Float" +problemType.computeType="Float" +problemType.useBeta=true +problemType.useBias=0 +problemType.biasSrcWhiteList=[3] +problemType.useE=false +problemType.useGateResidual=false +problemType.gateResidualDataTypeWhiteList=[] +problemType.useScaleAB="" +problemType.useScaleCD=false +problemType.useScaleAlphaVec=0 +problemType.biasDataTypeWhiteList=[] +problemType.highPrecisionAccumulate=true +problemType.useInitialStridesAB=false +problemType.useInitialStridesCD=false +problemType.stridedBatched=true +problemType.groupedGemm=false +problemType.useGradient=false +problemType.activationType="None" +problemType.activationArgLength=0 +problemType.activationComputeDataType="Float" +problemType.activationNoGuard=false +problemType.sparse=0 +problemType.f32XdlMathOp="Float" +problemType.supportDeviceUserArguments=true +problemType.outputAmaxD=false +problemType.swizzleTensorA=false +problemType.swizzleTensorB=false +problemType.metadataLayout=0 +problemType.mxBlockA=0 +problemType.mxBlockB=0 +problemType.mxBlockFreeA=1 +problemType.mxBlockFreeB=1 +problemType.mxTypeA="E8" +problemType.mxTypeB="E8" +problemType.mxScaleFormat=0 +problemType.fusedGemmA2A=false +problemType.stochasticRounding=false +problemPredicate=["And","OperationIdentifierEqual","BiasDataTypeWhiteList","GateResidualDataTypeWhiteList","BiasSrcWhiteList","AmaxDCheck","TypesEqual","HighPrecisionAccumulate","Activation","ActivationComputeType","ActivationNoGuard","UseGradient","UseBias","UseE","UseGateResidual","DataTypeE","StridedBatched","GroupedGemm","UseScaleAB","UseScaleCD","UseScaleAlphaVec","Sparse","F32XdlMathOp","SupportDeviceUserArguments","SwizzleTensorA","SwizzleTensorB","FusedGemmA2A","MXBlockA","MXBlockB","LeadingFree0SizesGreaterOrEqual","LeadingFree1SizesGreaterOrEqual","KernelLanguageCompatible","BufferLoadOffsetLimitCheck","BufferLoadOffsetLimitCheck_Beta","BufferStoreOffsetLimitCheck","WorkgroupMappingXCCCheck"] +taskPredicate=["WorkspaceCheck"] +hardwarePredicate=["AMDGPU","Processor"] diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.yaml new file mode 100644 index 000000000000..377dc300b518 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx942-streamk-fp8.yaml @@ -0,0 +1,520 @@ +1LDSBuffer: 0 +ActivationAlt: false +ActivationFuncCall: false +ActivationFused: true +AdaptiveGemm: 0 +AdaptiveGemmGSUA: 0 +AdaptiveGemmNTAB: 0 +AssertAIGreaterThanEqual: -1 +AssertAILessThanEqual: -1 +AssertFree0ElementMultiple: 1 +AssertFree1ElementMultiple: 1 +AssertSummationElementMultiple: 1 +AssignedDerivedParameters: true +AssignedProblemIndependentDerivedParameters: true +BaseName: Cijk_Alik_Bjlk_F8NSS_BH_UserArgs_MT256x224x32_MI-2Co4AlqSQ3bWHDytvOACWKfyQITh5ROTAj_nxv5gNk= +BufferLoad: true +BufferStore: true +CUCount: null +CUOccupancy: -1 +ClusterAIDMapping: -1 +ClusterBarrier: false +ClusterDim: +- 1 +- 1 +ClusterLocalRead: 0 +CodeObjectVersion: '4' +CompactLoopStore: false +ConvertAfterDS: false +CustomKernelName: '' +DebugPersistentKernelLoopForever: false +DebugStreamK: 0 +DepthU: 32 +DeviceNames: +- Device 0049 +- Device 0050 +DirectToLds: 0 +DirectToLdsA: false +DirectToLdsB: false +DirectToLdsMetadata: 0 +DirectToVgprA: false +DirectToVgprB: false +DirectToVgprMXSA: false +DirectToVgprMXSB: false +DirectToVgprSparseMetadata: false +DisableXdlArbStall: false +DtlPlusLdsBuf: 0 +ESMRuntimeGate: false +EdgeType: ShiftPtr +EnableESM2TrackValuVsrc: true +EnableF32XEmulationLds: false +EnableF32XdlMathOp: false +EnableMatrixInstruction: true +EnableStinkyTofuESM2: false +ExpandPointerSwap: false +ExpertSchedulingMode: 0 +ExtraLatencyForLR: 0 +ExtraMiLatencyLeft: -1 +ForceDisableShadowInit: false +ForceUnrollSubIter: false +GlobalReadPerMfma: 1.0 +GlobalReadVectorWidthA: 8 +GlobalReadVectorWidthB: 4 +GlobalSplitU: 0 +GlobalSplitUAlgorithm: MultipleBuffer +GlobalSplitUCoalesced: false +GlobalSplitUWorkGroupMappingRoundRobin: false +GlobalWriteVectorWidth: 8 +GroupLoadStore: false +GuaranteeNoPartialA: true +GuaranteeNoPartialB: false +GuaranteeNoPartialMetadata: true +HalfPLR: 0 +HalfPLRA: false +HalfPLRB: false +ISA: +- 9 +- 4 +- 2 +InitCIterWmma: 0 +InnerUnroll: 1 +InterleaveAlpha: 0 +InternalSupportParams: + KernArgsVersion: 3 + PersistentLoopArgsVersion: 0 + SupportCustomStaggerU: true + SupportCustomWGM: true + SupportOccupancyProbe: false + SupportStreamKPerTileExtraIters: true + SupportUserGSU: false + UseSFC: false + UseUniversalArgs: true +Kernel: true +KernelLanguage: Assembly +KernelNameMin: Cijk_Alik_Bjlk_F8NSS_BH_UserArgs_MT256x224x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS0_FDSI0_GRPM1_GRVWA8_GRVWB4_GSU0_GSUAMB_GLS0_ISA942_IU1_K1_LBSPPA256_LBSPPB1792_LBSPPM0_LPA8_LPB16_LPM0_LRVW8_LWPMn1_MIAV0_MIWT8_7_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB7_ONLL1_PGR2_PLR0_PKA1_SIA3_SS1_SPO0_SRVW0_SSO0_SVW8_SK3_SKXCCM8_TLDS1_ULSGRO0_USL1_UIOFGRO0_USFGRO0_VSn1_VWA8_VWB1_WSGRA0_WSGRB0_WS64_WG32_8_1 +LDSSegInterleaveOffsets: {} +LDSSegmentBRotate: 0 +LDSSegmentInterleave: 0 +LDSTrInst: false +LSCA: 32 +LSCB: 32 +LSPA: 64 +LSPB: 32 +LVCA: 4 +LVCB: 8 +LVPA: 8 +LVPB: 8 +LdsBlockSizePerPadA: 256 +LdsBlockSizePerPadB: 1792 +LdsBlockSizePerPadMXSA: 0 +LdsBlockSizePerPadMXSB: 0 +LdsBlockSizePerPadMetadata: 0 +LdsBytesNoAmax: 32064 +LdsInitCVgprs: false +LdsNumBytes: 32064 +LdsNumElementsAlignedA: 8448 +LdsNumElementsAlignedB: 7424 +LdsNumElementsAlignedMXSA: 0 +LdsNumElementsAlignedMXSB: 0 +LdsNumElementsAlignedMetadata: 0 +LdsOffsetA: 0 +LdsOffsetA_Blk: 16384 +LdsOffsetB: 8448 +LdsOffsetB_Blk: 24832 +LdsOffsetBias: 0 +LdsOffsetBiasGSU: 0 +LdsOffsetBiasNonGSU: 0 +LdsOffsetMXSA: 8448 +LdsOffsetMXSA_Blk: 24832 +LdsOffsetMXSB: 8448 +LdsOffsetMXSB_Blk: 24832 +LdsOffsetMetadata: 8448 +LdsOffsetMetadata_Blk: 24832 +LdsPadA: 8 +LdsPadB: 16 +LdsPadMXSA: 0 +LdsPadMXSB: 0 +LdsPadMetadata: 0 +LocalReadVectorWidth: 8 +LocalReadVectorWidthA: 8 +LocalReadVectorWidthB: 8 +LocalSplitU: 1 +LocalSplitUReuseLDS: 1 +LocalWritePerMfma: -1 +LocalWriteUseSgprA: false +LocalWriteUseSgprB: false +LocalWriteUseSgprMetadata: false +LoopIters: 1 +LoopUnroll: 32 +MFMA_BF16_1K: false +MIArchVgpr: false +MIBlock: +- 16 +- 16 +- 32 +- 1 +- 1 +- 1 +MIInputPerThread: 8 +MIInputPerThreadA: 8 +MIInputPerThreadB: 8 +MIInputPerThreadMetadata: 8 +MIOutputVectorWidth: 4 +MIRegPerOut: 1 +MIWaveGroup: +- 2 +- 2 +MIWaveTile: +- 8 +- 7 +MIWaveTileA: 8 +MIWaveTileB: 7 +MIWaveTileMetadata: 0 +MXLoadInst: BufferLoad +MXScaleFormat: NoSwizzle +MacroTile0: 256 +MacroTile1: 224 +MacroTileA: 256 +MacroTileB: 224 +MagicDivAlg: 2 +MathClocksUnrolledLoop: 0 +MatrixInstB: 1 +MatrixInstBM: 1 +MatrixInstBN: 1 +MatrixInstK: 32 +MatrixInstM: 16 +MatrixInstN: 16 +MatrixInstruction: +- 16 +- 16 +- 32 +- 1 +MaxLDS: 65536 +MaxOccupancy: 40 +MbskPrefetchMethod: 0 +MbskPrefetchOpt: 0 +MfmaInitCVgprs: false +MinGRIncPerMfma: 1 +Multicast: false +NoLdsWriteCode: false +NoReject: false +NoTailLoop: false +NonDTLTailLoopA: false +NonDTLTailLoopB: false +NonDTLTailLoopMetadata: false +NonTemporal: -1 +NonTemporalA: 0 +NonTemporalB: 0 +NonTemporalC: 0 +NonTemporalD: 0 +NonTemporalE: 0 +NonTemporalGate: 0 +NonTemporalMXSA: 0 +NonTemporalMXSB: 0 +NonTemporalMetadata: 0 +NonTemporalWS: 0 +NonVolatile: -1 +NonVolatileA: 0 +NonVolatileB: 0 +NonVolatileC: 0 +NonVolatileD: 0 +NonVolatileE: 0 +NonVolatileGate: 0 +NonVolatileMXSA: 0 +NonVolatileMXSB: 0 +NonVolatileMetadata: 0 +NonVolatileWS: 0 +NumElementsPerBatchStore: 0 +NumElementsPerThread: 224 +NumGlobalWriteVectorsPerThread: 28 +NumLdsBlk: 2 +NumLoadsA: 4 +NumLoadsB: 7 +NumLoadsCoalescedA: 1 +NumLoadsCoalescedB: 7 +NumLoadsPerpendicularA: 4 +NumLoadsPerpendicularB: 1 +NumThreads: 256 +NumTotalPackedLoadsA: -1 +NumTotalPackedLoadsB: -1 +NumWaveSplitK: 1 +NumWaves: 4 +OptNoLoadLoop: 1 +PackedC0IdxChars: +- I +PackedC0IndicesX: +- 0 +PackedC1IdxChars: +- J +PackedC1IndicesX: +- 1 +PersistentXCCMapping: 0 +PostLoopStoreInNll: false +PrefetchAcrossPersistent: 0 +PrefetchGL2: 0 +PrefetchGlobalRead: 2 +PrefetchLocalRead: 0 +PreloadKernArgs: true +ProblemType: + Activation: false + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: none + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: false + BiasDataTypeList: [] + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 13 + DataTypeA: 11 + DataTypeAmaxD: 0 + DataTypeB: 12 + DataTypeE: 0 + DataTypeMXSA: 22 + DataTypeMXSB: 22 + DestDataType: 0 + F32XdlMathOp: 0 + FusedGemmA2A: false + GateResidualDataTypeList: [] + Gradient: false + GroupedGemm: false + HighPrecisionAccumulate: true + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: + - 3 + - 0 + - 2 + IndexAssignmentsB: + - 1 + - 3 + - 2 + IndexAssignmentsLD: + - 4 + - 5 + - 6 + - 7 + IndexAssignmentsMetadata: + - 3 + - 0 + - 2 + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 1 + IndexUnrollM: 0 + IndicesBatch: + - 2 + IndicesFree: + - 0 + - 1 + IndicesSummation: + - 3 + Int4EncodingA: Signed + MXBlockA: 0 + MXBlockB: 0 + MXBlockFreeA: 1 + MXBlockFreeB: 1 + MacDataTypeA: 11 + MacDataTypeB: 12 + MetadataLayout: 0 + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMXSA: [] + MirrorDimsMXSB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + ScaleBlockSizesA: [] + ScaleZeroPointA: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SetConstStrideGate: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: false + StridedBatched: true + SupportUserArgs: true + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: false + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: true + UseBeta: true + UseBias: 0 + UseE: false + UseGateResidual: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: '' + UseScaleAlphaVec: 0 + UseScaleCD: false + isMixMode: true +ReuseAcrossPersistent: 0 +SFCWGM: +- - 1 + - 1 +- - 1 + - 1 +ScheduleGROverBarrier: 0 +ScheduleGlobalRead: 1 +ScheduleIterAlg: 3 +ScheduleLocalWrite: 1 +SolutionIndex: 1 +SolutionNameMin: Cijk_Alik_Bjlk_F8NSS_BH_UserArgs_MT256x224x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS0_FDSI0_GRPM1_GRVWA8_GRVWB4_GSU0_GSUAMB_GSUC0_GSUWGMRR0_GLS0_ISA942_IU1_K1_LBSPPA256_LBSPPB1792_LBSPPM0_LPA8_LPB16_LPM0_LRVW8_LWPMn1_MIAV0_MIWT8_7_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB7_ONLL1_PGR2_PLR0_PKA1_SIA3_SS1_SU0_SUM1_SUS32_SPO0_SRVW0_SSO0_SVW8_SK3_SKXCCM8_TLDS1_ULSGRO0_USL1_UIOFGRO0_USFGRO0_VSn1_VWA8_VWB1_WSGRA0_WSGRB0_WS64_WG32_8_1_WGM6_WGMXCC1_WGMXCCGn1 +SolutionUID: 0uR2wCMAxx2r +SourceSwap: true +SpaceFillingAlgo: [] +StaggerU: 0 +StaggerUMapping: 0 +StaggerUStride: 32 +StinkyTofuRegisterAllocation: 0 +StorePriorityOpt: false +StoreRemapVectorWidth: 0 +StoreSwapAddr: false +StoreSyncOpt: 0 +StoreVectorWidth: 8 +StreamKAtomic: 0 +StreamKFixupTreeReduction: 0 +SubGroup0: 8 +SubGroup1: 32 +SubGroupA: 8 +SubGroupB: 32 +SuppressNoLoadLoop: false +SwInstructionPrefetch: 1 +SwapGlobalReadOrder: 0 +TDMInst: 0 +TDMIterateMode: 0 +TDMLoadWaveSync: false +TDMPlusLdsBuf: 0 +TDMSplit: false +TailloopInNll: false +TemporalHint: -1 +TemporalHintA: 0 +TemporalHintB: 0 +TemporalHintC: 0 +TemporalHintD: 0 +TemporalHintE: 0 +TemporalHintGate: 0 +TemporalHintMXSA: 0 +TemporalHintMXSB: 0 +TemporalHintMetadata: 0 +TemporalHintWS: 0 +ThreadTile: +- 1 +- 1 +ThreadTile0: 32 +ThreadTile1: 7 +ThreadTileA: 32 +ThreadTileB: 7 +TileProcessingStrategy: StreamK +TransposeLDS: 1 +TransposeLDSMetadata: 1 +ULSGRODoubleG2L: 0 +UnrollLoopSwapGlobalReadOrder: 0 +UnrollMajorLDSA: true +UnrollMajorLDSB: false +UnrollMajorLDSMetadata: true +Use64bShadowLimit: true +Use64bShadowLimitMX: false +UseCustomMainLoopSchedule: 0 +UseDirect32XEmulation: false +UseDot2F32XEmulation: false +UseDotInstruction: false +UseDualFMAC: false +UseF32XEmulation: false +UseGeneralizedNLCOneA: false +UseGeneralizedNLCOneB: false +UseGeneralizedNLCOneMXSA: false +UseGeneralizedNLCOneMXSB: false +UseGeneralizedNLCOneMetadata: false +UseInstOffsetForGRO: 0 +UseMFMAF32XEmulation: false +UsePLRPack: 0 +UseSgprForGRO: 0 +UseSubtileImpl: false +Valid: true +VectorStore: -1 +VectorWidthA: 8 +VectorWidthB: 1 +WaveSeparateGlobalReadA: 0 +WaveSeparateGlobalReadB: 0 +WaveSeparateGlobalReadMetadata: 0 +WaveSplitK: false +WavefrontSize: 64 +WorkAssignment: StaticGrid +WorkGroup: +- 32 +- 8 +- 1 +WorkGroupMapping: 0 +WorkGroupMappingXCC: -1 +WorkGroupMappingXCCGroup: -1 +WorkGroupReduction: false +WorkQueueStealing: 0 +WorkspaceCheck: +- 4 +- 0 +- 0 +_DepthU: 32 +_DepthUA: 32 +_DepthUB: 32 +_DepthUMetadata: 32 +_DualPeriodMXFP8: false +_GlobalAccumulation: PartialsBuffer +_HasTemporalHint: 0 +_PersistentLoop: true +_PersistentVectorEpilogueLds: false +_PrefetchAcrossPersistentEnabled: false +_ScalePeriodUA: 32 +_ScalePeriodUB: 32 +_ScaleReuseA: 1 +_ScaleReuseB: 1 +_ScheduleIterAlg: 3 +_SeparateEpilogueLds: false +_StinkyTofuOptLevel: 0 +_UseSgprForGRO: 0 +_VectorStore: 1 +_WorkspaceSizePerElemBias: 0 +_WorkspaceSizePerElemC: 4 +_inputVWA_was_auto: false +_inputVWB_was_auto: false +_staggerStrideShift: 0 +enableGLTrA: false +enableGLTrB: false +enableLDSTrA: false +enableLDSTrB: false +enableLDSTrMXSA: false +enableLDSTrMXSB: false +enableLDSTrMetadata: 0 +enableTDMA: false +enableTDMB: false +enableTDMMetadata: false +ldsNumBytesA: 8448 +ldsNumBytesB: 7232 +ldsNumBytesMXSA: 0 +ldsNumBytesMXSB: 0 +ldsNumBytesMetadata: 0 +numSubTiles: 1 +reorderGRInstForDTVA: false +reorderGRInstForDTVB: false +tailLoopOptA: true +tailLoopOptB: true diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.msgpack new file mode 100644 index 000000000000..70675943ea74 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.predicates.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.predicates.msgpack new file mode 100644 index 000000000000..c416c3650fa9 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.predicates.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.predicates.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.predicates.yaml new file mode 100644 index 000000000000..5ebe49832279 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.predicates.yaml @@ -0,0 +1,130 @@ +problem: + type: And + value: + - type: BoundSizeMultiple + index: -1 + value: 32 + - type: OperationIdentifierEqual + value: Contraction_l_Alik_Bljk_Cijk_Dijk + - type: BiasDataTypeWhiteList + value: + - 0 + - type: GateResidualDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: + - 3 + - type: AmaxDCheck + value: false + - type: ActivationEnumWhiteList + value: + - None + - Gelu + - Relu + - Sigmoid + - Silu + - Clamp + - type: TypesEqual + value: + - Float8 + - Float8 + - Float + - Float + - Float8 + - Float8 + - type: HighPrecisionAccumulate + value: true + - type: Activation + value: Hipblaslt_all + - type: ActivationComputeType + value: Float + - type: ActivationNoGuard + value: false + - type: UseGradient + value: false + - type: UseBias + value: 1 + - type: UseE + value: false + - type: UseGateResidual + value: false + - type: DataTypeE + value: Float + - type: StridedBatched + value: true + - type: GroupedGemm + value: true + - type: UseScaleAB + value: Scalar + - type: UseScaleCD + value: false + - type: UseScaleAlphaVec + value: 1 + - type: Sparse + value: 0 + - type: F32XdlMathOp + value: Float + - type: SupportDeviceUserArguments + value: true + - type: SwizzleTensorA + value: false + - type: SwizzleTensorB + value: false + - type: FusedGemmA2A + value: false + - type: MXBlockA + value: 32 + - type: DataTypeMXSA + value: E8 + - type: MXBlockFreeA + value: 7 + - type: MXBlockB + value: 32 + - type: DataTypeMXSB + value: E8 + - type: MXBlockFreeB + value: 4 + - type: WorkgroupNumberCheck + index: 0 + value: + - 64 + - 64 + - 1 + - type: LeadingFree0SizesGreaterOrEqual + value: 1 + - type: LeadingFree1SizesGreaterOrEqual + value: 1 + - type: KernelLanguageCompatible + value: Assembly + - type: BufferLoadOffsetLimitCheck + value: + ShiftPtrElemB: 0 + ShiftPtrElemA: 0 + DUorMT1: 64 + DUorMT0: 64 + - type: BufferLoadOffsetLimitCheck_Beta + value: 64 + - type: BufferStoreOffsetLimitCheck + value: 64 + - type: GlobalSplitUCheckMinK + value: + - 32 + - 1 + - type: WorkgroupMappingXCCCheck + value: + - 2 + - -1 +task: + type: And + value: + - type: WorkspaceCheck + - type: LaunchLimits +hardware: + type: AMDGPU + value: + type: And + value: + - type: Processor + value: gfx950 + - type: PciChipId + value: 30112 diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.record.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.record.txt new file mode 100644 index 000000000000..a0a79fcaa90e --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.record.txt @@ -0,0 +1,129 @@ +solutionName="Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_Bias_GG_HA_S_SAB_SAV_UserArgs_MT64x64x256_MI16x16x1_SN_LDSB0_AFC0_AG0_AFEM1_AFEM1_ASEM32_CB0_CD1_1_CLR1_CADS0_DTLA1_DTLB1_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS1_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA16_GRVWB16_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA950_ICIW0_IU1_K1_LDSTI0_LBSPPA0_LBSPPB0_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA0_LPB0_LPMXSA0_LPMXSB0_LPM0_LRVWn1_LWPMn1_MIAV0_MIWT2_2_MXLIBL_MXSFHPS_MO40_MGRIPM3_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PGR1_PLR1_PKA1_SGROB0_SIA3_SS0_SU0_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW4_SK0_SKFTR0_SKXCCM0_SIP1_SGRO0_TDMI0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSMn1_ULSGRO0_USL0_USLMX0_UIOFGRO0_UPLRP0_USFGROn1_USI1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG32_8_1_WGM16_WGMXCC2_WGMXCCGn1" +kernelName="" +sizeMapping.waveNum=4 +sizeMapping.WavefrontSize=64 +sizeMapping.workGroup=[32,8,1] +sizeMapping.macroTile=[64,64,1] +sizeMapping.threadTile=[1,1] +sizeMapping.matrixInstruction=[16,16,128,1] +sizeMapping.grvwA=16 +sizeMapping.grvwB=16 +sizeMapping.gwvwC=4 +sizeMapping.gwvwD=4 +sizeMapping.depthU=256 +sizeMapping.staggerU=0 +sizeMapping.staggerUMapping=0 +sizeMapping.globalSplitUPGR=16 +sizeMapping.globalSplitU=1 +sizeMapping.staggerStrideShift=0 +sizeMapping.workGroupMapping=16 +sizeMapping.packBatchDims=0 +sizeMapping.magicDivAlg=2 +sizeMapping.tileProcessingStrategy="None" +sizeMapping.workAssignment="StaticGrid" +sizeMapping.streamKAtomic=0 +sizeMapping.prefetchAcrossPersistent=0 +sizeMapping.sourceKernel=false +sizeMapping.globalAccumulation=2 +sizeMapping.adaptiveGemmGSUA=0 +sizeMapping.workspaceSizePerElemC=4 +sizeMapping.workspaceSizePerElemBias=0 +sizeMapping.activationFused=true +sizeMapping.workGroupMappingXCC=2 +sizeMapping.workGroupMappingXCCGroup=-1 +sizeMapping.globalSplitUCoalesced=false +sizeMapping.globalSplitUWorkGroupMappingRoundRobin=false +sizeMapping.CUOccupancy=-1 +sizeMapping.PrefetchGlobalRead=1 +sizeMapping.MathClocksUnrolledLoop=0 +sizeMapping.synchronizerSizePerWG=256 +sizeMapping.nonTemporalA=0 +sizeMapping.nonTemporalB=0 +sizeMapping.temporalHintA=0 +sizeMapping.temporalHintB=0 +sizeMapping.hasTemporalHint=false +sizeMapping.adaptiveGemmNTAB=0 +sizeMapping.customMainLoopScheduling=0 +sizeMapping.useSubtileImpl=true +sizeMapping.NonTemporalD=4 +sizeMapping.WaveSeparateGlobalReadA=0 +sizeMapping.WaveSeparateGlobalReadB=0 +sizeMapping.UnrollLoopSwapGlobalReadOrder=0 +sizeMapping.DirectToVgprA=false +sizeMapping.DirectToVgprB=false +sizeMapping.NumLoadsCoalescedA=1 +sizeMapping.NumLoadsCoalescedB=1 +sizeMapping.WaveGroup=[2,2] +sizeMapping.VectorWidthA=1 +sizeMapping.VectorWidthB=1 +sizeMapping.LocalSplitU=1 +sizeMapping.DirectToLdsA=true +sizeMapping.DirectToLdsB=true +sizeMapping.OneLDSBuffer=false +sizeMapping.SourceSwap=false +sizeMapping.UnrollMajorLDSA=1 +sizeMapping.UnrollMajorLDSB=1 +sizeMapping.enableLDSTrA=false +sizeMapping.enableLDSTrB=false +sizeMapping.ExpertSchedulingMode=0 +sizeMapping.clusterDim=[1,1] +sizeMapping.BufferLoad=true +sizeMapping.BufferStore=true +internalArgsSupport.version=3 +internalArgsSupport.persistentLoopArgsVersion=0 +internalArgsSupport.gsu=false +internalArgsSupport.wgm=true +internalArgsSupport.staggerU=true +internalArgsSupport.perTileExtraIters=false +internalArgsSupport.useUniversalArgs=true +internalArgsSupport.useSFC=false +problemType.operationIdentifier="Contraction_l_Alik_Bljk_Cijk_Dijk" +problemType.transA=true +problemType.transB=false +problemType.computeInputTypeA="Float8" +problemType.computeInputTypeB="Float8" +problemType.aType="Float8" +problemType.bType="Float8" +problemType.cType="Float" +problemType.dType="Float" +problemType.eType="Float" +problemType.computeType="Float" +problemType.useBeta=true +problemType.useBias=1 +problemType.biasSrcWhiteList=[3] +problemType.useE=false +problemType.useGateResidual=false +problemType.gateResidualDataTypeWhiteList=[] +problemType.useScaleAB="Scalar" +problemType.useScaleCD=false +problemType.useScaleAlphaVec=1 +problemType.biasDataTypeWhiteList=[0] +problemType.highPrecisionAccumulate=true +problemType.useInitialStridesAB=false +problemType.useInitialStridesCD=false +problemType.stridedBatched=true +problemType.groupedGemm=true +problemType.useGradient=false +problemType.activationType="Hipblaslt_all" +problemType.activationArgLength=2 +problemType.activationComputeDataType="Float" +problemType.activationNoGuard=false +problemType.sparse=0 +problemType.f32XdlMathOp="Float" +problemType.supportDeviceUserArguments=true +problemType.outputAmaxD=false +problemType.swizzleTensorA=false +problemType.swizzleTensorB=false +problemType.metadataLayout=0 +problemType.mxBlockA=32 +problemType.mxBlockB=32 +problemType.mxBlockFreeA=7 +problemType.mxBlockFreeB=4 +problemType.mxTypeA="E8" +problemType.mxTypeB="E8" +problemType.mxScaleFormat=1 +problemType.fusedGemmA2A=false +problemType.stochasticRounding=false +problemPredicate=["And","BoundSizeMultiple","OperationIdentifierEqual","BiasDataTypeWhiteList","GateResidualDataTypeWhiteList","BiasSrcWhiteList","AmaxDCheck","ActivationEnumWhiteList","TypesEqual","HighPrecisionAccumulate","Activation","ActivationComputeType","ActivationNoGuard","UseGradient","UseBias","UseE","UseGateResidual","DataTypeE","StridedBatched","GroupedGemm","UseScaleAB","UseScaleCD","UseScaleAlphaVec","Sparse","F32XdlMathOp","SupportDeviceUserArguments","SwizzleTensorA","SwizzleTensorB","FusedGemmA2A","MXBlockA","DataTypeMXSA","MXBlockFreeA","MXBlockB","DataTypeMXSB","MXBlockFreeB","WorkgroupNumberCheck","LeadingFree0SizesGreaterOrEqual","LeadingFree1SizesGreaterOrEqual","KernelLanguageCompatible","BufferLoadOffsetLimitCheck","BufferLoadOffsetLimitCheck_Beta","BufferStoreOffsetLimitCheck","GlobalSplitUCheckMinK","WorkgroupMappingXCCCheck"] +taskPredicate=["And","WorkspaceCheck","LaunchLimits"] +hardwarePredicate=["AMDGPU","And","Processor","PciChipId"] diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.yaml new file mode 100644 index 000000000000..c3438c8f3473 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx-free-blocks.yaml @@ -0,0 +1,576 @@ +1LDSBuffer: 0 +ActivationAlt: false +ActivationFuncCall: false +ActivationFused: true +AdaptiveGemm: 0 +AdaptiveGemmGSUA: 0 +AdaptiveGemmNTAB: 0 +AssertAIGreaterThanEqual: -1 +AssertAILessThanEqual: -1 +AssertFree0ElementMultiple: 1 +AssertFree1ElementMultiple: 1 +AssertSummationElementMultiple: 32 +AssignedDerivedParameters: true +AssignedProblemIndependentDerivedParameters: true +BaseName: Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_Bias_HAUzuxVKq5nzha5MufpHbE31RJPTuQv2X3-kwKww_odV8= +BufferLoad: true +BufferStore: true +CUCount: null +CUOccupancy: -1 +ClusterAIDMapping: -1 +ClusterBarrier: false +ClusterDim: +- 1 +- 1 +ClusterLocalRead: 1 +CodeObjectVersion: '4' +CompactLoopStore: false +ConvertAfterDS: false +CustomKernelName: '' +DebugPersistentKernelLoopForever: false +DebugStreamK: 0 +DepthU: 256 +DeviceNames: +- Device 75a0 +DirectToLds: 1 +DirectToLdsA: true +DirectToLdsB: true +DirectToLdsMXSA: true +DirectToLdsMXSB: true +DirectToLdsMetadata: 0 +DirectToVgprA: false +DirectToVgprB: false +DirectToVgprMXSA: false +DirectToVgprMXSB: false +DirectToVgprSparseMetadata: false +DisableXdlArbStall: true +DtlPlusLdsBuf: 0 +ESMRuntimeGate: false +EdgeType: ShiftPtr +EnableESM2TrackValuVsrc: true +EnableF32XdlMathOp: false +EnableMatrixInstruction: true +EnableStinkyTofuESM2: false +ExpandPointerSwap: true +ExpertSchedulingMode: 0 +ExtraLatencyForLR: 0 +ExtraMiLatencyLeft: -1 +ForceDisableShadowInit: false +ForceUnrollSubIter: false +GlobalReadPerMfma: 1.0 +GlobalReadVectorWidthA: 16 +GlobalReadVectorWidthB: 16 +GlobalReadVectorWidthMXSA: 2 +GlobalReadVectorWidthMXSB: 2 +GlobalSplitU: 1 +GlobalSplitUAlgorithm: MultipleBuffer +GlobalSplitUCoalesced: false +GlobalSplitUWorkGroupMappingRoundRobin: false +GlobalWriteVectorWidth: 1 +GroupLoadStore: false +GuaranteeNoPartialA: true +GuaranteeNoPartialB: true +GuaranteeNoPartialMetadata: true +HalfPLR: 0 +HalfPLRA: false +HalfPLRB: false +ISA: +- 9 +- 5 +- 0 +InitCIterWmma: 0 +InnerUnroll: 1 +InterleaveAlpha: 0 +InternalSupportParams: + KernArgsVersion: 3 + PersistentLoopArgsVersion: 0 + SupportCustomStaggerU: true + SupportCustomWGM: true + SupportOccupancyProbe: false + SupportStreamKPerTileExtraIters: false + SupportUserGSU: false + UseSFC: false + UseUniversalArgs: true +Kernel: true +KernelLanguage: Assembly +LDSSegInterleaveOffsets: {} +LDSSegmentBRotate: 0 +LDSSegmentInterleave: 0 +LDSTrInst: false +LSCA: 256 +LSCB: 256 +LSCMXSA: 0 +LSCMXSB: 0 +LSPA: 16 +LSPB: 16 +LSPMXSA: 0 +LSPMXSB: 0 +LVCA: 16 +LVCB: 16 +LVCMXSA: 0 +LVCMXSB: 0 +LVPA: 1 +LVPB: 1 +LVPMXSA: 0 +LVPMXSB: 0 +LdsBlockSizePerPadA: 0 +LdsBlockSizePerPadB: 0 +LdsBlockSizePerPadMXSA: 0 +LdsBlockSizePerPadMXSB: 0 +LdsBlockSizePerPadMetadata: 0 +LdsBytesNoAmax: 67584 +LdsInitCVgprs: false +LdsNumBytes: 67584 +LdsNumElementsAlignedA: 16384 +LdsNumElementsAlignedB: 16384 +LdsNumElementsAlignedMXSA: 512 +LdsNumElementsAlignedMXSB: 512 +LdsNumElementsAlignedMetadata: 0 +LdsOffsetA: 0 +LdsOffsetA_Blk: 33792 +LdsOffsetB: 17408 +LdsOffsetB_Blk: 51200 +LdsOffsetBias: 0 +LdsOffsetBiasGSU: 0 +LdsOffsetBiasNonGSU: 0 +LdsOffsetMXSA: 16384 +LdsOffsetMXSA_Blk: 50176 +LdsOffsetMXSB: 16896 +LdsOffsetMXSB_Blk: 50688 +LdsOffsetMetadata: 17408 +LdsOffsetMetadata_Blk: 51200 +LdsPadA: 0 +LdsPadB: 0 +LdsPadMXSA: 0 +LdsPadMXSB: 0 +LdsPadMetadata: 0 +LocalReadVectorWidth: -1 +LocalReadVectorWidthA: 32 +LocalReadVectorWidthB: 32 +LocalReadVectorWidthMXS: 1 +LocalReadVectorWidthMXSA: 1 +LocalReadVectorWidthMXSB: 1 +LocalSplitU: 1 +LocalSplitUReuseLDS: 1 +LocalWritePerMfma: -1 +LocalWriteUseSgprA: true +LocalWriteUseSgprB: true +LocalWriteUseSgprMXSA: true +LocalWriteUseSgprMXSB: true +LocalWriteUseSgprMetadata: false +LoopIters: 2 +LoopUnroll: 256 +MFMA_BF16_1K: false +MIArchVgpr: false +MIBlock: +- 16 +- 16 +- 128 +- 1 +- 1 +- 1 +MIInputPerThread: 32 +MIInputPerThreadA: 32 +MIInputPerThreadB: 32 +MIInputPerThreadMXSA: 1 +MIInputPerThreadMXSB: 1 +MIInputPerThreadMetadata: 32 +MIOutputVectorWidth: 4 +MIRegPerOut: 1 +MIWaveGroup: +- 2 +- 2 +MIWaveTile: +- 2 +- 2 +MIWaveTileA: 2 +MIWaveTileB: 2 +MIWaveTileMXSA: 2 +MIWaveTileMXSB: 2 +MIWaveTileMetadata: 0 +MXLoadInst: BufferLoad +MXScaleFormat: HostPreSwizzle +MacroTile0: 64 +MacroTile1: 64 +MacroTileA: 64 +MacroTileB: 64 +MacroTileMXSA: 64 +MacroTileMXSB: 64 +MagicDivAlg: 2 +MathClocksUnrolledLoop: 0 +MatrixInstB: 1 +MatrixInstBM: 1 +MatrixInstBN: 1 +MatrixInstK: 128 +MatrixInstM: 16 +MatrixInstN: 16 +MatrixInstruction: +- 16 +- 16 +- 128 +- 1 +MaxLDS: 163840 +MaxOccupancy: 40 +MbskPrefetchMethod: 0 +MfmaInitCVgprs: false +MinGRIncPerMfma: 3 +Multicast: false +NoLdsWriteCode: true +NoReject: false +NoTailLoop: false +NonDTLTailLoopA: true +NonDTLTailLoopB: true +NonDTLTailLoopMXSA: true +NonDTLTailLoopMXSB: true +NonDTLTailLoopMetadata: false +NonTemporal: -1 +NonTemporalA: 0 +NonTemporalB: 0 +NonTemporalC: 0 +NonTemporalD: 4 +NonTemporalE: 0 +NonTemporalGate: 0 +NonTemporalMXSA: 0 +NonTemporalMXSB: 0 +NonTemporalMetadata: 0 +NonTemporalWS: 0 +NonVolatile: -1 +NonVolatileA: 0 +NonVolatileB: 0 +NonVolatileC: 0 +NonVolatileD: 0 +NonVolatileE: 0 +NonVolatileGate: 0 +NonVolatileMXSA: 0 +NonVolatileMXSB: 0 +NonVolatileMetadata: 0 +NonVolatileWS: 0 +NumElementsPerBatchStore: 0 +NumElementsPerThread: 16 +NumGlobalWriteVectorsPerThread: 16 +NumLdsBlk: 2 +NumLoadsA: 4 +NumLoadsB: 4 +NumLoadsCoalescedA: 1 +NumLoadsCoalescedB: 1 +NumLoadsCoalescedMXSA: 1 +NumLoadsCoalescedMXSB: 1 +NumLoadsPerpendicularA: 4 +NumLoadsPerpendicularB: 4 +NumLoadsPerpendicularMXSA: 0 +NumLoadsPerpendicularMXSB: 0 +NumThreads: 256 +NumTotalPackedLoadsA: -1 +NumTotalPackedLoadsB: -1 +NumTotalPackedLoadsMXSA: 0 +NumTotalPackedLoadsMXSB: 0 +NumWaveSplitK: 1 +NumWaves: 4 +OptNoLoadLoop: 0 +PackedC0IdxChars: +- I +PackedC0IndicesX: +- 0 +PackedC1IdxChars: +- J +PackedC1IndicesX: +- 1 +PersistentXCCMapping: 0 +PostLoopStoreInNll: false +PrefetchAcrossPersistent: 0 +PrefetchGL2: 0 +PrefetchGlobalRead: 1 +PrefetchLocalRead: 1 +PreloadKernArgs: true +ProblemType: + Activation: true + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: hipblaslt_all + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: true + BiasDataTypeList: + - 0 + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 15 + DataTypeA: 15 + DataTypeAmaxD: 0 + DataTypeB: 15 + DataTypeE: 0 + DataTypeMXSA: 22 + DataTypeMXSB: 22 + DestDataType: 0 + F32XdlMathOp: 0 + FusedGemmA2A: false + GateResidualDataTypeList: [] + Gradient: false + GroupedGemm: true + HighPrecisionAccumulate: true + Index0: 0 + Index01A: 0 + Index01B: 1 + Index01MXSA: 0 + Index01MXSB: 1 + Index1: 1 + IndexAssignmentsA: + - 3 + - 0 + - 2 + IndexAssignmentsB: + - 3 + - 1 + - 2 + IndexAssignmentsLD: + - 4 + - 5 + - 6 + - 7 + IndexAssignmentsMXSA: + - 3 + - 0 + - 2 + IndexAssignmentsMXSB: + - 3 + - 1 + - 2 + IndexAssignmentsMetadata: + - 3 + - 0 + - 2 + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndexUnrollM: 0 + IndexUnrollMXSA: 0 + IndexUnrollMXSB: 0 + IndicesBatch: + - 2 + IndicesFree: + - 0 + - 1 + IndicesSummation: + - 3 + Int4EncodingA: Signed + MXBlockA: 32 + MXBlockB: 32 + MXBlockFreeA: 7 + MXBlockFreeB: 4 + MacDataTypeA: 15 + MacDataTypeB: 15 + MetadataLayout: 0 + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMXSA: [] + MirrorDimsMXSB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + ScaleBlockSizesA: [] + ScaleZeroPointA: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SetConstStrideGate: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: false + StridedBatched: true + SupportUserArgs: true + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: false + TLUB: false + TLUMXSA: false + TLUMXSB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseBias: 1 + UseE: false + UseGateResidual: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: Scalar + UseScaleAlphaVec: 1 + UseScaleCD: false + isMixMode: false +ReuseAcrossPersistent: 0 +SFCWGM: +- - 1 + - 1 +- - 1 + - 1 +ScheduleGROverBarrier: 0 +ScheduleGlobalRead: 1 +ScheduleIterAlg: 3 +ScheduleLocalWrite: 1 +SolutionIndex: 0 +SolutionNameMin: Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_Bias_GG_HA_S_SAB_SAV_UserArgs_MT64x64x256_MI16x16x1_SN_LDSB0_AFC0_AG0_AFEM1_AFEM1_ASEM32_CB0_CD1_1_CLR1_CADS0_DTLA1_DTLB1_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS1_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA16_GRVWB16_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA950_ICIW0_IU1_K1_LDSTI0_LBSPPA0_LBSPPB0_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA0_LPB0_LPMXSA0_LPMXSB0_LPM0_LRVWn1_LWPMn1_MIAV0_MIWT2_2_MXLIBL_MXSFHPS_MO40_MGRIPM3_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PGR1_PLR1_PKA1_SGROB0_SIA3_SS0_SU0_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW4_SK0_SKFTR0_SKXCCM0_SIP1_SGRO0_TDMI0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSMn1_ULSGRO0_USL0_USLMX0_UIOFGRO0_UPLRP0_USFGROn1_USI1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG32_8_1_WGM16_WGMXCC2_WGMXCCGn1 +SourceSwap: false +SpaceFillingAlgo: [] +StaggerU: 0 +StaggerUMapping: 0 +StaggerUStride: 256 +StinkyTofuRegisterAllocation: 0 +StorePriorityOpt: false +StoreRemapVectorWidth: 0 +StoreSwapAddr: false +StoreSyncOpt: 0 +StoreVectorWidth: 4 +StreamKAtomic: 0 +StreamKFixupTreeReduction: 0 +SubGroup0: 8 +SubGroup1: 32 +SubGroupA: 8 +SubGroupB: 32 +SubGroupMXSA: 8 +SubGroupMXSB: 32 +SuppressNoLoadLoop: false +SwInstructionPrefetch: true +SwapGlobalReadOrder: 0 +TDMInst: 0 +TDMIterateMode: 0 +TDMLoadWaveSync: false +TDMPlusLdsBuf: 0 +TDMSplit: false +TailloopInNll: false +TemporalHint: -1 +TemporalHintA: 0 +TemporalHintB: 0 +TemporalHintC: 0 +TemporalHintD: 0 +TemporalHintE: 0 +TemporalHintGate: 0 +TemporalHintMXSA: 0 +TemporalHintMXSB: 0 +TemporalHintMetadata: 0 +TemporalHintWS: 0 +ThreadTile: +- 1 +- 1 +ThreadTile0: 8 +ThreadTile1: 2 +ThreadTileA: 8 +ThreadTileB: 2 +ThreadTileMXSA: 8 +ThreadTileMXSB: 2 +TileProcessingStrategy: None +TransposeLDS: 1 +TransposeLDSMetadata: -1 +ULSGRODoubleG2L: 0 +UnrollLoopSwapGlobalReadOrder: 0 +UnrollMajorLDSA: true +UnrollMajorLDSB: true +UnrollMajorLDSMXSA: true +UnrollMajorLDSMXSB: true +Use64bShadowLimit: false +Use64bShadowLimitMX: false +UseCustomMainLoopSchedule: 0 +UseDirect32XEmulation: false +UseDot2F32XEmulation: false +UseDotInstruction: false +UseDualFMAC: false +UseF32XEmulation: false +UseGeneralizedNLCOneA: false +UseGeneralizedNLCOneB: false +UseGeneralizedNLCOneMXSA: false +UseGeneralizedNLCOneMXSB: false +UseGeneralizedNLCOneMetadata: false +UseInstOffsetForGRO: 0 +UseMFMAF32XEmulation: false +UsePLRPack: 0 +UseSgprForGRO: -1 +UseSubtileImpl: true +Valid: true +VectorStore: -1 +VectorWidthA: 1 +VectorWidthB: 1 +VectorWidthMXSA: 1 +VectorWidthMXSB: 1 +WaveSeparateGlobalReadA: 0 +WaveSeparateGlobalReadB: 0 +WaveSeparateGlobalReadMXSA: 0 +WaveSeparateGlobalReadMXSB: 0 +WaveSeparateGlobalReadMetadata: 0 +WaveSplitK: false +WavefrontSize: 64 +WorkAssignment: StaticGrid +WorkGroup: +- 32 +- 8 +- 1 +WorkGroupMapping: 16 +WorkGroupMappingXCC: 2 +WorkGroupMappingXCCGroup: -1 +WorkGroupReduction: false +WorkQueueStealing: 0 +WorkspaceCheck: +- 4 +- 0 +- 1 +_ABTilePairA: AB_B8 +_ABTilePairB: AB_B8 +_DepthU: 256 +_DepthUA: 256 +_DepthUB: 256 +_DepthUMXSA: 8 +_DepthUMXSB: 8 +_DepthUMetadata: 256 +_DualPeriodMXFP8: false +_GlobalAccumulation: MultipleBuffer +_HasTemporalHint: 0 +_PersistentLoop: false +_PersistentVectorEpilogueLds: false +_PrefetchAcrossPersistentEnabled: false +_ScalePeriodUA: 256 +_ScalePeriodUB: 256 +_ScaleReuseA: 1 +_ScaleReuseB: 1 +_ScheduleIterAlg: 3 +_SeparateEpilogueLds: false +_StinkyTofuOptLevel: 0 +_UseSgprForGRO: 0 +_VectorStore: 1 +_WorkspaceSizePerElemBias: 0 +_WorkspaceSizePerElemC: 4 +_inputVWA_was_auto: false +_inputVWB_was_auto: false +_staggerStrideShift: 0 +enableGLTrA: false +enableGLTrB: false +enableLDSTrA: false +enableLDSTrB: false +enableLDSTrMXSA: false +enableLDSTrMXSB: false +enableLDSTrMetadata: 0 +enableTDMA: false +enableTDMB: false +enableTDMMetadata: false +ldsNumBytesA: 16384 +ldsNumBytesB: 16384 +ldsNumBytesMXSA: 512 +ldsNumBytesMXSB: 512 +ldsNumBytesMetadata: 0 +numSubTiles: 1 +reorderGRInstForDTVA: false +reorderGRInstForDTVB: false +tailLoopOptA: false +tailLoopOptB: false +tailLoopOptMXSA: false +tailLoopOptMXSB: false diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.msgpack new file mode 100644 index 000000000000..2a77eba82f84 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.predicates.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.predicates.msgpack new file mode 100644 index 000000000000..b76f28195484 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.predicates.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.predicates.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.predicates.yaml new file mode 100644 index 000000000000..1dc38556e1e2 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.predicates.yaml @@ -0,0 +1,130 @@ +problem: + type: And + value: + - type: BoundSizeMultiple + index: -1 + value: 32 + - type: OperationIdentifierEqual + value: Contraction_l_Alik_Bljk_Cijk_Dijk + - type: BiasDataTypeWhiteList + value: + - 0 + - type: GateResidualDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: + - 3 + - type: AmaxDCheck + value: false + - type: ActivationEnumWhiteList + value: + - None + - Gelu + - Relu + - Sigmoid + - Silu + - Clamp + - type: TypesEqual + value: + - Float8 + - Float8 + - Float + - Float + - Float8 + - Float8 + - type: HighPrecisionAccumulate + value: true + - type: Activation + value: Hipblaslt_all + - type: ActivationComputeType + value: Float + - type: ActivationNoGuard + value: false + - type: UseGradient + value: false + - type: UseBias + value: 1 + - type: UseE + value: false + - type: UseGateResidual + value: false + - type: DataTypeE + value: Float + - type: StridedBatched + value: true + - type: GroupedGemm + value: true + - type: UseScaleAB + value: Scalar + - type: UseScaleCD + value: false + - type: UseScaleAlphaVec + value: 1 + - type: Sparse + value: 0 + - type: F32XdlMathOp + value: Float + - type: SupportDeviceUserArguments + value: true + - type: SwizzleTensorA + value: false + - type: SwizzleTensorB + value: false + - type: FusedGemmA2A + value: false + - type: MXBlockA + value: 32 + - type: DataTypeMXSA + value: E8 + - type: MXBlockFreeA + value: 1 + - type: MXBlockB + value: 32 + - type: DataTypeMXSB + value: E8 + - type: MXBlockFreeB + value: 1 + - type: WorkgroupNumberCheck + index: 0 + value: + - 64 + - 64 + - 1 + - type: LeadingFree0SizesGreaterOrEqual + value: 1 + - type: LeadingFree1SizesGreaterOrEqual + value: 1 + - type: KernelLanguageCompatible + value: Assembly + - type: BufferLoadOffsetLimitCheck + value: + ShiftPtrElemB: 0 + ShiftPtrElemA: 0 + DUorMT1: 64 + DUorMT0: 64 + - type: BufferLoadOffsetLimitCheck_Beta + value: 64 + - type: BufferStoreOffsetLimitCheck + value: 64 + - type: GlobalSplitUCheckMinK + value: + - 32 + - 1 + - type: WorkgroupMappingXCCCheck + value: + - 2 + - -1 +task: + type: And + value: + - type: WorkspaceCheck + - type: LaunchLimits +hardware: + type: AMDGPU + value: + type: And + value: + - type: Processor + value: gfx950 + - type: PciChipId + value: 30112 diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.record.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.record.txt new file mode 100644 index 000000000000..7bcf8bfa5b7f --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.record.txt @@ -0,0 +1,129 @@ +solutionName="Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_Bias_GG_HA_S_SAB_SAV_UserArgs_MT64x64x256_MI16x16x1_SN_LDSB0_AFC0_AG0_AFEM1_AFEM1_ASEM32_CB0_CD1_1_CLR1_CADS0_DTLA1_DTLB1_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS1_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA16_GRVWB16_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA950_ICIW0_IU1_K1_LDSTI0_LBSPPA0_LBSPPB0_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA0_LPB0_LPMXSA0_LPMXSB0_LPM0_LRVWn1_LWPMn1_MIAV0_MIWT2_2_MXLIBL_MXSFHPS_MO40_MGRIPM3_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PGR1_PLR1_PKA1_SGROB0_SIA3_SS0_SU0_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW4_SK0_SKFTR0_SKXCCM0_SIP1_SGRO0_TDMI0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSMn1_ULSGRO0_USL0_USLMX0_UIOFGRO0_UPLRP0_USFGROn1_USI1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG32_8_1_WGM16_WGMXCC2_WGMXCCGn1" +kernelName="" +sizeMapping.waveNum=4 +sizeMapping.WavefrontSize=64 +sizeMapping.workGroup=[32,8,1] +sizeMapping.macroTile=[64,64,1] +sizeMapping.threadTile=[1,1] +sizeMapping.matrixInstruction=[16,16,128,1] +sizeMapping.grvwA=16 +sizeMapping.grvwB=16 +sizeMapping.gwvwC=4 +sizeMapping.gwvwD=4 +sizeMapping.depthU=256 +sizeMapping.staggerU=0 +sizeMapping.staggerUMapping=0 +sizeMapping.globalSplitUPGR=16 +sizeMapping.globalSplitU=1 +sizeMapping.staggerStrideShift=0 +sizeMapping.workGroupMapping=16 +sizeMapping.packBatchDims=0 +sizeMapping.magicDivAlg=2 +sizeMapping.tileProcessingStrategy="None" +sizeMapping.workAssignment="StaticGrid" +sizeMapping.streamKAtomic=0 +sizeMapping.prefetchAcrossPersistent=0 +sizeMapping.sourceKernel=false +sizeMapping.globalAccumulation=2 +sizeMapping.adaptiveGemmGSUA=0 +sizeMapping.workspaceSizePerElemC=4 +sizeMapping.workspaceSizePerElemBias=0 +sizeMapping.activationFused=true +sizeMapping.workGroupMappingXCC=2 +sizeMapping.workGroupMappingXCCGroup=-1 +sizeMapping.globalSplitUCoalesced=false +sizeMapping.globalSplitUWorkGroupMappingRoundRobin=false +sizeMapping.CUOccupancy=-1 +sizeMapping.PrefetchGlobalRead=1 +sizeMapping.MathClocksUnrolledLoop=0 +sizeMapping.synchronizerSizePerWG=256 +sizeMapping.nonTemporalA=0 +sizeMapping.nonTemporalB=0 +sizeMapping.temporalHintA=0 +sizeMapping.temporalHintB=0 +sizeMapping.hasTemporalHint=false +sizeMapping.adaptiveGemmNTAB=0 +sizeMapping.customMainLoopScheduling=0 +sizeMapping.useSubtileImpl=true +sizeMapping.NonTemporalD=4 +sizeMapping.WaveSeparateGlobalReadA=0 +sizeMapping.WaveSeparateGlobalReadB=0 +sizeMapping.UnrollLoopSwapGlobalReadOrder=0 +sizeMapping.DirectToVgprA=false +sizeMapping.DirectToVgprB=false +sizeMapping.NumLoadsCoalescedA=1 +sizeMapping.NumLoadsCoalescedB=1 +sizeMapping.WaveGroup=[2,2] +sizeMapping.VectorWidthA=1 +sizeMapping.VectorWidthB=1 +sizeMapping.LocalSplitU=1 +sizeMapping.DirectToLdsA=true +sizeMapping.DirectToLdsB=true +sizeMapping.OneLDSBuffer=false +sizeMapping.SourceSwap=false +sizeMapping.UnrollMajorLDSA=1 +sizeMapping.UnrollMajorLDSB=1 +sizeMapping.enableLDSTrA=false +sizeMapping.enableLDSTrB=false +sizeMapping.ExpertSchedulingMode=0 +sizeMapping.clusterDim=[1,1] +sizeMapping.BufferLoad=true +sizeMapping.BufferStore=true +internalArgsSupport.version=3 +internalArgsSupport.persistentLoopArgsVersion=0 +internalArgsSupport.gsu=false +internalArgsSupport.wgm=true +internalArgsSupport.staggerU=true +internalArgsSupport.perTileExtraIters=false +internalArgsSupport.useUniversalArgs=true +internalArgsSupport.useSFC=false +problemType.operationIdentifier="Contraction_l_Alik_Bljk_Cijk_Dijk" +problemType.transA=true +problemType.transB=false +problemType.computeInputTypeA="Float8" +problemType.computeInputTypeB="Float8" +problemType.aType="Float8" +problemType.bType="Float8" +problemType.cType="Float" +problemType.dType="Float" +problemType.eType="Float" +problemType.computeType="Float" +problemType.useBeta=true +problemType.useBias=1 +problemType.biasSrcWhiteList=[3] +problemType.useE=false +problemType.useGateResidual=false +problemType.gateResidualDataTypeWhiteList=[] +problemType.useScaleAB="Scalar" +problemType.useScaleCD=false +problemType.useScaleAlphaVec=1 +problemType.biasDataTypeWhiteList=[0] +problemType.highPrecisionAccumulate=true +problemType.useInitialStridesAB=false +problemType.useInitialStridesCD=false +problemType.stridedBatched=true +problemType.groupedGemm=true +problemType.useGradient=false +problemType.activationType="Hipblaslt_all" +problemType.activationArgLength=2 +problemType.activationComputeDataType="Float" +problemType.activationNoGuard=false +problemType.sparse=0 +problemType.f32XdlMathOp="Float" +problemType.supportDeviceUserArguments=true +problemType.outputAmaxD=false +problemType.swizzleTensorA=false +problemType.swizzleTensorB=false +problemType.metadataLayout=0 +problemType.mxBlockA=32 +problemType.mxBlockB=32 +problemType.mxBlockFreeA=1 +problemType.mxBlockFreeB=1 +problemType.mxTypeA="E8" +problemType.mxTypeB="E8" +problemType.mxScaleFormat=1 +problemType.fusedGemmA2A=false +problemType.stochasticRounding=false +problemPredicate=["And","BoundSizeMultiple","OperationIdentifierEqual","BiasDataTypeWhiteList","GateResidualDataTypeWhiteList","BiasSrcWhiteList","AmaxDCheck","ActivationEnumWhiteList","TypesEqual","HighPrecisionAccumulate","Activation","ActivationComputeType","ActivationNoGuard","UseGradient","UseBias","UseE","UseGateResidual","DataTypeE","StridedBatched","GroupedGemm","UseScaleAB","UseScaleCD","UseScaleAlphaVec","Sparse","F32XdlMathOp","SupportDeviceUserArguments","SwizzleTensorA","SwizzleTensorB","FusedGemmA2A","MXBlockA","DataTypeMXSA","MXBlockFreeA","MXBlockB","DataTypeMXSB","MXBlockFreeB","WorkgroupNumberCheck","LeadingFree0SizesGreaterOrEqual","LeadingFree1SizesGreaterOrEqual","KernelLanguageCompatible","BufferLoadOffsetLimitCheck","BufferLoadOffsetLimitCheck_Beta","BufferStoreOffsetLimitCheck","GlobalSplitUCheckMinK","WorkgroupMappingXCCCheck"] +taskPredicate=["And","WorkspaceCheck","LaunchLimits"] +hardwarePredicate=["AMDGPU","And","Processor","PciChipId"] diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.yaml new file mode 100644 index 000000000000..b94ebed57194 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-mx.yaml @@ -0,0 +1,576 @@ +1LDSBuffer: 0 +ActivationAlt: false +ActivationFuncCall: false +ActivationFused: true +AdaptiveGemm: 0 +AdaptiveGemmGSUA: 0 +AdaptiveGemmNTAB: 0 +AssertAIGreaterThanEqual: -1 +AssertAILessThanEqual: -1 +AssertFree0ElementMultiple: 1 +AssertFree1ElementMultiple: 1 +AssertSummationElementMultiple: 32 +AssignedDerivedParameters: true +AssignedProblemIndependentDerivedParameters: true +BaseName: Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_Bias_HAUzuxVKq5nzha5MufpHbE31RJPTuQv2X3-kwKww_odV8= +BufferLoad: true +BufferStore: true +CUCount: null +CUOccupancy: -1 +ClusterAIDMapping: -1 +ClusterBarrier: false +ClusterDim: +- 1 +- 1 +ClusterLocalRead: 1 +CodeObjectVersion: '4' +CompactLoopStore: false +ConvertAfterDS: false +CustomKernelName: '' +DebugPersistentKernelLoopForever: false +DebugStreamK: 0 +DepthU: 256 +DeviceNames: +- Device 75a0 +DirectToLds: 1 +DirectToLdsA: true +DirectToLdsB: true +DirectToLdsMXSA: true +DirectToLdsMXSB: true +DirectToLdsMetadata: 0 +DirectToVgprA: false +DirectToVgprB: false +DirectToVgprMXSA: false +DirectToVgprMXSB: false +DirectToVgprSparseMetadata: false +DisableXdlArbStall: true +DtlPlusLdsBuf: 0 +ESMRuntimeGate: false +EdgeType: ShiftPtr +EnableESM2TrackValuVsrc: true +EnableF32XdlMathOp: false +EnableMatrixInstruction: true +EnableStinkyTofuESM2: false +ExpandPointerSwap: true +ExpertSchedulingMode: 0 +ExtraLatencyForLR: 0 +ExtraMiLatencyLeft: -1 +ForceDisableShadowInit: false +ForceUnrollSubIter: false +GlobalReadPerMfma: 1.0 +GlobalReadVectorWidthA: 16 +GlobalReadVectorWidthB: 16 +GlobalReadVectorWidthMXSA: 2 +GlobalReadVectorWidthMXSB: 2 +GlobalSplitU: 1 +GlobalSplitUAlgorithm: MultipleBuffer +GlobalSplitUCoalesced: false +GlobalSplitUWorkGroupMappingRoundRobin: false +GlobalWriteVectorWidth: 1 +GroupLoadStore: false +GuaranteeNoPartialA: true +GuaranteeNoPartialB: true +GuaranteeNoPartialMetadata: true +HalfPLR: 0 +HalfPLRA: false +HalfPLRB: false +ISA: +- 9 +- 5 +- 0 +InitCIterWmma: 0 +InnerUnroll: 1 +InterleaveAlpha: 0 +InternalSupportParams: + KernArgsVersion: 3 + PersistentLoopArgsVersion: 0 + SupportCustomStaggerU: true + SupportCustomWGM: true + SupportOccupancyProbe: false + SupportStreamKPerTileExtraIters: false + SupportUserGSU: false + UseSFC: false + UseUniversalArgs: true +Kernel: true +KernelLanguage: Assembly +LDSSegInterleaveOffsets: {} +LDSSegmentBRotate: 0 +LDSSegmentInterleave: 0 +LDSTrInst: false +LSCA: 256 +LSCB: 256 +LSCMXSA: 0 +LSCMXSB: 0 +LSPA: 16 +LSPB: 16 +LSPMXSA: 0 +LSPMXSB: 0 +LVCA: 16 +LVCB: 16 +LVCMXSA: 0 +LVCMXSB: 0 +LVPA: 1 +LVPB: 1 +LVPMXSA: 0 +LVPMXSB: 0 +LdsBlockSizePerPadA: 0 +LdsBlockSizePerPadB: 0 +LdsBlockSizePerPadMXSA: 0 +LdsBlockSizePerPadMXSB: 0 +LdsBlockSizePerPadMetadata: 0 +LdsBytesNoAmax: 67584 +LdsInitCVgprs: false +LdsNumBytes: 67584 +LdsNumElementsAlignedA: 16384 +LdsNumElementsAlignedB: 16384 +LdsNumElementsAlignedMXSA: 512 +LdsNumElementsAlignedMXSB: 512 +LdsNumElementsAlignedMetadata: 0 +LdsOffsetA: 0 +LdsOffsetA_Blk: 33792 +LdsOffsetB: 17408 +LdsOffsetB_Blk: 51200 +LdsOffsetBias: 0 +LdsOffsetBiasGSU: 0 +LdsOffsetBiasNonGSU: 0 +LdsOffsetMXSA: 16384 +LdsOffsetMXSA_Blk: 50176 +LdsOffsetMXSB: 16896 +LdsOffsetMXSB_Blk: 50688 +LdsOffsetMetadata: 17408 +LdsOffsetMetadata_Blk: 51200 +LdsPadA: 0 +LdsPadB: 0 +LdsPadMXSA: 0 +LdsPadMXSB: 0 +LdsPadMetadata: 0 +LocalReadVectorWidth: -1 +LocalReadVectorWidthA: 32 +LocalReadVectorWidthB: 32 +LocalReadVectorWidthMXS: 1 +LocalReadVectorWidthMXSA: 1 +LocalReadVectorWidthMXSB: 1 +LocalSplitU: 1 +LocalSplitUReuseLDS: 1 +LocalWritePerMfma: -1 +LocalWriteUseSgprA: true +LocalWriteUseSgprB: true +LocalWriteUseSgprMXSA: true +LocalWriteUseSgprMXSB: true +LocalWriteUseSgprMetadata: false +LoopIters: 2 +LoopUnroll: 256 +MFMA_BF16_1K: false +MIArchVgpr: false +MIBlock: +- 16 +- 16 +- 128 +- 1 +- 1 +- 1 +MIInputPerThread: 32 +MIInputPerThreadA: 32 +MIInputPerThreadB: 32 +MIInputPerThreadMXSA: 1 +MIInputPerThreadMXSB: 1 +MIInputPerThreadMetadata: 32 +MIOutputVectorWidth: 4 +MIRegPerOut: 1 +MIWaveGroup: +- 2 +- 2 +MIWaveTile: +- 2 +- 2 +MIWaveTileA: 2 +MIWaveTileB: 2 +MIWaveTileMXSA: 2 +MIWaveTileMXSB: 2 +MIWaveTileMetadata: 0 +MXLoadInst: BufferLoad +MXScaleFormat: HostPreSwizzle +MacroTile0: 64 +MacroTile1: 64 +MacroTileA: 64 +MacroTileB: 64 +MacroTileMXSA: 64 +MacroTileMXSB: 64 +MagicDivAlg: 2 +MathClocksUnrolledLoop: 0 +MatrixInstB: 1 +MatrixInstBM: 1 +MatrixInstBN: 1 +MatrixInstK: 128 +MatrixInstM: 16 +MatrixInstN: 16 +MatrixInstruction: +- 16 +- 16 +- 128 +- 1 +MaxLDS: 163840 +MaxOccupancy: 40 +MbskPrefetchMethod: 0 +MfmaInitCVgprs: false +MinGRIncPerMfma: 3 +Multicast: false +NoLdsWriteCode: true +NoReject: false +NoTailLoop: false +NonDTLTailLoopA: true +NonDTLTailLoopB: true +NonDTLTailLoopMXSA: true +NonDTLTailLoopMXSB: true +NonDTLTailLoopMetadata: false +NonTemporal: -1 +NonTemporalA: 0 +NonTemporalB: 0 +NonTemporalC: 0 +NonTemporalD: 4 +NonTemporalE: 0 +NonTemporalGate: 0 +NonTemporalMXSA: 0 +NonTemporalMXSB: 0 +NonTemporalMetadata: 0 +NonTemporalWS: 0 +NonVolatile: -1 +NonVolatileA: 0 +NonVolatileB: 0 +NonVolatileC: 0 +NonVolatileD: 0 +NonVolatileE: 0 +NonVolatileGate: 0 +NonVolatileMXSA: 0 +NonVolatileMXSB: 0 +NonVolatileMetadata: 0 +NonVolatileWS: 0 +NumElementsPerBatchStore: 0 +NumElementsPerThread: 16 +NumGlobalWriteVectorsPerThread: 16 +NumLdsBlk: 2 +NumLoadsA: 4 +NumLoadsB: 4 +NumLoadsCoalescedA: 1 +NumLoadsCoalescedB: 1 +NumLoadsCoalescedMXSA: 1 +NumLoadsCoalescedMXSB: 1 +NumLoadsPerpendicularA: 4 +NumLoadsPerpendicularB: 4 +NumLoadsPerpendicularMXSA: 0 +NumLoadsPerpendicularMXSB: 0 +NumThreads: 256 +NumTotalPackedLoadsA: -1 +NumTotalPackedLoadsB: -1 +NumTotalPackedLoadsMXSA: 0 +NumTotalPackedLoadsMXSB: 0 +NumWaveSplitK: 1 +NumWaves: 4 +OptNoLoadLoop: 0 +PackedC0IdxChars: +- I +PackedC0IndicesX: +- 0 +PackedC1IdxChars: +- J +PackedC1IndicesX: +- 1 +PersistentXCCMapping: 0 +PostLoopStoreInNll: false +PrefetchAcrossPersistent: 0 +PrefetchGL2: 0 +PrefetchGlobalRead: 1 +PrefetchLocalRead: 1 +PreloadKernArgs: true +ProblemType: + Activation: true + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: hipblaslt_all + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: true + BiasDataTypeList: + - 0 + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 15 + DataTypeA: 15 + DataTypeAmaxD: 0 + DataTypeB: 15 + DataTypeE: 0 + DataTypeMXSA: 22 + DataTypeMXSB: 22 + DestDataType: 0 + F32XdlMathOp: 0 + FusedGemmA2A: false + GateResidualDataTypeList: [] + Gradient: false + GroupedGemm: true + HighPrecisionAccumulate: true + Index0: 0 + Index01A: 0 + Index01B: 1 + Index01MXSA: 0 + Index01MXSB: 1 + Index1: 1 + IndexAssignmentsA: + - 3 + - 0 + - 2 + IndexAssignmentsB: + - 3 + - 1 + - 2 + IndexAssignmentsLD: + - 4 + - 5 + - 6 + - 7 + IndexAssignmentsMXSA: + - 3 + - 0 + - 2 + IndexAssignmentsMXSB: + - 3 + - 1 + - 2 + IndexAssignmentsMetadata: + - 3 + - 0 + - 2 + IndexUnroll: 3 + IndexUnrollA: 0 + IndexUnrollB: 0 + IndexUnrollM: 0 + IndexUnrollMXSA: 0 + IndexUnrollMXSB: 0 + IndicesBatch: + - 2 + IndicesFree: + - 0 + - 1 + IndicesSummation: + - 3 + Int4EncodingA: Signed + MXBlockA: 32 + MXBlockB: 32 + MXBlockFreeA: 1 + MXBlockFreeB: 1 + MacDataTypeA: 15 + MacDataTypeB: 15 + MetadataLayout: 0 + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMXSA: [] + MirrorDimsMXSB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + ScaleBlockSizesA: [] + ScaleZeroPointA: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SetConstStrideGate: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: false + StridedBatched: true + SupportUserArgs: true + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: false + TLUB: false + TLUMXSA: false + TLUMXSB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: true + TransposeB: false + UseBeta: true + UseBias: 1 + UseE: false + UseGateResidual: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: Scalar + UseScaleAlphaVec: 1 + UseScaleCD: false + isMixMode: false +ReuseAcrossPersistent: 0 +SFCWGM: +- - 1 + - 1 +- - 1 + - 1 +ScheduleGROverBarrier: 0 +ScheduleGlobalRead: 1 +ScheduleIterAlg: 3 +ScheduleLocalWrite: 1 +SolutionIndex: 0 +SolutionNameMin: Cijk_Alik_Bljk_F8SS_MXAE8B32_MXBE8B32_BH_Bias_GG_HA_S_SAB_SAV_UserArgs_MT64x64x256_MI16x16x1_SN_LDSB0_AFC0_AG0_AFEM1_AFEM1_ASEM32_CB0_CD1_1_CLR1_CADS0_DTLA1_DTLB1_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS1_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA16_GRVWB16_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA950_ICIW0_IU1_K1_LDSTI0_LBSPPA0_LBSPPB0_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA0_LPB0_LPMXSA0_LPMXSB0_LPM0_LRVWn1_LWPMn1_MIAV0_MIWT2_2_MXLIBL_MXSFHPS_MO40_MGRIPM3_NTn1_NTA0_NTB0_NTC0_NTD4_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PGR1_PLR1_PKA1_SGROB0_SIA3_SS0_SU0_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW4_SK0_SKFTR0_SKXCCM0_SIP1_SGRO0_TDMI0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSMn1_ULSGRO0_USL0_USLMX0_UIOFGRO0_UPLRP0_USFGROn1_USI1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG32_8_1_WGM16_WGMXCC2_WGMXCCGn1 +SourceSwap: false +SpaceFillingAlgo: [] +StaggerU: 0 +StaggerUMapping: 0 +StaggerUStride: 256 +StinkyTofuRegisterAllocation: 0 +StorePriorityOpt: false +StoreRemapVectorWidth: 0 +StoreSwapAddr: false +StoreSyncOpt: 0 +StoreVectorWidth: 4 +StreamKAtomic: 0 +StreamKFixupTreeReduction: 0 +SubGroup0: 8 +SubGroup1: 32 +SubGroupA: 8 +SubGroupB: 32 +SubGroupMXSA: 8 +SubGroupMXSB: 32 +SuppressNoLoadLoop: false +SwInstructionPrefetch: true +SwapGlobalReadOrder: 0 +TDMInst: 0 +TDMIterateMode: 0 +TDMLoadWaveSync: false +TDMPlusLdsBuf: 0 +TDMSplit: false +TailloopInNll: false +TemporalHint: -1 +TemporalHintA: 0 +TemporalHintB: 0 +TemporalHintC: 0 +TemporalHintD: 0 +TemporalHintE: 0 +TemporalHintGate: 0 +TemporalHintMXSA: 0 +TemporalHintMXSB: 0 +TemporalHintMetadata: 0 +TemporalHintWS: 0 +ThreadTile: +- 1 +- 1 +ThreadTile0: 8 +ThreadTile1: 2 +ThreadTileA: 8 +ThreadTileB: 2 +ThreadTileMXSA: 8 +ThreadTileMXSB: 2 +TileProcessingStrategy: None +TransposeLDS: 1 +TransposeLDSMetadata: -1 +ULSGRODoubleG2L: 0 +UnrollLoopSwapGlobalReadOrder: 0 +UnrollMajorLDSA: true +UnrollMajorLDSB: true +UnrollMajorLDSMXSA: true +UnrollMajorLDSMXSB: true +Use64bShadowLimit: false +Use64bShadowLimitMX: false +UseCustomMainLoopSchedule: 0 +UseDirect32XEmulation: false +UseDot2F32XEmulation: false +UseDotInstruction: false +UseDualFMAC: false +UseF32XEmulation: false +UseGeneralizedNLCOneA: false +UseGeneralizedNLCOneB: false +UseGeneralizedNLCOneMXSA: false +UseGeneralizedNLCOneMXSB: false +UseGeneralizedNLCOneMetadata: false +UseInstOffsetForGRO: 0 +UseMFMAF32XEmulation: false +UsePLRPack: 0 +UseSgprForGRO: -1 +UseSubtileImpl: true +Valid: true +VectorStore: -1 +VectorWidthA: 1 +VectorWidthB: 1 +VectorWidthMXSA: 1 +VectorWidthMXSB: 1 +WaveSeparateGlobalReadA: 0 +WaveSeparateGlobalReadB: 0 +WaveSeparateGlobalReadMXSA: 0 +WaveSeparateGlobalReadMXSB: 0 +WaveSeparateGlobalReadMetadata: 0 +WaveSplitK: false +WavefrontSize: 64 +WorkAssignment: StaticGrid +WorkGroup: +- 32 +- 8 +- 1 +WorkGroupMapping: 16 +WorkGroupMappingXCC: 2 +WorkGroupMappingXCCGroup: -1 +WorkGroupReduction: false +WorkQueueStealing: 0 +WorkspaceCheck: +- 4 +- 0 +- 1 +_ABTilePairA: AB_B8 +_ABTilePairB: AB_B8 +_DepthU: 256 +_DepthUA: 256 +_DepthUB: 256 +_DepthUMXSA: 8 +_DepthUMXSB: 8 +_DepthUMetadata: 256 +_DualPeriodMXFP8: false +_GlobalAccumulation: MultipleBuffer +_HasTemporalHint: 0 +_PersistentLoop: false +_PersistentVectorEpilogueLds: false +_PrefetchAcrossPersistentEnabled: false +_ScalePeriodUA: 256 +_ScalePeriodUB: 256 +_ScaleReuseA: 1 +_ScaleReuseB: 1 +_ScheduleIterAlg: 3 +_SeparateEpilogueLds: false +_StinkyTofuOptLevel: 0 +_UseSgprForGRO: 0 +_VectorStore: 1 +_WorkspaceSizePerElemBias: 0 +_WorkspaceSizePerElemC: 4 +_inputVWA_was_auto: false +_inputVWB_was_auto: false +_staggerStrideShift: 0 +enableGLTrA: false +enableGLTrB: false +enableLDSTrA: false +enableLDSTrB: false +enableLDSTrMXSA: false +enableLDSTrMXSB: false +enableLDSTrMetadata: 0 +enableTDMA: false +enableTDMB: false +enableTDMMetadata: false +ldsNumBytesA: 16384 +ldsNumBytesB: 16384 +ldsNumBytesMXSA: 512 +ldsNumBytesMXSB: 512 +ldsNumBytesMetadata: 0 +numSubTiles: 1 +reorderGRInstForDTVA: false +reorderGRInstForDTVB: false +tailLoopOptA: false +tailLoopOptB: false +tailLoopOptMXSA: false +tailLoopOptMXSB: false diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.msgpack new file mode 100644 index 000000000000..3bccade5053e Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.predicates.msgpack b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.predicates.msgpack new file mode 100644 index 000000000000..0dd82b98d905 Binary files /dev/null and b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.predicates.msgpack differ diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.predicates.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.predicates.yaml new file mode 100644 index 000000000000..5d11e22a2a6e --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.predicates.yaml @@ -0,0 +1,117 @@ +problem: + type: And + value: + - type: OperationIdentifierEqual + value: Contraction_l_Ailk_Bljk_Cijk_Dijk + - type: BiasDataTypeWhiteList + value: + - 0 + - 4 + - type: GateResidualDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: + - 3 + - type: AmaxDCheck + value: false + - type: TypesEqual + value: + - Half + - Half + - Half + - Half + - Half + - Half + - type: HighPrecisionAccumulate + value: true + - type: Activation + value: None + - type: ActivationComputeType + value: Float + - type: ActivationNoGuard + value: false + - type: UseGradient + value: false + - type: UseBias + value: 0 + - type: UseE + value: false + - type: UseGateResidual + value: false + - type: DataTypeE + value: Half + - type: StridedBatched + value: true + - type: GroupedGemm + value: false + - type: UseScaleAB + value: '' + - type: UseScaleCD + value: false + - type: UseScaleAlphaVec + value: 0 + - type: Sparse + value: 0 + - type: F32XdlMathOp + value: Float + - type: SupportDeviceUserArguments + value: false + - type: SwizzleTensorA + value: false + - type: SwizzleTensorB + value: false + - type: FusedGemmA2A + value: false + - type: MXBlockA + value: 0 + - type: MXBlockB + value: 0 + - type: SynchronizerSizeCheck + index: 0 + value: + - 16 + - 16 + - 1 + - 4 + - 1 + - 1 + - type: WorkgroupNumberCheck + index: 0 + value: + - 16 + - 16 + - 1 + - type: LeadingFree0SizesGreaterOrEqual + value: 4 + - type: LeadingFree1SizesGreaterOrEqual + value: 1 + - type: KernelLanguageCompatible + value: Assembly + - type: BufferLoadOffsetLimitCheck + value: + ShiftPtrElemB: 0 + ShiftPtrElemA: 4 + DUorMT1: 16 + DUorMT0: 16 + - type: BufferLoadOffsetLimitCheck_Beta + value: 16 + - type: BufferStoreOffsetLimitCheck + value: 16 + - type: GlobalSplitUCheckMinK + value: + - 32 + - 1 + - type: WorkgroupMappingXCCCheck + value: + - 1 + - -1 +task: + type: And + value: + - type: WorkspaceCheck + - type: LaunchLimits +hardware: + type: AMDGPU + value: + type: Processor + value: gfx950 diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.record.txt b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.record.txt new file mode 100644 index 000000000000..6169177ba76a --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.record.txt @@ -0,0 +1,139 @@ +solutionName="Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSU1_GSUAMBSK_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA950_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM3_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGM8_WGMXCC1_WGMXCCGn1" +kernelName="Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA950_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM3_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1" +sizeMapping.waveNum=1 +sizeMapping.WavefrontSize=64 +sizeMapping.workGroup=[16,4,1] +sizeMapping.macroTile=[16,16,1] +sizeMapping.threadTile=[1,1] +sizeMapping.matrixInstruction=[16,16,16,1] +sizeMapping.grvwA=4 +sizeMapping.grvwB=4 +sizeMapping.gwvwC=4 +sizeMapping.gwvwD=4 +sizeMapping.depthU=16 +sizeMapping.staggerU=32 +sizeMapping.staggerUMapping=0 +sizeMapping.globalSplitUPGR=16 +sizeMapping.globalSplitU=1 +sizeMapping.staggerStrideShift=3 +sizeMapping.workGroupMapping=8 +sizeMapping.packBatchDims=0 +sizeMapping.magicDivAlg=2 +sizeMapping.tileProcessingStrategy="None" +sizeMapping.workAssignment="StaticGrid" +sizeMapping.streamKAtomic=0 +sizeMapping.prefetchAcrossPersistent=0 +sizeMapping.sourceKernel=false +sizeMapping.globalAccumulation=3 +sizeMapping.adaptiveGemmGSUA=0 +sizeMapping.workspaceSizePerElemC=4 +sizeMapping.workspaceSizePerElemBias=0 +sizeMapping.activationFused=true +sizeMapping.workGroupMappingXCC=1 +sizeMapping.workGroupMappingXCCGroup=-1 +sizeMapping.globalSplitUCoalesced=false +sizeMapping.globalSplitUWorkGroupMappingRoundRobin=false +sizeMapping.CUOccupancy=5 +sizeMapping.PrefetchGlobalRead=0 +sizeMapping.MathClocksUnrolledLoop=288 +sizeMapping.synchronizerSizePerWG=4 +sizeMapping.nonTemporalA=0 +sizeMapping.nonTemporalB=0 +sizeMapping.temporalHintA=0 +sizeMapping.temporalHintB=0 +sizeMapping.hasTemporalHint=false +sizeMapping.adaptiveGemmNTAB=0 +sizeMapping.customMainLoopScheduling=0 +sizeMapping.useSubtileImpl=false +sizeMapping.NonTemporalD=0 +sizeMapping.WaveSeparateGlobalReadA=0 +sizeMapping.WaveSeparateGlobalReadB=0 +sizeMapping.UnrollLoopSwapGlobalReadOrder=0 +sizeMapping.DirectToVgprA=false +sizeMapping.DirectToVgprB=false +sizeMapping.NumLoadsCoalescedA=1 +sizeMapping.NumLoadsCoalescedB=1 +sizeMapping.WaveGroup=[1,1] +sizeMapping.VectorWidthA=1 +sizeMapping.VectorWidthB=1 +sizeMapping.LocalSplitU=1 +sizeMapping.DirectToLdsA=false +sizeMapping.DirectToLdsB=false +sizeMapping.OneLDSBuffer=false +sizeMapping.SourceSwap=false +sizeMapping.UnrollMajorLDSA=0 +sizeMapping.UnrollMajorLDSB=0 +sizeMapping.enableLDSTrA=false +sizeMapping.enableLDSTrB=false +sizeMapping.ExpertSchedulingMode=0 +sizeMapping.clusterDim=[1,1] +sizeMapping.BufferLoad=true +sizeMapping.BufferStore=true +internalArgsSupport.version=3 +internalArgsSupport.persistentLoopArgsVersion=0 +internalArgsSupport.gsu=true +internalArgsSupport.wgm=true +internalArgsSupport.staggerU=true +internalArgsSupport.perTileExtraIters=false +internalArgsSupport.useUniversalArgs=true +internalArgsSupport.useSFC=false +problemType.operationIdentifier="Contraction_l_Ailk_Bljk_Cijk_Dijk" +problemType.transA=false +problemType.transB=false +problemType.computeInputTypeA="Half" +problemType.computeInputTypeB="Half" +problemType.aType="Half" +problemType.bType="Half" +problemType.cType="Half" +problemType.dType="Half" +problemType.eType="Half" +problemType.computeType="Float" +problemType.useBeta=true +problemType.useBias=0 +problemType.biasSrcWhiteList=[3] +problemType.useE=false +problemType.useGateResidual=false +problemType.gateResidualDataTypeWhiteList=[] +problemType.useScaleAB="" +problemType.useScaleCD=false +problemType.useScaleAlphaVec=0 +problemType.biasDataTypeWhiteList=[0,4] +problemType.highPrecisionAccumulate=true +problemType.useInitialStridesAB=false +problemType.useInitialStridesCD=false +problemType.stridedBatched=true +problemType.groupedGemm=false +problemType.useGradient=false +problemType.activationType="None" +problemType.activationArgLength=0 +problemType.activationComputeDataType="Float" +problemType.activationNoGuard=false +problemType.sparse=0 +problemType.f32XdlMathOp="Float" +problemType.supportDeviceUserArguments=false +problemType.outputAmaxD=false +problemType.swizzleTensorA=false +problemType.swizzleTensorB=false +problemType.metadataLayout=0 +problemType.mxBlockA=0 +problemType.mxBlockB=0 +problemType.mxBlockFreeA=1 +problemType.mxBlockFreeB=1 +problemType.mxTypeA="E8" +problemType.mxTypeB="E8" +problemType.mxScaleFormat=0 +problemType.fusedGemmA2A=false +problemType.stochasticRounding=false +problemPredicate=["And","OperationIdentifierEqual","BiasDataTypeWhiteList","GateResidualDataTypeWhiteList","BiasSrcWhiteList","AmaxDCheck","TypesEqual","HighPrecisionAccumulate","Activation","ActivationComputeType","ActivationNoGuard","UseGradient","UseBias","UseE","UseGateResidual","DataTypeE","StridedBatched","GroupedGemm","UseScaleAB","UseScaleCD","UseScaleAlphaVec","Sparse","F32XdlMathOp","SupportDeviceUserArguments","SwizzleTensorA","SwizzleTensorB","FusedGemmA2A","MXBlockA","MXBlockB","SynchronizerSizeCheck","WorkgroupNumberCheck","LeadingFree0SizesGreaterOrEqual","LeadingFree1SizesGreaterOrEqual","KernelLanguageCompatible","BufferLoadOffsetLimitCheck","BufferLoadOffsetLimitCheck_Beta","BufferStoreOffsetLimitCheck","GlobalSplitUCheckMinK","WorkgroupMappingXCCCheck"] +taskPredicate=["And","WorkspaceCheck","LaunchLimits"] +hardwarePredicate=["AMDGPU","Processor"] +customKernel.name="Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA950_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM3_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1" +customKernel.args=[{"semantic":"GemmInfo","type":"uint32"},{"semantic":"InternalArgs","type":"uint32"},{"semantic":"InternalArgs1","type":"uint32"},{"semantic":"NumWorkGroups","type":"uint32"},{"semantic":"SizeFree0","type":"uint32"},{"semantic":"SizeFree1","type":"uint32"},{"semantic":"SizeFree2","type":"uint32"},{"semantic":"SizeSum","type":"uint32"},{"semantic":"AddressD","type":"address"},{"semantic":"AddressC","type":"address"},{"semantic":"AddressA","type":"address"},{"semantic":"AddressB","type":"address"},{"semantic":"StrideD0","type":"uint32"},{"semantic":"StrideD1","type":"uint32"},{"semantic":"StrideC0","type":"uint32"},{"semantic":"StrideC1","type":"uint32"},{"semantic":"StrideA0","type":"uint32"},{"semantic":"StrideA1","type":"uint32"},{"semantic":"StrideB0","type":"uint32"},{"semantic":"StrideB1","type":"uint32"},{"semantic":"Alpha","type":"uint32"},{"semantic":"Beta","type":"uint32"},{"semantic":"AddressTD","type":"address"},{"semantic":"Synchronizer","type":"address"},{"semantic":"GSUSync","type":"uint32"}] +customKernel.macrotile=[16,16,16] +customKernel.threads=[64,1,1] +customKernel.grid=["TilesXYBatchGSU","One","One"] +customKernel.workspaceType="None" +customKernel.workspaceSizePerElemC=0 +customKernel.workspaceSizePerElemBias=0 +customKernel.generated=true +customKernel.uniformSummationOrder=false diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.yaml new file mode 100644 index 000000000000..76e8f369c81b --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_data/solution_records/gfx950-plain.yaml @@ -0,0 +1,568 @@ +1LDSBuffer: 0 +ActivationAlt: false +ActivationFuncCall: false +ActivationFused: true +AdaptiveGemm: 0 +AdaptiveGemmGSUA: 0 +AdaptiveGemmNTAB: 0 +AssertAIGreaterThanEqual: -1 +AssertAILessThanEqual: -1 +AssertFree0ElementMultiple: 1 +AssertFree1ElementMultiple: 1 +AssertSummationElementMultiple: 1 +AssignedDerivedParameters: true +AssignedProblemIndependentDerivedParameters: true +BaseName: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDN0zJfaQLd2kQv63oThWj7iikfbvmsZur9TRrDKvGnYo= +BufferLoad: true +BufferStore: true +CUOccupancy: 5 +ClusterBarrier: false +ClusterDim: +- 1 +- 1 +ClusterLocalRead: 1 +CodeObjectVersion: '4' +CompactLoopStore: false +ConvertAfterDS: false +CustomKernel: + args: + - semantic: GemmInfo + type: uint32 + - semantic: InternalArgs + type: uint32 + - semantic: InternalArgs1 + type: uint32 + - semantic: NumWorkGroups + type: uint32 + - semantic: SizeFree0 + type: uint32 + - semantic: SizeFree1 + type: uint32 + - semantic: SizeFree2 + type: uint32 + - semantic: SizeSum + type: uint32 + - semantic: AddressD + type: address + - semantic: AddressC + type: address + - semantic: AddressA + type: address + - semantic: AddressB + type: address + - semantic: StrideD0 + type: uint32 + - semantic: StrideD1 + type: uint32 + - semantic: StrideC0 + type: uint32 + - semantic: StrideC1 + type: uint32 + - semantic: StrideA0 + type: uint32 + - semantic: StrideA1 + type: uint32 + - semantic: StrideB0 + type: uint32 + - semantic: StrideB1 + type: uint32 + - semantic: Alpha + type: uint32 + - semantic: Beta + type: uint32 + - semantic: AddressTD + type: address + - semantic: Synchronizer + type: address + - semantic: GSUSync + type: uint32 + generated: true + grid: + - TilesXYBatchGSU + - One + - One + macrotile: + - 16 + - 16 + - 16 + name: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA950_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM3_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1 + threads: + - 64 + - 1 + - 1 + workspaceSizePerElemBias: 0 + workspaceSizePerElemC: 0 + workspaceType: None +DebugPersistentKernelLoopForever: false +DebugStreamK: 0 +DepthU: 16 +DirectToLds: 0 +DirectToLdsA: false +DirectToLdsB: false +DirectToLdsMetadata: 0 +DirectToVgprA: false +DirectToVgprB: false +DirectToVgprMXSA: false +DirectToVgprMXSB: false +DirectToVgprSparseMetadata: false +DisableXdlArbStall: true +DtlPlusLdsBuf: 0 +ESMRuntimeGate: false +EdgeType: ShiftPtr +EnableESM2TrackValuVsrc: true +EnableF32XdlMathOp: false +EnableMatrixInstruction: true +EnableStinkyTofuESM2: false +ExpandPointerSwap: false +ExpertSchedulingMode: 0 +ExtraLatencyForLR: 0 +ExtraMiLatencyLeft: -1 +ForceDisableShadowInit: false +ForceUnrollSubIter: false +GlobalReadPerMfma: 1.0 +GlobalReadVectorWidthA: 4 +GlobalReadVectorWidthB: 4 +GlobalSplitU: 1 +GlobalSplitUAlgorithm: MultipleBufferSingleKernel +GlobalSplitUCoalesced: false +GlobalSplitUWorkGroupMappingRoundRobin: false +GlobalWriteVectorWidth: 1 +GroupLoadStore: false +GuaranteeNoPartialA: false +GuaranteeNoPartialB: true +GuaranteeNoPartialMetadata: true +HalfPLR: 0 +HalfPLRA: false +HalfPLRB: false +ISA: +- 9 +- 5 +- 0 +InitCIterWmma: 0 +InnerUnroll: 1 +InterleaveAlpha: 0 +InternalSupportParams: + KernArgsVersion: 3 + PersistentLoopArgsVersion: 0 + SupportCustomStaggerU: true + SupportCustomWGM: true + SupportStreamKPerTileExtraIters: false + SupportUserGSU: true + UseSFC: false + UseUniversalArgs: true +Kernel: true +KernelLanguage: Assembly +KernelNameMin: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMBSK_GLS0_HPLR0_ISA950_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM3_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGMXCC1 +LDSSegInterleaveOffsets: null +LDSSegmentInterleave: 0 +LDSTrInst: false +LSCA: 16 +LSCB: 16 +LSPA: 16 +LSPB: 16 +LVCA: 4 +LVCB: 4 +LVPA: 4 +LVPB: 4 +LdsBlockSizePerPadA: 128 +LdsBlockSizePerPadB: 128 +LdsBlockSizePerPadMXSA: 0 +LdsBlockSizePerPadMXSB: 0 +LdsBlockSizePerPadMetadata: 0 +LdsBytesNoAmax: 2560 +LdsInitCVgprs: false +LdsNumBytes: 2560 +LdsNumElementsAlignedA: 640 +LdsNumElementsAlignedB: 640 +LdsNumElementsAlignedMXSA: 0 +LdsNumElementsAlignedMXSB: 0 +LdsNumElementsAlignedMetadata: 0 +LdsOffsetA: 0 +LdsOffsetA_Blk: 0 +LdsOffsetB: 640 +LdsOffsetB_Blk: 0 +LdsOffsetBias: 0 +LdsOffsetBiasGSU: 0 +LdsOffsetBiasNonGSU: 0 +LdsOffsetMXSA: 640 +LdsOffsetMXSA_Blk: 0 +LdsOffsetMXSB: 640 +LdsOffsetMXSB_Blk: 0 +LdsOffsetMetadata: 640 +LdsOffsetMetadata_Blk: 0 +LdsPadA: 16 +LdsPadB: 16 +LdsPadMXSA: 0 +LdsPadMXSB: 0 +LdsPadMetadata: -1 +LocalReadVectorWidth: -1 +LocalReadVectorWidthA: 4 +LocalReadVectorWidthB: 4 +LocalSplitU: 1 +LocalSplitUReuseLDS: 1 +LocalWritePerMfma: -1 +LocalWriteUseSgprA: false +LocalWriteUseSgprB: false +LocalWriteUseSgprMetadata: false +LoopIters: 1 +LoopUnroll: 16 +MFMA_BF16_1K: false +MIArchVgpr: false +MIBlock: +- 16 +- 16 +- 16 +- 1 +- 1 +- 1 +MIInputPerThread: 4 +MIInputPerThreadA: 4 +MIInputPerThreadB: 4 +MIInputPerThreadMetadata: 4 +MIOutputVectorWidth: 4 +MIRegPerOut: 1 +MIWaveGroup: +- 1 +- 1 +MIWaveTile: +- 1 +- 1 +MIWaveTileA: 1 +MIWaveTileB: 1 +MIWaveTileMetadata: 0 +MXLoadInst: BufferLoad +MXScaleFormat: NoSwizzle +MacroTile0: 16 +MacroTile1: 16 +MacroTileA: 16 +MacroTileB: 16 +MagicDivAlg: 2 +MathClocksUnrolledLoop: 288 +MatrixInstB: 1 +MatrixInstBM: 1 +MatrixInstBN: 1 +MatrixInstK: 16 +MatrixInstM: 16 +MatrixInstN: 16 +MatrixInstruction: +- 16 +- 16 +- 16 +- 1 +MaxLDS: 163840 +MaxOccupancy: 64 +MbskPrefetchMethod: 0 +MfmaInitCVgprs: false +MinGRIncPerMfma: 3 +Multicast: false +NoLdsWriteCode: false +NoReject: false +NoTailLoop: false +NonDTLTailLoopA: false +NonDTLTailLoopB: false +NonDTLTailLoopMetadata: false +NonTemporal: -1 +NonTemporalA: 0 +NonTemporalB: 0 +NonTemporalC: 0 +NonTemporalD: 0 +NonTemporalE: 0 +NonTemporalGate: 0 +NonTemporalMXSA: 0 +NonTemporalMXSB: 0 +NonTemporalMetadata: 0 +NonTemporalWS: 0 +NonVolatile: -1 +NonVolatileA: 0 +NonVolatileB: 0 +NonVolatileC: 0 +NonVolatileD: 0 +NonVolatileE: 0 +NonVolatileGate: 0 +NonVolatileMXSA: 0 +NonVolatileMXSB: 0 +NonVolatileMetadata: 0 +NonVolatileWS: 0 +NumElementsPerBatchStore: 0 +NumElementsPerThread: 4 +NumGlobalWriteVectorsPerThread: 4 +NumLdsBlk: 2 +NumLoadsA: 1 +NumLoadsB: 1 +NumLoadsCoalescedA: 1 +NumLoadsCoalescedB: 1 +NumLoadsPerpendicularA: 1 +NumLoadsPerpendicularB: 1 +NumThreads: 64 +NumTotalPackedLoadsA: -1 +NumTotalPackedLoadsB: -1 +NumWaveSplitK: 1 +NumWaves: 1 +OptNoLoadLoop: 0 +PackedC0IdxChars: +- I +PackedC0IndicesX: +- 0 +PackedC1IdxChars: +- J +PackedC1IndicesX: +- 1 +PersistentXCCMapping: 0 +PrefetchAcrossPersistent: 0 +PrefetchGL2: 0 +PrefetchGlobalRead: 0 +PrefetchLocalRead: 0 +PreloadKernArgs: false +ProblemType: + Activation: false + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: none + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: false + BiasDataTypeList: + - 0 + - 4 + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 4 + DataTypeA: 4 + DataTypeAmaxD: 0 + DataTypeB: 4 + DataTypeE: 4 + DataTypeMXSA: 22 + DataTypeMXSB: 22 + DestDataType: 4 + F32XdlMathOp: 0 + FusedGemmA2A: false + GateResidualDataTypeList: [] + Gradient: false + GroupedGemm: false + HighPrecisionAccumulate: true + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: + - 0 + - 3 + - 2 + IndexAssignmentsB: + - 3 + - 1 + - 2 + IndexAssignmentsLD: + - 4 + - 5 + - 6 + - 7 + IndexAssignmentsMetadata: + - 3 + - 0 + - 2 + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 0 + IndexUnrollM: 0 + IndicesBatch: + - 2 + IndicesFree: + - 0 + - 1 + IndicesSummation: + - 3 + MXBlockA: 0 + MXBlockB: 0 + MacDataTypeA: 4 + MacDataTypeB: 4 + MetadataLayout: 0 + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMXSA: [] + MirrorDimsMXSB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SetConstStrideGate: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: false + StridedBatched: true + SupportUserArgs: false + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: true + TLUB: false + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: false + TransposeB: false + UseBeta: true + UseBias: 0 + UseE: false + UseGateResidual: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: '' + UseScaleAlphaVec: 0 + UseScaleCD: false + isMixMode: false +ReuseAcrossPersistent: 0 +SFCWGM: +- - 1 + - 1 +- - 1 + - 1 +ScheduleGROverBarrier: 0 +ScheduleGlobalRead: 1 +ScheduleIterAlg: 3 +ScheduleLocalWrite: 1 +SolutionIndex: 0 +SolutionNameMin: Cijk_Ailk_Bljk_HHS_BH_MT16x16x16_MI16x16x1_SN_LDSB0_AFC0_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA4_GRVWB4_GSU1_GSUAMBSK_GSUC0_GSUWGMRR0_GLS0_HPLR0_ISA950_ICIW0_IU1_K1_LDSSI0_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPMn1_LRVWn1_LWPMn1_MIAV0_MIWT1_1_MXLIBL_MXSFNS_MO64_MGRIPM3_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTG0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVG0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL0_PAP0_PGL0_PGR0_PLR0_PKA0_RAP0_SGROB0_SIA3_SS0_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW4_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMLWS0_TDMPLB0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THG0_THMXSA0_THMXSB0_THM0_THWS0_TPSN_TLDS0_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGM8_WGMXCC1_WGMXCCGn1 +SourceSwap: false +SpaceFillingAlgo: [] +StaggerU: 32 +StaggerUMapping: 0 +StaggerUStride: 256 +StorePriorityOpt: false +StoreRemapVectorWidth: 0 +StoreSwapAddr: false +StoreSyncOpt: 0 +StoreVectorWidth: 4 +StreamKAtomic: 0 +StreamKFixupTreeReduction: 0 +SubGroup0: 4 +SubGroup1: 16 +SubGroupA: 4 +SubGroupB: 16 +SuppressNoLoadLoop: false +SwInstructionPrefetch: 1 +SwapGlobalReadOrder: 0 +SynchronizerSizeCheck: 1 +TDMInst: 0 +TDMIterateMode: 0 +TDMLoadWaveSync: false +TDMPlusLdsBuf: 0 +TDMSplit: false +TailloopInNll: false +TemporalHint: -1 +TemporalHintA: 0 +TemporalHintB: 0 +TemporalHintC: 0 +TemporalHintD: 0 +TemporalHintE: 0 +TemporalHintGate: 0 +TemporalHintMXSA: 0 +TemporalHintMXSB: 0 +TemporalHintMetadata: 0 +TemporalHintWS: 0 +ThreadTile: +- 1 +- 1 +ThreadTile0: 4 +ThreadTile1: 1 +ThreadTileA: 4 +ThreadTileB: 1 +TileProcessingStrategy: None +TransposeLDS: 0 +TransposeLDSMetadata: -1 +ULSGRODoubleG2L: 0 +UnrollLoopSwapGlobalReadOrder: 0 +UnrollMajorLDSA: 0 +UnrollMajorLDSB: 0 +Use64bShadowLimit: true +Use64bShadowLimitMX: false +UseCustomMainLoopSchedule: 0 +UseDirect32XEmulation: false +UseDot2F32XEmulation: false +UseDotInstruction: false +UseDualFMAC: false +UseF32XEmulation: false +UseGeneralizedNLCOneA: false +UseGeneralizedNLCOneB: false +UseGeneralizedNLCOneMXSA: false +UseGeneralizedNLCOneMXSB: false +UseGeneralizedNLCOneMetadata: false +UseInstOffsetForGRO: 0 +UseMFMAF32XEmulation: false +UsePLRPack: 0 +UseSgprForGRO: -1 +UseSubtileImpl: false +Valid: true +VectorStore: -1 +VectorWidthA: 1 +VectorWidthB: 1 +WaveSeparateGlobalReadA: 0 +WaveSeparateGlobalReadB: 0 +WaveSeparateGlobalReadMetadata: 0 +WaveSplitK: false +WavefrontSize: 64 +WorkAssignment: StaticGrid +WorkGroup: +- 16 +- 4 +- 1 +WorkGroupMapping: 8 +WorkGroupMappingXCC: 1 +WorkGroupMappingXCCGroup: -1 +WorkGroupReduction: false +WorkQueueStealing: 0 +WorkspaceCheck: +- 4 +- 0 +- 1 +_DepthU: 16 +_DepthUA: 16 +_DepthUB: 16 +_DepthUMetadata: 16 +_GlobalAccumulation: MultipleBufferSingleKernel +_HasTemporalHint: 0 +_PersistentLoop: false +_PersistentVectorEpilogueLds: false +_PrefetchAcrossPersistentEnabled: false +_ScheduleIterAlg: 3 +_SeparateEpilogueLds: false +_StinkyTofuOptLevel: 0 +_UseSgprForGRO: false +_VectorStore: 1 +_WorkspaceSizePerElemBias: 0 +_WorkspaceSizePerElemC: 4 +_inputVWA_was_auto: true +_inputVWB_was_auto: true +_staggerStrideShift: 3 +enableGLTrA: false +enableGLTrB: false +enableLDSTrA: false +enableLDSTrB: false +enableLDSTrMXSA: false +enableLDSTrMXSB: false +enableLDSTrMetadata: 0 +enableTDMA: false +enableTDMB: false +enableTDMMetadata: false +ldsNumBytesA: 640 +ldsNumBytesB: 640 +ldsNumBytesMXSA: 0 +ldsNumBytesMXSB: 0 +ldsNumBytesMetadata: 0 +numSubTiles: 1 +reorderGRInstForDTVA: false +reorderGRInstForDTVB: false +tailLoopOptA: true +tailLoopOptB: true diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_solution_state_catalog.py b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_solution_state_catalog.py new file mode 100644 index 000000000000..66e46da756b4 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_solution_state_catalog.py @@ -0,0 +1,35 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""The C++ solution-state tables have to stay the Python tables. + +solutionFromState copies the data-type names and the build chip ids by hand. +The installed tensilelite test tree does not contain SolutionState.cpp, so +this test compares Python to the checked-in lists. SolutionRecord reads the +C++ lists back through supportedBuildChipIds() and supportedDataTypeNames(). +""" + +from pathlib import Path + +import pytest + +from rocisa.enum import DataTypeEnum + +from tensilelite.Common.Architectures import GFX_CHIP_IDS + +pytestmark = pytest.mark.unit + + +def _lines(name): + path = Path(__file__).resolve().parent / "test_data" / name + return [line.strip() for line in path.read_text().splitlines() if line.strip()] + + +def test_build_chip_ids_match_python(): + python = [chip for chips in GFX_CHIP_IDS.values() for chip in chips] + assert _lines("catalog_chip_ids.txt") == python + + +def test_data_type_names_match_python(): + python = [item.name for item in DataTypeEnum if item.name != "Count"] + assert _lines("catalog_data_types.txt") == python diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_solution_state_record.py b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_solution_state_record.py new file mode 100644 index 000000000000..104c46a453e5 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/test_solution_state_record.py @@ -0,0 +1,317 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""The C++ record builder has to stay on the Python builder. + +solutionFromState and Solution.FromOriginalState are maintained together. +A field added on the Python side has to show up here, or this test fails. +Updating the checked-in record then fails SolutionRecord in tensilelite-tests +until the C++ builder emits the same lines. There is no Python binding: the +two tests share the files in test_data/solution_records and nothing else. + +Each fixture is a derived state .yaml, the map FromOriginalState reads, +with the files built from it: + + .msgpack the state, as solutionFromState reads it + .record.txt one line per field of the library record, and the + type sequence of each predicate tree + .predicates.yaml the three predicate trees with their values, in + the {type, index, value} form state() gives them + .predicates.msgpack the same trees for the C++ test, which has no + YAML parser + +SolutionRecord in tensilelite-tests builds the record from the msgpack, writes +it with MessagePackWriteSolutionData and compares the predicate trees node by +node with .predicates.msgpack, so a threshold, an operand or a feature +the C++ builder drops fails there. The fixtures are found by listing the +directory, so adding one needs no change to either test. + +The two plain-kernel states of each architecture are the custom.config blocks of +generated kernels. The others come from derive_solution_records.py, which runs a +real library logic file or benchmark config through the Python derivation; see +that module for the fixtures that edit a field the checked-in sources do not +reach. The fixtures together have to cover the cases listed in +test_fixtures_cover_the_predicate_families, so a swapped operand or an unread +feature cannot hide behind a symmetric kernel. +""" + +import contextlib +import io +import json +import copy +from pathlib import Path + +import msgpack +import pytest +import yaml + +from tensilelite.Common import state +from tensilelite.Contractions import ( + CustomKernel, + InternalArgsSupport, + ProblemPredicate, + ProblemType, + SizeMapping, + TaskPredicate, +) +from tensilelite import Hardware + +pytestmark = pytest.mark.unit + +_MX_SCALE_FORMAT = {"NoSwizzle": 0, "HostPreSwizzle": 1, "InMemorySwizzle": 2} + + +def records_dir(): + return Path(__file__).resolve().parent / "test_data" / "solution_records" + + +def fixture_names(): + """Every derived state in the directory: .yaml, not .predicates.yaml.""" + return sorted( + path.name[: -len(".yaml")] + for path in records_dir().glob("*.yaml") + if not path.name.endswith(".predicates.yaml") + ) + + +FIXTURES = fixture_names() + + +def _tags(predicate): + found = [] + + def walk(node): + if isinstance(node, dict) and "type" in node: + found.append(node["type"]) + value = node.get("value") + if isinstance(value, list): + for item in value: + walk(item) + elif isinstance(value, dict): + walk(value) + + walk(state(predicate)) + return found + + +def _json(value): + return json.dumps(value, separators=(",", ":")) + + +def _built(derived): + problem = ProblemType.FromOriginalState(derived["ProblemType"]) + problem.mxScaleFormat = _MX_SCALE_FORMAT.get(derived.get("MXScaleFormat", "NoSwizzle"), 0) + with contextlib.redirect_stdout(io.StringIO()): + hardware = Hardware.HardwarePredicate.FromHardware( + derived["ISA"], derived.get("CUCount"), derived.get("DeviceNames"), logicFile="" + ) + return { + "problem": problem, + "problemPredicate": ProblemPredicate.FromOriginalState(derived, problem), + "taskPredicate": TaskPredicate.FromOriginalState(derived, problem), + "hardware": hardware, + } + + +def library_record(derived): + """The library record FromOriginalState builds from one derived state.""" + built = _built(derived) + record = { + "solutionName": derived.get("SolutionNameMin", ""), + "kernelName": derived.get("KernelNameMin", ""), + "sizeMapping": state(SizeMapping.FromOriginalState(derived)), + "internalArgsSupport": state(InternalArgsSupport.FromOriginalState(derived)), + "problemType": state(built["problem"]), + "problemPredicate": _tags(built["problemPredicate"]), + "taskPredicate": _tags(built["taskPredicate"]), + "hardwarePredicate": _tags(built["hardware"]), + } + if "CustomKernel" in derived: + record["customKernel"] = state(CustomKernel.FromOriginalState(derived["CustomKernel"])) + return record + + +def predicate_payload(derived): + """Predicate arguments, not only their types. A threshold change fails this.""" + built = _built(derived) + return { + "problem": state(built["problemPredicate"]), + "task": state(built["taskPredicate"]), + "hardware": state(built["hardware"]), + } + + +def record_lines(record): + lines = [] + + def emit(prefix, value): + if isinstance(value, dict): + for key, item in value.items(): + emit(f"{prefix}.{key}" if prefix else str(key), item) + else: + lines.append(f"{prefix}={_json(value)}") + + for key, value in record.items(): + emit(key, value) + return lines + + +def _load_derived(name): + path = records_dir() / f"{name}.yaml" + with path.open() as handle: + return yaml.safe_load(handle) + + +@pytest.mark.parametrize("name", FIXTURES) +def test_python_builder_matches_checked_in_record(name): + derived = _load_derived(name) + expected = (records_dir() / f"{name}.record.txt").read_text().splitlines() + assert record_lines(library_record(derived)) == expected + + +@pytest.mark.parametrize("name", FIXTURES) +def test_predicate_arguments_match_checked_in_payload(name): + derived = _load_derived(name) + path = records_dir() / f"{name}.predicates.yaml" + with path.open() as handle: + expected = yaml.safe_load(handle) + assert predicate_payload(derived) == expected + + +@pytest.mark.parametrize("name", FIXTURES) +def test_msgpack_state_matches_yaml(name): + """solutionFromState reads the msgpack copy. It has to be this YAML.""" + derived = _load_derived(name) + packed = (records_dir() / f"{name}.msgpack").read_bytes() + assert msgpack.unpackb(packed, strict_map_key=False) == derived + + +@pytest.mark.parametrize("name", FIXTURES) +def test_predicate_msgpack_matches_yaml(name): + """The C++ test reads the msgpack copy of the predicate trees. It has to be the YAML.""" + with (records_dir() / f"{name}.predicates.yaml").open() as handle: + expected = yaml.safe_load(handle) + packed = (records_dir() / f"{name}.predicates.msgpack").read_bytes() + assert msgpack.unpackb(packed, strict_map_key=False) == expected + + +def _predicate_types(node): + found = [] + if isinstance(node, dict): + if "type" in node: + found.append(node["type"]) + for item in node.values(): + found.extend(_predicate_types(item)) + elif isinstance(node, list): + for item in node: + found.extend(_predicate_types(item)) + return found + + +def test_fixtures_cover_the_predicate_families(): + """A fixture set of symmetric kernels cannot tell A from B or M from N. + + Each case below is one way a C++ builder can be wrong and still match a + square, default-feature kernel. A fixture that stops covering a case fails + here, not silently in the C++ comparison. + """ + states = {name: _load_derived(name) for name in FIXTURES} + payloads = {name: predicate_payload(state) for name, state in states.items()} + + def problem(state): + return state["ProblemType"] + + def some(test): + return [name for name, state in states.items() if test(state)] + + assert some(lambda s: s["MacroTile0"] != s["MacroTile1"]), "no non-square tile" + assert some(lambda s: s["MacroTile0"] >= 128 or s["MacroTile1"] >= 128), "no large tile" + assert some( + lambda s: s["GlobalReadVectorWidthA"] != s["GlobalReadVectorWidthB"] + ), "GlobalReadVectorWidthA is always B" + assert len({s["DepthU"] for s in states.values()}) >= 3, "DepthU barely varies" + assert some(lambda s: problem(s)["DataTypeA"] != problem(s)["DataTypeB"]), "A type is B type" + assert len({tuple(s["ISA"]) for s in states.values()}) >= 4, "too few architectures" + + assert some(lambda s: problem(s).get("MXBlockA", 0) > 0), "no MX kernel" + assert some(lambda s: s.get("MXScaleFormat", "NoSwizzle") != "NoSwizzle"), "no MX scale format" + assert some( + lambda s: problem(s).get("MXBlockA", 0) > 0 + and problem(s).get("MXBlockFreeA", 1) != problem(s).get("MXBlockFreeB", 1) + ), "MXBlockFree is always 1 or the same for A and B" + assert some( + lambda s: len(problem(s).get("ScaleBlockSizesA", [])) > 1 + and problem(s).get("Int4EncodingA") == "UnsignedBias8" + ), "no int4 kernel with several group sizes and UnsignedBias8" + assert some(lambda s: problem(s).get("StochasticRounding")), "no stochastic rounding" + assert some(lambda s: s.get("TileProcessingStrategy") == "StreamK"), "no StreamK kernel" + assert some(lambda s: s.get("ReuseAcrossPersistent")), "no ReuseAcrossPersistent kernel" + assert some( + lambda s: s["InternalSupportParams"].get("PersistentLoopArgsVersion") == 2 + ), "no PersistentLoopArgsVersion 2 kernel" + assert some( + lambda s: s["InternalSupportParams"].get("SupportOccupancyProbe") + ), "no occupancy probe" + + types = {name: set(_predicate_types(payload)) for name, payload in payloads.items()} + assert any("Or" in kinds for kinds in types.values()), "no Or predicate" + assert any("ScaleBlockSizeA" in kinds for kinds in types.values()) + assert any("GlobalSplitUCheckMinK" in kinds for kinds in types.values()) + + +def test_mx_block_free_is_recorded_with_mx_predicates(): + derived = copy.deepcopy(_load_derived("gfx950-plain")) + derived["ProblemType"]["MXBlockA"] = 16 + derived["ProblemType"]["MXBlockFreeA"] = 7 + record = library_record(derived) + assert record["problemType"]["mxBlockFreeA"] == 7 + assert "MXBlockA" in record["problemPredicate"] + assert "MXBlockFreeA" in record["problemPredicate"] + + +def test_occupancy_probe_support_is_recorded(): + derived = copy.deepcopy(_load_derived("gfx950-plain")) + derived["InternalSupportParams"]["SupportOccupancyProbe"] = True + record = library_record(derived) + assert record["internalArgsSupport"]["occupancyProbe"] is True + + +@pytest.mark.parametrize("declared", (True, False)) +def test_custom_kernel_uniform_summation_order_is_recorded(declared): + derived = copy.deepcopy(_load_derived("gfx950-plain")) + derived["CustomKernel"]["uniformSummationOrder"] = declared + record = library_record(derived) + assert record["customKernel"]["uniformSummationOrder"] is declared + assert f"customKernel.uniformSummationOrder={_json(declared)}" in record_lines(record) + + +def test_custom_kernel_uniform_summation_order_defaults_to_false(): + derived = copy.deepcopy(_load_derived("gfx950-plain")) + assert "uniformSummationOrder" not in derived["CustomKernel"] + record = library_record(derived) + assert record["customKernel"]["uniformSummationOrder"] is False + + +def _pack(value): + return msgpack.packb(value, use_bin_type=True) + + +def write_fixture(name): + """The files built from .yaml. Each C++ comparison reads them back.""" + derived = _load_derived(name) + directory = records_dir() + (directory / f"{name}.msgpack").write_bytes(_pack(derived)) + lines = record_lines(library_record(derived)) + (directory / f"{name}.record.txt").write_text("\n".join(lines) + "\n") + payload = predicate_payload(derived) + (directory / f"{name}.predicates.yaml").write_text(yaml.safe_dump(payload, sort_keys=False)) + (directory / f"{name}.predicates.msgpack").write_bytes(_pack(payload)) + return len(lines) + + +if __name__ == "__main__": + # Regenerates the checked-in files after a deliberate Python change, or after + # derive_solution_records.py wrote a new .yaml. The C++ test then fails + # until solutionFromState builds the new record and predicate trees. + for name in fixture_names(): + print(name, write_fixture(name)) diff --git a/projects/hipblaslt/tensilelite/tests/CMakeLists.txt b/projects/hipblaslt/tensilelite/tests/CMakeLists.txt index 2e599c94205a..ac987acfc898 100644 --- a/projects/hipblaslt/tensilelite/tests/CMakeLists.txt +++ b/projects/hipblaslt/tensilelite/tests/CMakeLists.txt @@ -68,6 +68,31 @@ target_sources(tensilelite-tests # (see tensilelite/src/Tensile.cpp). The test also writes msgpack mapping # files directly via , so it has no meaning in YAML builds. if(NOT TENSILELITE_ENABLE_YAML) + # The solution-record fixtures live in TEST_DATA_DIR next to the other test + # data, so TestData::Instance() finds them from the binary's own location + # (build tree and installed tree alike). They are copied by a build step + # and not at configure time, so an edited fixture reaches the binary's data + # directory on the next build without a reconfigure. The target has no + # outputs, so it runs on every build; copying a handful of small files is + # cheap. + set(SOLUTION_STATE_RECORD_SOURCE_DIR + "${CMAKE_CURRENT_SOURCE_DIR}/../tensilelite/Tests/unit/test_data/solution_records") + set(SOLUTION_STATE_CATALOG_SOURCE_DIR + "${CMAKE_CURRENT_SOURCE_DIR}/../tensilelite/Tests/unit/test_data") + set(SOLUTION_STATE_FIXTURE_DIR "${TEST_DATA_DIR}/solution-state") + add_custom_target(tensilelite-solution-state-fixtures + COMMAND ${CMAKE_COMMAND} -E rm -rf "${SOLUTION_STATE_FIXTURE_DIR}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${SOLUTION_STATE_FIXTURE_DIR}/catalog" + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${SOLUTION_STATE_RECORD_SOURCE_DIR}" + "${SOLUTION_STATE_FIXTURE_DIR}/record" + COMMAND ${CMAKE_COMMAND} -E copy + "${SOLUTION_STATE_CATALOG_SOURCE_DIR}/catalog_chip_ids.txt" + "${SOLUTION_STATE_CATALOG_SOURCE_DIR}/catalog_data_types.txt" + "${SOLUTION_STATE_FIXTURE_DIR}/catalog" + COMMENT "Copying solution-state fixtures into ${SOLUTION_STATE_FIXTURE_DIR}") + add_dependencies(tensilelite-tests tensilelite-solution-state-fixtures) + target_sources(tensilelite-tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/MasterSolutionLibraryMapping_test.cpp" @@ -76,6 +101,8 @@ if(NOT TENSILELITE_ENABLE_YAML) # msgpack BIN, and mapRawBytes has no YAML implementation. "${CMAKE_CURRENT_SOURCE_DIR}/IndexedLibraryLoad_test.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MessagePackWriting_test.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/SolutionState_test.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/SolutionRecord_test.cpp" ) endif() diff --git a/projects/hipblaslt/tensilelite/tests/SolutionRecord_test.cpp b/projects/hipblaslt/tensilelite/tests/SolutionRecord_test.cpp new file mode 100644 index 000000000000..568f107902c1 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tests/SolutionRecord_test.cpp @@ -0,0 +1,761 @@ +// Copyright Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +// solutionFromState has to emit the record Solution.FromOriginalState builds. +// Each fixture in solution-state/record is a derived state (.msgpack) +// with what the Python builder makes of it: +// +// .record.txt one line per field of the library record, and the +// type sequence of each predicate tree +// .predicates.msgpack the problem, task and hardware predicate trees +// with their values, in the {type, index, value} +// form Python's state() gives them +// +// test_solution_state_record.py fails when the Python builder changes either +// file. This test fails after they are regenerated, until this builder emits +// the same record and the same trees. The trees are compared node by node, so +// a threshold, an operand or a whole predicate the builder drops names its path. +// The fixtures are the directory listing, not a list kept in this file. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include "TestData.hpp" + +#include + +using namespace TensileLite; + +namespace +{ + std::string jsonString(std::string const& text) + { + std::string out = "\""; + for(unsigned char c : text) + { + if(c == '\\' || c == '"') + out.push_back('\\'); + out.push_back(static_cast(c)); + } + out.push_back('"'); + return out; + } + + std::string jsonBool(bool value) + { + return value ? "true" : "false"; + } + + template + std::string jsonNumber(T value) + { + return std::to_string(value); + } + + std::string jsonInts(std::vector const& values) + { + std::string out = "["; + for(size_t i = 0; i < values.size(); ++i) + { + if(i != 0) + out += ','; + out += std::to_string(values[i]); + } + out += ']'; + return out; + } + + template + std::string jsonArray(std::array const& values) + { + std::string out = "["; + for(size_t i = 0; i < N; ++i) + { + if(i != 0) + out += ','; + out += std::to_string(values[i]); + } + out += ']'; + return out; + } + + std::string jsonDim(TensileLite::dim3 const& value, int count) + { + int const parts[] + = {static_cast(value.x), static_cast(value.y), static_cast(value.z)}; + std::string out = "["; + for(int i = 0; i < count; ++i) + { + if(i != 0) + out += ','; + out += std::to_string(parts[i]); + } + out += ']'; + return out; + } + + std::string jsonDataTypes(std::vector const& types) + { + std::string out = "["; + for(size_t i = 0; i < types.size(); ++i) + { + if(i != 0) + out += ','; + out += std::to_string(static_cast(types[i])); + } + out += ']'; + return out; + } + + std::string jsonType(rocisa::DataType type) + { + return jsonString(rocisa::toString(type)); + } + + template + void collect(Predicates::Predicate const& predicate, std::vector& out) + { + out.push_back(predicate.type()); + if(auto const* conj = dynamic_cast const*>(&predicate)) + { + for(auto const& term : conj->value) + collect(*term, out); + } + else if(auto const* disj = dynamic_cast const*>(&predicate)) + { + for(auto const& term : disj->value) + collect(*term, out); + } + else if(auto const* neg = dynamic_cast const*>(&predicate)) + { + collect(*neg->value, out); + } + } + + void collectHardware(Predicates::Predicate const& predicate, + std::vector& out) + { + out.push_back(predicate.type()); + if(auto const* gpu + = dynamic_cast const*>(&predicate)) + collect(*gpu->value, out); + } + + std::string jsonStrings(std::vector const& values) + { + std::string out = "["; + for(size_t i = 0; i < values.size(); ++i) + { + if(i != 0) + out += ','; + out += jsonString(values[i]); + } + out += ']'; + return out; + } + + std::string jsonArgs(std::vector const& args) + { + std::string out = "["; + for(size_t i = 0; i < args.size(); ++i) + { + if(i != 0) + out += ','; + // Contractions.CustomKernel writes `padding` only for an argument that has one. + out += "{"; + if(args[i].padding != 0) + out += "\"padding\":" + std::to_string(args[i].padding) + ","; + out += "\"semantic\":" + jsonString(toString(args[i].semantic)) + + ",\"type\":" + jsonString(toString(args[i].type)) + "}"; + } + out += ']'; + return out; + } + + std::string jsonGrid(vector3 const& grid) + { + CustomGridSize const parts[] = {grid.x, grid.y, grid.z}; + std::string out = "["; + for(int i = 0; i < 3; ++i) + { + if(i != 0) + out += ','; + out += jsonString(toString(parts[i])); + } + out += ']'; + return out; + } + + std::map recordLines(ContractionSolution const& solution) + { + std::map lines; + auto add = [&](std::string key, std::string value) { + lines.emplace(std::move(key), std::move(value)); + }; + + add("solutionName", jsonString(solution.solutionName)); + add("kernelName", jsonString(solution.kernelName)); + + auto const& size = solution.sizeMapping; + add("sizeMapping.waveNum", jsonNumber(size.waveNum)); + add("sizeMapping.WavefrontSize", jsonNumber(size.WavefrontSize)); + add("sizeMapping.workGroup", jsonDim(size.workGroupSize, 3)); + add("sizeMapping.macroTile", jsonDim(size.macroTile, 3)); + add("sizeMapping.threadTile", jsonDim(size.threadTile, 2)); + add("sizeMapping.matrixInstruction", jsonArray(size.matrixInstruction)); + add("sizeMapping.grvwA", jsonNumber(size.grvwA)); + add("sizeMapping.grvwB", jsonNumber(size.grvwB)); + add("sizeMapping.gwvwC", jsonNumber(size.gwvwC)); + add("sizeMapping.gwvwD", jsonNumber(size.gwvwD)); + add("sizeMapping.depthU", jsonNumber(size.depthU)); + add("sizeMapping.staggerU", jsonNumber(size.staggerU)); + add("sizeMapping.staggerUMapping", jsonNumber(size.staggerUMapping)); + add("sizeMapping.globalSplitUPGR", jsonNumber(size.globalSplitUPGR)); + add("sizeMapping.globalSplitU", jsonNumber(size.globalSplitU)); + add("sizeMapping.staggerStrideShift", jsonNumber(size.staggerStrideShift)); + add("sizeMapping.workGroupMapping", jsonNumber(size.workGroupMapping)); + add("sizeMapping.packBatchDims", jsonNumber(size.packBatchDims)); + add("sizeMapping.magicDivAlg", jsonNumber(size.magicDivAlg)); + add("sizeMapping.tileProcessingStrategy", + jsonString(toString(size.tileProcessingStrategy))); + add("sizeMapping.workAssignment", jsonString(toString(size.workAssignment))); + add("sizeMapping.streamKAtomic", jsonNumber(size.streamKAtomic)); + add("sizeMapping.prefetchAcrossPersistent", jsonNumber(size.prefetchAcrossPersistent)); + add("sizeMapping.sourceKernel", jsonBool(size.sourceKernel)); + add("sizeMapping.globalAccumulation", jsonNumber(size.globalAccumulation)); + add("sizeMapping.adaptiveGemmGSUA", jsonNumber(size.adaptiveGemmGSUA)); + add("sizeMapping.workspaceSizePerElemC", jsonNumber(size.workspaceSizePerElemC)); + add("sizeMapping.workspaceSizePerElemBias", jsonNumber(size.workspaceSizePerElemBias)); + add("sizeMapping.activationFused", jsonBool(size.activationFused)); + add("sizeMapping.workGroupMappingXCC", jsonNumber(size.workGroupMappingXCC)); + add("sizeMapping.workGroupMappingXCCGroup", jsonNumber(size.workGroupMappingXCCGroup)); + add("sizeMapping.globalSplitUCoalesced", jsonBool(size.globalSplitUCoalesced)); + add("sizeMapping.globalSplitUWorkGroupMappingRoundRobin", + jsonBool(size.globalSplitUWorkGroupMappingRoundRobin)); + add("sizeMapping.CUOccupancy", jsonNumber(size.CUOccupancy)); + add("sizeMapping.PrefetchGlobalRead", jsonNumber(size.PrefetchGlobalRead)); + add("sizeMapping.MathClocksUnrolledLoop", jsonNumber(size.MathClocksUnrolledLoop)); + add("sizeMapping.synchronizerSizePerWG", jsonNumber(size.synchronizerSizePerWG)); + add("sizeMapping.nonTemporalA", jsonNumber(size.nonTemporalA)); + add("sizeMapping.nonTemporalB", jsonNumber(size.nonTemporalB)); + add("sizeMapping.temporalHintA", jsonNumber(size.temporalHintA)); + add("sizeMapping.temporalHintB", jsonNumber(size.temporalHintB)); + add("sizeMapping.hasTemporalHint", jsonBool(size.hasTemporalHint)); + add("sizeMapping.adaptiveGemmNTAB", jsonNumber(size.adaptiveGemmNTAB)); + add("sizeMapping.customMainLoopScheduling", jsonNumber(size.customMainLoopScheduling)); + add("sizeMapping.useSubtileImpl", jsonBool(size.useSubtileImpl)); + add("sizeMapping.NonTemporalD", jsonNumber(size.NonTemporalD)); + add("sizeMapping.WaveSeparateGlobalReadA", jsonNumber(size.WaveSeparateGlobalReadA)); + add("sizeMapping.WaveSeparateGlobalReadB", jsonNumber(size.WaveSeparateGlobalReadB)); + add("sizeMapping.UnrollLoopSwapGlobalReadOrder", + jsonNumber(size.UnrollLoopSwapGlobalReadOrder)); + add("sizeMapping.DirectToVgprA", jsonBool(size.DirectToVgprA)); + add("sizeMapping.DirectToVgprB", jsonBool(size.DirectToVgprB)); + add("sizeMapping.NumLoadsCoalescedA", jsonNumber(size.NumLoadsCoalescedA)); + add("sizeMapping.NumLoadsCoalescedB", jsonNumber(size.NumLoadsCoalescedB)); + add("sizeMapping.WaveGroup", jsonArray(size.waveGroup)); + add("sizeMapping.VectorWidthA", jsonNumber(size.VectorWidthA)); + add("sizeMapping.VectorWidthB", jsonNumber(size.VectorWidthB)); + add("sizeMapping.LocalSplitU", jsonNumber(size.LocalSplitU)); + add("sizeMapping.DirectToLdsA", jsonBool(size.DirectToLdsA)); + add("sizeMapping.DirectToLdsB", jsonBool(size.DirectToLdsB)); + add("sizeMapping.OneLDSBuffer", jsonBool(size.OneLDSBuffer)); + add("sizeMapping.SourceSwap", jsonBool(size.SourceSwap)); + add("sizeMapping.UnrollMajorLDSA", jsonNumber(size.UnrollMajorLDSA)); + add("sizeMapping.UnrollMajorLDSB", jsonNumber(size.UnrollMajorLDSB)); + add("sizeMapping.enableLDSTrA", jsonBool(size.enableLDSTrA)); + add("sizeMapping.enableLDSTrB", jsonBool(size.enableLDSTrB)); + add("sizeMapping.ExpertSchedulingMode", jsonNumber(size.expertSchedulingMode)); + add("sizeMapping.clusterDim", jsonDim(size.clusterDim, 2)); + add("sizeMapping.BufferLoad", jsonBool(size.BufferLoad)); + add("sizeMapping.BufferStore", jsonBool(size.BufferStore)); + + auto const& args = solution.internalArgsSupport; + add("internalArgsSupport.version", jsonNumber(args.version)); + add("internalArgsSupport.persistentLoopArgsVersion", + jsonNumber(args.persistentLoopArgsVersion)); + add("internalArgsSupport.gsu", jsonBool(args.gsu)); + add("internalArgsSupport.wgm", jsonBool(args.wgm)); + add("internalArgsSupport.staggerU", jsonBool(args.staggerU)); + add("internalArgsSupport.perTileExtraIters", jsonBool(args.perTileExtraIters)); + // InternalArgsSupport.state() names the probe only when it is set. + if(args.occupancyProbe) + add("internalArgsSupport.occupancyProbe", jsonBool(args.occupancyProbe)); + add("internalArgsSupport.useUniversalArgs", jsonBool(args.useUniversalArgs)); + add("internalArgsSupport.useSFC", jsonBool(args.useSFC)); + + auto const& problem = solution.problemType; + add("problemType.operationIdentifier", jsonString(problem.operationIdentifier)); + add("problemType.transA", jsonBool(problem.transA)); + add("problemType.transB", jsonBool(problem.transB)); + add("problemType.computeInputTypeA", jsonType(problem.computeInputTypeA)); + add("problemType.computeInputTypeB", jsonType(problem.computeInputTypeB)); + add("problemType.aType", jsonType(problem.aType)); + add("problemType.bType", jsonType(problem.bType)); + add("problemType.cType", jsonType(problem.cType)); + add("problemType.dType", jsonType(problem.dType)); + add("problemType.eType", jsonType(problem.eType)); + add("problemType.computeType", jsonType(problem.computeType)); + add("problemType.useBeta", jsonBool(problem.useBeta)); + add("problemType.useBias", jsonNumber(problem.useBias)); + add("problemType.biasSrcWhiteList", jsonInts(problem.biasSrcWhiteList)); + add("problemType.useE", jsonBool(problem.useE)); + add("problemType.useGateResidual", jsonBool(problem.useGateResidual)); + add("problemType.gateResidualDataTypeWhiteList", + jsonDataTypes(problem.gateResidualDataTypeWhiteList)); + add("problemType.useScaleAB", jsonString(problem.useScaleAB)); + add("problemType.useScaleCD", jsonBool(problem.useScaleCD)); + add("problemType.useScaleAlphaVec", jsonNumber(problem.useScaleAlphaVec)); + add("problemType.biasDataTypeWhiteList", jsonDataTypes(problem.biasDataTypeWhiteList)); + add("problemType.highPrecisionAccumulate", jsonBool(problem.highPrecisionAccumulate)); + add("problemType.useInitialStridesAB", jsonBool(problem.useInitialStridesAB)); + add("problemType.useInitialStridesCD", jsonBool(problem.useInitialStridesCD)); + add("problemType.stridedBatched", jsonBool(problem.stridedBatched)); + add("problemType.groupedGemm", jsonBool(problem.groupedGemm)); + add("problemType.useGradient", jsonBool(problem.useGradient)); + add("problemType.activationType", jsonString(ToString(problem.activationType))); + add("problemType.activationArgLength", jsonNumber(problem.activationArgLength)); + add("problemType.activationComputeDataType", jsonType(problem.activationComputeDataType)); + add("problemType.activationNoGuard", jsonBool(problem.activationNoGuard)); + add("problemType.sparse", jsonNumber(problem.sparse)); + add("problemType.f32XdlMathOp", jsonType(problem.f32XdlMathOp)); + add("problemType.supportDeviceUserArguments", jsonBool(problem.supportDeviceUserArguments)); + add("problemType.outputAmaxD", jsonBool(problem.outputAmaxD)); + add("problemType.swizzleTensorA", jsonBool(problem.swizzleTensorA)); + add("problemType.swizzleTensorB", jsonBool(problem.swizzleTensorB)); + add("problemType.metadataLayout", jsonNumber(problem.metadataLayout)); + add("problemType.mxBlockA", jsonNumber(problem.mxBlockA)); + add("problemType.mxBlockB", jsonNumber(problem.mxBlockB)); + add("problemType.mxBlockFreeA", jsonNumber(problem.mxBlockFreeA)); + add("problemType.mxBlockFreeB", jsonNumber(problem.mxBlockFreeB)); + add("problemType.mxTypeA", jsonType(problem.mxTypeA)); + add("problemType.mxTypeB", jsonType(problem.mxTypeB)); + add("problemType.mxScaleFormat", jsonNumber(problem.mxScaleFormat)); + add("problemType.fusedGemmA2A", jsonBool(problem.fusedGemmA2A)); + add("problemType.stochasticRounding", jsonBool(problem.stochasticRounding)); + + std::vector tags; + collect(*solution.problemPredicate, tags); + add("problemPredicate", jsonStrings(tags)); + tags.clear(); + collect(*solution.taskPredicate, tags); + add("taskPredicate", jsonStrings(tags)); + tags.clear(); + collectHardware(*solution.hardwarePredicate, tags); + add("hardwarePredicate", jsonStrings(tags)); + + auto const& kernel = solution.customKernel; + add("customKernel.name", jsonString(kernel.name)); + add("customKernel.args", jsonArgs(kernel.args)); + add("customKernel.macrotile", jsonDim(kernel.macrotile, 3)); + add("customKernel.threads", jsonDim(kernel.threads, 3)); + add("customKernel.grid", jsonGrid(kernel.grid)); + add("customKernel.workspaceType", jsonString(toString(kernel.workspaceType))); + add("customKernel.workspaceSizePerElemC", jsonNumber(kernel.workspaceSizePerElemC)); + add("customKernel.workspaceSizePerElemBias", jsonNumber(kernel.workspaceSizePerElemBias)); + add("customKernel.generated", jsonBool(kernel.generated)); + add("customKernel.uniformSummationOrder", jsonBool(kernel.uniformSummationOrder)); + return lines; + } + + // Fixtures sit in the test data directory next to the binary, in the build + // tree and in an installed tree alike. A missing file names its path. + std::string fixturePath(std::string const& relative) + { + return TestData::Instance().file("solution-state/" + relative).string(); + } + + std::ifstream openFixture(std::string const& path, std::ios::openmode mode) + { + std::ifstream in(path, mode); + if(!in) + throw std::runtime_error("cannot open test fixture " + path + " (test data directory " + + TestData::Instance().dataDir().string() + ")"); + return in; + } + + std::vector readBytes(std::string const& path) + { + auto in = openFixture(path, std::ios::binary); + return std::vector(std::istreambuf_iterator(in), + std::istreambuf_iterator()); + } + + std::string readText(std::string const& path) + { + auto in = openFixture(path, std::ios::in); + std::ostringstream buffer; + buffer << in.rdbuf(); + return buffer.str(); + } + + std::map parseLines(std::string const& text) + { + std::map lines; + std::istringstream stream(text); + std::string line; + while(std::getline(stream, line)) + { + if(line.empty()) + continue; + auto split = line.find('='); + lines.emplace(line.substr(0, split), line.substr(split + 1)); + } + return lines; + } + + std::shared_ptr solutionOf(std::string const& name, + msgpack::object_handle& handle) + { + auto bytes = readBytes(fixturePath("record/" + name + ".msgpack")); + if(bytes.empty()) + throw std::runtime_error(name + ".msgpack is empty"); + handle = msgpack::unpack(bytes.data(), bytes.size()); + auto solution = solutionFromState(handle.get()); + if(!solution) + throw std::runtime_error("solutionFromState returned no solution for " + name); + return solution; + } + + void expectRecord(std::string const& name) + { + msgpack::object_handle handle; + auto solution = solutionOf(name, handle); + + auto expected = parseLines(readText(fixturePath("record/" + name + ".record.txt"))); + auto got = recordLines(*solution); + for(auto const& [key, value] : expected) + { + auto found = got.find(key); + EXPECT_TRUE(found != got.end()) << name << " has no C++ value for " << key; + if(found != got.end()) + EXPECT_EQ(found->second, value) << name << " " << key; + } + // Python records a custom kernel only for a state that has one. + bool const hasCustomKernel = expected.count("customKernel.name") != 0; + for(auto const& entry : got) + { + if(!hasCustomKernel && entry.first.rfind("customKernel.", 0) == 0) + continue; + EXPECT_TRUE(expected.count(entry.first) != 0) + << name << " records " << entry.first << " that the Python record has not"; + } + } + + std::string clip(std::string text) + { + constexpr size_t limit = 120; + if(text.size() > limit) + text = text.substr(0, limit) + "..."; + return text; + } + + std::string show(msgpack::object const& value) + { + std::ostringstream out; + out << value; + return clip(out.str()); + } + + bool isInteger(msgpack::object const& value) + { + return value.type == msgpack::type::POSITIVE_INTEGER + || value.type == msgpack::type::NEGATIVE_INTEGER; + } + + bool isString(msgpack::object const& value) + { + return value.type == msgpack::type::STR; + } + + std::string text(msgpack::object const& value) + { + return std::string(value.via.str.ptr, value.via.str.size); + } + + // The map a predicate node is, by key; empty when the keys are not strings. + std::map keyed(msgpack::object const& value, + std::string& problem) + { + std::map out; + for(uint32_t i = 0; i < value.via.map.size; ++i) + { + auto const& pair = value.via.map.ptr[i]; + if(!isString(pair.key)) + { + problem = "a map key is not a string: " + show(pair.key); + return {}; + } + out.emplace(text(pair.key), &pair.val); + } + return out; + } + + // " ()" for a node that is a predicate, so a path reads as a tree of + // predicates and not as bare indices. + std::string typeTag(msgpack::object const& value) + { + if(value.type != msgpack::type::MAP) + return ""; + std::string problem; + auto fields = keyed(value, problem); + auto type = fields.find("type"); + if(type == fields.end() || !isString(*type->second)) + return ""; + return ":" + text(*type->second); + } + + // The enum value of a DataType name, or -1. Python's state() carries a + // DataType as its enum value; the library writer encodes the name. + int dataTypeValue(std::string const& name) + { + for(int i = 0; i < static_cast(rocisa::DataType::Count); ++i) + if(rocisa::toString(static_cast(i)) == name) + return i; + return -1; + } + + // The two white lists hold DataTypes. Compares the C++ list, whichever + // form it is in, with the enum values Python wrote. + std::string dataTypeListMismatch(std::string const& path, + msgpack::object const& expected, + msgpack::object const& actual) + { + if(expected.type != msgpack::type::ARRAY || actual.type != msgpack::type::ARRAY) + return path + ": a DataType list is not an array"; + if(expected.via.array.size != actual.via.array.size) + return path + ": Python has " + std::to_string(expected.via.array.size) + + " types, C++ has " + std::to_string(actual.via.array.size) + "; Python " + + show(expected) + ", C++ " + show(actual); + for(uint32_t i = 0; i < expected.via.array.size; ++i) + { + auto const& want = expected.via.array.ptr[i]; + auto const& have = actual.via.array.ptr[i]; + auto const value + = isString(have) + ? dataTypeValue(text(have)) + : (have.type == msgpack::type::POSITIVE_INTEGER ? int(have.via.u64) : -1); + if(want.type != msgpack::type::POSITIVE_INTEGER || int(want.via.u64) != value) + return path + "[" + std::to_string(i) + "]: Python " + show(want) + ", C++ " + + show(have); + } + return ""; + } + + // The path and values of the first node where two trees differ, in the + // order of a depth-first walk with map keys sorted; empty when they are equal. + // Integers compare by value whatever width they were packed in, and a bool + // is not an integer. + std::string firstMismatch(std::string const& path, + msgpack::object const& expected, + msgpack::object const& actual) + { + auto differ = [&](std::string const& why) { return path + ": " + why; }; + auto values = [&] { return "Python " + show(expected) + ", C++ " + show(actual); }; + + if(isInteger(expected) && isInteger(actual)) + { + bool same = expected.type == actual.type + && (expected.type == msgpack::type::POSITIVE_INTEGER + ? expected.via.u64 == actual.via.u64 + : expected.via.i64 == actual.via.i64); + return same ? "" : differ(values()); + } + if(expected.type != actual.type) + return differ("different kinds: " + values()); + + switch(expected.type) + { + case msgpack::type::NIL: + return ""; + case msgpack::type::BOOLEAN: + return expected.via.boolean == actual.via.boolean ? "" : differ(values()); + case msgpack::type::FLOAT32: + case msgpack::type::FLOAT64: + return expected.via.f64 == actual.via.f64 ? "" : differ(values()); + case msgpack::type::STR: + return text(expected) == text(actual) ? "" : differ(values()); + case msgpack::type::ARRAY: + { + auto const common = std::min(expected.via.array.size, actual.via.array.size); + for(uint32_t i = 0; i < common; ++i) + { + auto const& item = expected.via.array.ptr[i]; + auto next = path + "[" + std::to_string(i) + typeTag(item) + "]"; + auto diff = firstMismatch(next, item, actual.via.array.ptr[i]); + if(!diff.empty()) + return diff; + } + // The common elements agree, so the longer side has the extra ones. + if(expected.via.array.size > actual.via.array.size) + { + auto const& extra = expected.via.array.ptr[common]; + return differ("C++ stops after " + std::to_string(common) + " of " + + std::to_string(expected.via.array.size) + " elements; Python has " + + "[" + std::to_string(common) + typeTag(extra) + "] " + show(extra)); + } + if(expected.via.array.size < actual.via.array.size) + { + auto const& extra = actual.via.array.ptr[common]; + return differ("Python stops after " + std::to_string(common) + " of " + + std::to_string(actual.via.array.size) + " elements; C++ has [" + + std::to_string(common) + typeTag(extra) + "] " + show(extra)); + } + return ""; + } + case msgpack::type::MAP: + { + std::string problem; + auto want = keyed(expected, problem); + auto got = keyed(actual, problem); + if(!problem.empty()) + return differ(problem); + auto const type = want.find("type"); + bool const dataTypes = type != want.end() && isString(*type->second) + && (text(*type->second) == "BiasDataTypeWhiteList" + || text(*type->second) == "GateResidualDataTypeWhiteList"); + for(auto const& [key, value] : want) + { + auto found = got.find(key); + if(found == got.end()) + return differ("C++ has no '" + key + "' (Python " + show(*value) + ")"); + auto diff = dataTypes && key == "value" + ? dataTypeListMismatch(path + "." + key, *value, *found->second) + : firstMismatch(path + "." + key, *value, *found->second); + if(!diff.empty()) + return diff; + } + for(auto const& [key, value] : got) + if(want.find(key) == want.end()) + return differ("Python has no '" + key + "' (C++ " + show(*value) + ")"); + return ""; + } + default: + return differ("unsupported kind in the fixture: " + values()); + } + } + + // The predicate trees of a built solution, encoded by the same traits the + // library writer uses, and compared with what Python's state() gave. + void expectPredicates(std::string const& name) + { + msgpack::object_handle handle; + auto solution = solutionOf(name, handle); + + auto const bytes = MessagePackWriteSolutionData(*solution); + auto const written + = msgpack::unpack(reinterpret_cast(bytes.data()), bytes.size()); + ASSERT_EQ(written.get().type, msgpack::type::MAP) << name; + std::string problem; + auto const fields = keyed(written.get(), problem); + ASSERT_TRUE(problem.empty()) << name << ": " << problem; + + auto const expectedBytes = readBytes(fixturePath("record/" + name + ".predicates.msgpack")); + ASSERT_FALSE(expectedBytes.empty()) << name << ".predicates.msgpack is empty"; + auto const expected = msgpack::unpack(expectedBytes.data(), expectedBytes.size()); + ASSERT_EQ(expected.get().type, msgpack::type::MAP) << name; + auto const wanted = keyed(expected.get(), problem); + ASSERT_TRUE(problem.empty()) << name << ": " << problem; + + for(auto const& [tree, key] : + std::map{{"problem", "problemPredicate"}, + {"task", "taskPredicate"}, + {"hardware", "hardwarePredicate"}}) + { + auto const have = fields.find(key); + auto const want = wanted.find(tree); + ASSERT_TRUE(have != fields.end()) << name << ": the written solution has no " << key; + ASSERT_TRUE(want != wanted.end()) << name << ": the fixture has no " << tree; + auto diff = firstMismatch(tree, *want->second, *have->second); + EXPECT_TRUE(diff.empty()) << name << " " << diff; + } + } + + // Every fixture in the data directory, by the name of its record file. + std::vector fixtureNames() + { + std::string const suffix = ".record.txt"; + std::vector names; + for(auto const& path : TestData::Instance().glob("solution-state/record/*" + suffix)) + { + auto file = path.filename().string(); + names.push_back(file.substr(0, file.size() - suffix.size())); + } + std::sort(names.begin(), names.end()); + return names; + } + + std::vector fileLines(std::string const& path) + { + std::vector lines; + auto in = openFixture(path, std::ios::in); + std::string line; + while(std::getline(in, line)) + { + if(!line.empty()) + lines.push_back(line); + } + return lines; + } + +} + +TEST(SolutionRecord, CatalogMatchesSolutionState) +{ + auto chips = supportedBuildChipIds(); + auto names = supportedDataTypeNames(); + EXPECT_EQ(chips, fileLines(fixturePath("catalog/catalog_chip_ids.txt"))); + EXPECT_EQ(names, fileLines(fixturePath("catalog/catalog_data_types.txt"))); +} + +TEST(SolutionRecord, FixturesAreComplete) +{ + auto const names = fixtureNames(); + ASSERT_FALSE(names.empty()) << "no fixture under " << fixturePath("record") + << " (test data directory " + << TestData::Instance().dataDir().string() << ")"; + for(auto const& name : names) + for(auto const* extension : + {".yaml", ".msgpack", ".record.txt", ".predicates.yaml", ".predicates.msgpack"}) + { + auto const path = fixturePath("record/" + name + extension); + EXPECT_TRUE(std::filesystem::is_regular_file(path)) << "missing fixture file " << path; + } +} + +TEST(SolutionRecord, MatchesThePythonBuilder) +{ + for(auto const& name : fixtureNames()) + { + SCOPED_TRACE(name); + expectRecord(name); + } +} + +TEST(SolutionRecord, PredicateValuesMatchThePythonBuilder) +{ + for(auto const& name : fixtureNames()) + { + SCOPED_TRACE(name); + expectPredicates(name); + } +} diff --git a/projects/hipblaslt/tensilelite/tests/SolutionState_test.cpp b/projects/hipblaslt/tensilelite/tests/SolutionState_test.cpp new file mode 100644 index 000000000000..427bda2f5aa3 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tests/SolutionState_test.cpp @@ -0,0 +1,794 @@ +// Copyright Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +// solutionFromState builds a TensileLite solution record from a derived +// solution state. These tests pack that state with msgpack::packer. They do +// not read assembly kernels. + +#include + +#include +#include +#include +#include +#include +#include + +#include + +// The headers declaring ContractionProblemGemm come before , which +// reaches the Windows SDK headers that define the enum name CONST as a macro. +#include +#include +#include +#include +#include +#include + +#include + +#include + +using namespace TensileLite; + +namespace +{ + using ProblemPredicate = Predicates::Predicate; + using ProblemAnd = Predicates::And; + using Packer = msgpack::packer; + + class StateMap + { + public: + void boolean(std::string key, bool value) + { + add(std::move(key), [value](Packer& packer, msgpack::sbuffer&) { packer.pack(value); }); + } + + void integer(std::string key, int64_t value) + { + add(std::move(key), [value](Packer& packer, msgpack::sbuffer&) { packer.pack(value); }); + } + + // The assembler writes an empty string as float64 0. msgpack-cxx packs + // an integral double as an integer, so write the note's encoding. + void emptyString(std::string key) + { + add(std::move(key), [](Packer&, msgpack::sbuffer& buffer) { + char const bytes[] = {char(0xcb), 0, 0, 0, 0, 0, 0, 0, 0}; + buffer.write(bytes, sizeof(bytes)); + }); + } + + void text(std::string key, std::string value) + { + add(std::move(key), [value](Packer& packer, msgpack::sbuffer&) { packer.pack(value); }); + } + + void integers(std::string key, std::vector values) + { + add(std::move(key), [values](Packer& packer, msgpack::sbuffer&) { + packer.pack_array(values.size()); + for(auto value : values) + packer.pack(value); + }); + } + + // Msgpack integer keys, the form the assembler writes for {0: 1}. + void indexMap(std::string key, std::vector> entries) + { + add(std::move(key), [entries](Packer& packer, msgpack::sbuffer&) { + packer.pack_map(entries.size()); + for(auto const& entry : entries) + { + packer.pack(entry.first); + packer.pack(entry.second); + } + }); + } + + void nested(std::string key, StateMap const& child) + { + add(std::move(key), [child](Packer& packer, msgpack::sbuffer& buffer) { + child.write(packer, buffer); + }); + } + + void maps(std::string key, std::vector children) + { + add(std::move(key), [children](Packer& packer, msgpack::sbuffer& buffer) { + packer.pack_array(children.size()); + for(auto const& child : children) + child.write(packer, buffer); + }); + } + + void textKeyedNumbers(std::string key, + std::vector> entries) + { + add(std::move(key), [entries](Packer& packer, msgpack::sbuffer&) { + packer.pack_map(entries.size()); + for(auto const& entry : entries) + { + packer.pack(entry.first); + packer.pack(entry.second); + } + }); + } + + msgpack::object_handle unpack() const + { + msgpack::sbuffer buffer; + Packer packer(buffer); + write(packer, buffer); + return msgpack::unpack(buffer.data(), buffer.size()); + } + + private: + using Field = std::pair>; + + void add(std::string key, std::function writeValue) + { + m_fields.emplace_back(std::move(key), std::move(writeValue)); + } + + void write(Packer& packer, msgpack::sbuffer& buffer) const + { + packer.pack_map(m_fields.size()); + for(auto const& field : m_fields) + { + packer.pack(field.first); + field.second(packer, buffer); + } + } + + std::vector m_fields; + }; + + struct StateOptions + { + bool batched = true; + bool mirrorSummationA = false; + bool swappedSummation = false; + bool dimAssertions = false; + bool assertSizeAsScalar = false; + bool emptySizeEqual = false; + bool skippedSizeEqual = false; + char const* dataType = "Half"; + char const* dataTypeA = nullptr; + char const* dataTypeB = nullptr; + bool stochasticRounding = false; + std::vector scaleBlockSizesA; + char const* tileStrategy = nullptr; + char const* workAssignment = nullptr; + std::vector clusterDim; + int kernArgsVersion = 2; + int persistentLoopArgsVersion = -1; + bool sizeBounds = false; + bool badIdealKey = false; + bool oneLdsBuffer = false; + bool bufferLoadOff = false; + std::vector isa; + char const* deviceName = nullptr; + // A hand-written CustomKernel block. uniformSummationOrder is left out + // of it when negative, and written as false or true otherwise. + bool customKernel = false; + int customUniformSummationOrder = -1; + }; + + StateMap problemType(StateOptions const& options) + { + StateMap type; + if(options.swappedSummation) + { + // Summation 4 is listed before 3, so bound order is not problem-index + // order. 4 is mirrored on A and 3 is mirrored on B. + type.integer("TotalIndices", 5); + type.integers("IndicesSummation", {4, 3}); + type.integer("NumIndicesC", 3); + type.integers("IndicesBatch", {2}); + type.integers("IndicesFree", {0, 1}); + type.integers("IndexAssignmentsA", {0, 4, 3, 2}); + type.integers("IndexAssignmentsB", {4, 1, 3, 2}); + type.integers("MirrorDimsA", {4}); + type.integers("MirrorDimsB", {3}); + } + else + { + type.integer("TotalIndices", 4); + type.integers("IndicesSummation", {3}); + type.integer("NumIndicesC", 3); + type.integers("IndicesBatch", {2}); + type.integers("IndicesFree", {0, 1}); + type.integers("IndexAssignmentsA", {0, 3, 2}); + type.integers("IndexAssignmentsB", {3, 1, 2}); + if(options.mirrorSummationA) + type.integers("MirrorDimsA", {3}); + } + type.text("DataType", options.dataType); + type.text("DestDataType", options.dataType); + type.text("ComputeDataType", "Float"); + if(options.dataTypeA) + type.text("DataTypeA", options.dataTypeA); + if(options.dataTypeB) + type.text("DataTypeB", options.dataTypeB); + if(options.stochasticRounding) + type.boolean("StochasticRounding", true); + if(!options.scaleBlockSizesA.empty()) + type.integers("ScaleBlockSizesA", options.scaleBlockSizesA); + if(options.batched) + type.boolean("Batched", true); + type.text("ActivationComputeDataType", "Float"); + type.boolean("TransposeA", false); + type.boolean("TransposeB", true); + type.boolean("ComplexConjugateA", false); + type.boolean("ComplexConjugateB", false); + type.boolean("HighPrecisionAccumulate", true); + type.boolean("TLUA", false); + type.boolean("TLUB", false); + type.boolean("SwizzleTensorA", false); + type.boolean("SwizzleTensorB", false); + type.boolean("FusedGemmA2A", false); + type.boolean("UseBeta", true); + type.boolean("StridedBatched", true); + return type; + } + + StateMap internalSupport(StateOptions const& options) + { + StateMap support; + support.integer("KernArgsVersion", options.kernArgsVersion); + if(options.persistentLoopArgsVersion >= 0) + support.integer("PersistentLoopArgsVersion", options.persistentLoopArgsVersion); + support.boolean("SupportUserGSU", true); + support.boolean("SupportCustomWGM", false); + support.boolean("SupportCustomStaggerU", true); + support.boolean("UseUniversalArgs", true); + support.boolean("UseSFC", false); + return support; + } + + // A small derived solution state: the map Solution.FromOriginalState reads. + // SolutionNameMin is float 0, which the metadata note uses for an empty + // string. _GlobalAccumulation is the string "null", which the note uses + // for nil. + StateMap derivedState(StateOptions options = {}) + { + StateMap state; + state.emptyString("SolutionNameMin"); + state.text("KernelNameMin", "small_kernel"); + state.integer("SolutionIndex", 7); + state.boolean("DebugKernel", false); + state.nested("ProblemType", problemType(options)); + state.integers("ISA", + options.isa.empty() ? std::vector{9, 4, 2} : options.isa); + if(options.deviceName) + state.text("DeviceNames", options.deviceName); + state.text("_GlobalAccumulation", "null"); + state.integer("PrefetchGlobalRead", 2); + state.boolean("EnableMatrixInstruction", false); + state.integer("ThreadTile0", 4); + state.integer("ThreadTile1", 2); + state.integer("NumElementsPerBatchStore", 0); + state.integer("NumElementsPerThread", 4); + state.integer("NumThreads", 64); + state.integer("WavefrontSize", 64); + state.integers("SpaceFillingAlgo", {}); + state.integer("WorkGroupMapping", 5); + state.boolean("DirectToVgprA", false); + state.boolean("DirectToVgprB", false); + state.boolean("DirectToLdsA", false); + state.boolean("DirectToLdsB", false); + state.integers("WorkGroup", {64, 1, 1}); + state.integer("MacroTile0", 32); + state.integer("MacroTile1", 8); + state.integers("ThreadTile", {4, 2, 1}); + state.integers("MatrixInstruction", {16, 16, 16, 1}); + state.integer("GlobalReadVectorWidthA", 4); + state.integer("GlobalReadVectorWidthB", 2); + state.integer("StoreVectorWidth", 4); + state.integer("DepthU", 16); + state.integer("GlobalSplitU", 1); + state.text("KernelLanguage", "Assembly"); + state.integer("_WorkspaceSizePerElemC", 4); + state.integer("_WorkspaceSizePerElemBias", 0); + state.boolean("ActivationFused", true); + state.integer("WorkGroupMappingXCC", 4); + state.integer("WorkGroupMappingXCCGroup", 8); + state.boolean("GlobalSplitUCoalesced", false); + state.boolean("GlobalSplitUWorkGroupMappingRoundRobin", false); + state.integer("CUOccupancy", 3); + state.integer("MathClocksUnrolledLoop", 11); + state.integer("NonTemporalA", 0); + state.integer("NonTemporalB", 0); + state.integer("UseCustomMainLoopSchedule", 0); + state.integer("NonTemporalD", 0); + state.integer("WaveSeparateGlobalReadA", 0); + state.integer("WaveSeparateGlobalReadB", 0); + state.integer("UnrollLoopSwapGlobalReadOrder", 0); + state.integer("NumLoadsCoalescedA", 1); + state.integer("NumLoadsCoalescedB", 1); + state.integers("MIWaveGroup", {1, 1}); + state.integer("VectorWidthA", 1); + state.integer("VectorWidthB", 1); + state.integer("LocalSplitU", 1); + state.integers("ClusterDim", + options.clusterDim.empty() ? std::vector{1, 1, 1} + : options.clusterDim); + state.integers("PackedC0IndicesX", {}); + state.nested("InternalSupportParams", internalSupport(options)); + if(options.tileStrategy) + state.text("TileProcessingStrategy", options.tileStrategy); + if(options.workAssignment) + state.text("WorkAssignment", options.workAssignment); + if(options.oneLdsBuffer) + state.integer("1LDSBuffer", 1); + if(options.bufferLoadOff) + state.integer("BufferLoad", 0); + if(options.sizeBounds) + { + StateMap lower; + lower.text("tag", "SizeGreaterThan"); + lower.integer("index", 0); + lower.integer("value", 31); + StateMap upper; + upper.text("tag", "SizeLessThan"); + upper.integer("index", 0); + upper.integer("value", 129); + state.maps("CustomKernelSizePredicate", {lower, upper}); + } + if(options.customKernel) + { + StateMap kernel; + kernel.text("name", "hand_written"); + kernel.maps("args", {}); + kernel.integers("macrotile", {32, 8, 16}); + kernel.integers("threads", {64, 1, 1}); + kernel.maps("grid", {}); + if(options.customUniformSummationOrder >= 0) + kernel.boolean("uniformSummationOrder", options.customUniformSummationOrder != 0); + state.nested("CustomKernel", kernel); + } + if(options.badIdealKey) + state.textKeyedNumbers("Ideals", {{"nope", 1.0}}); + + if(options.assertSizeAsScalar) + state.integer("AssertSizeEqual", 1); + else if(options.emptySizeEqual) + state.indexMap("AssertSizeEqual", {}); + else if(options.skippedSizeEqual) + state.indexMap("AssertSizeEqual", {{0, -1}}); + else if(options.dimAssertions) + { + state.indexMap("AssertSizeEqual", {{0, 32}, {1, -1}}); + state.indexMap("AssertSizeGreaterThan", {{1, 8}}); + state.indexMap("AssertSizeLessThan", {{3, 16385}}); + state.indexMap("AssertStrideAEqual", {{0, 1}}); + state.indexMap("AssertStrideBEqual", {{0, 1}, {1, 64}}); + state.indexMap("AssertStrideCEqual", {{1, 4}}); + state.indexMap("AssertStrideDEqual", {{2, 7}}); + } + return state; + } + + StateOptions with(void (*edit)(StateOptions&)) + { + StateOptions options; + edit(options); + return options; + } + + std::shared_ptr build(StateOptions options = {}) + { + auto handle = derivedState(options).unpack(); + return solutionFromState(handle.get()); + } + + template + Predicates::And const& + conjunction(std::shared_ptr> const& predicate) + { + auto const* all = dynamic_cast const*>(predicate.get()); + EXPECT_NE(all, nullptr); + if(!all) + throw std::runtime_error("predicate is not a conjunction"); + return *all; + } + + template + Predicate const* onlyDirect(std::shared_ptr const& root) + { + Predicate const* found = nullptr; + for(auto const& child : conjunction(root).value) + { + if(auto cast = std::dynamic_pointer_cast(child)) + { + EXPECT_EQ(found, nullptr) << Predicate::Type(); + found = cast.get(); + } + } + return found; + } + + template + void expectIndexValue(Predicate const* predicate, size_t index, size_t value) + { + ASSERT_NE(predicate, nullptr) << Predicate::Type(); + EXPECT_EQ(predicate->type(), Predicate::Type()); + EXPECT_EQ(predicate->index, index); + EXPECT_EQ(predicate->value, value); + } +} + +TEST(SolutionState, RecordFromDerivedState) +{ + auto solution = build(); + ASSERT_NE(solution, nullptr); + + EXPECT_EQ(solution->solutionName, ""); + EXPECT_EQ(solution->kernelName, "small_kernel"); + EXPECT_FALSE(solution->debugKernel); + EXPECT_EQ(solution->index, 7); + EXPECT_EQ(solution->libraryLogicIndex, 7); + EXPECT_TRUE(solution->ideals.empty()); + EXPECT_EQ(solution->linearModel.slope, 1.0); + EXPECT_EQ(solution->linearModel.intercept, 0.0); + EXPECT_EQ(solution->linearModel.max, 1000.0); + EXPECT_TRUE(solution->customKernel.name.empty()); + + auto const& type = solution->problemType; + EXPECT_EQ(type.operationIdentifier, "Contraction_l_Ailk_Bljk_Cijk_Dijk"); + EXPECT_FALSE(type.transA); + EXPECT_TRUE(type.transB); + EXPECT_EQ(type.aType, rocisa::DataType::Half); + EXPECT_EQ(type.bType, rocisa::DataType::Half); + EXPECT_EQ(type.cType, rocisa::DataType::Half); + EXPECT_EQ(type.dType, rocisa::DataType::Half); + EXPECT_EQ(type.eType, rocisa::DataType::Half); + EXPECT_EQ(type.computeInputTypeA, rocisa::DataType::Half); + EXPECT_EQ(type.computeInputTypeB, rocisa::DataType::Half); + EXPECT_EQ(type.computeType, rocisa::DataType::Float); + EXPECT_TRUE(type.highPrecisionAccumulate); + EXPECT_TRUE(type.useBeta); + EXPECT_TRUE(type.stridedBatched); + EXPECT_FALSE(type.groupedGemm); + EXPECT_EQ(type.activationType, ActivationType::None); + EXPECT_EQ(type.activationComputeDataType, rocisa::DataType::Float); + EXPECT_EQ(type.f32XdlMathOp, rocisa::DataType::Float); + EXPECT_EQ(type.mxBlockA, 0); + EXPECT_EQ(type.mxTypeA, rocisa::DataType::Float); + EXPECT_EQ(type.mxScaleFormat, 0); + + auto const& size = solution->sizeMapping; + EXPECT_EQ(size.waveNum, 1); + EXPECT_EQ(size.workGroupSize, TensileLite::dim3(64, 1, 1)); + EXPECT_EQ(size.macroTile, TensileLite::dim3(32, 8, 1)); + EXPECT_EQ(size.threadTile, TensileLite::dim3(4, 2, 1)); + EXPECT_EQ(size.matrixInstruction, (std::array{16, 16, 16, 1})); + EXPECT_EQ(size.grvwA, 4); + EXPECT_EQ(size.grvwB, 2); + EXPECT_EQ(size.gwvwC, 4); + EXPECT_EQ(size.gwvwD, 4); + EXPECT_EQ(size.depthU, 16); + EXPECT_EQ(size.globalSplitUPGR, 16); + EXPECT_EQ(size.globalSplitU, 1); + EXPECT_EQ(size.workGroupMapping, 5); + EXPECT_EQ(size.staggerU, 0); + EXPECT_EQ(size.magicDivAlg, 1); + EXPECT_EQ(size.tileProcessingStrategy, TileProcessingStrategy::None); + EXPECT_EQ(size.workAssignment, WorkAssignment::StaticGrid); + EXPECT_FALSE(size.sourceKernel); + EXPECT_EQ(size.globalAccumulation, 0); + EXPECT_EQ(size.workspaceSizePerElemC, 4); + EXPECT_EQ(size.workspaceSizePerElemBias, 0); + EXPECT_TRUE(size.activationFused); + EXPECT_EQ(size.workGroupMappingXCC, 4); + EXPECT_EQ(size.workGroupMappingXCCGroup, 8); + EXPECT_EQ(size.CUOccupancy, 3); + EXPECT_EQ(size.PrefetchGlobalRead, 2); + EXPECT_EQ(size.MathClocksUnrolledLoop, 11); + EXPECT_EQ(size.synchronizerSizePerWG, 32); + EXPECT_EQ(size.waveGroup, (std::array{1, 1})); + EXPECT_EQ(size.clusterDim, TensileLite::dim3(1, 1, 1)); + EXPECT_EQ(size.expertSchedulingMode, 0); + EXPECT_EQ(size.WavefrontSize, 64); + EXPECT_FALSE(size.OneLDSBuffer); + EXPECT_FALSE(size.SourceSwap); + EXPECT_EQ(size.UnrollMajorLDSA, -1); + EXPECT_EQ(size.UnrollMajorLDSB, -1); + EXPECT_FALSE(size.enableLDSTrA); + EXPECT_FALSE(size.enableLDSTrB); + EXPECT_TRUE(size.BufferLoad); + EXPECT_TRUE(size.BufferStore); + EXPECT_FALSE(type.stochasticRounding); + + auto const& support = solution->internalArgsSupport; + EXPECT_EQ(support.version, 2); + EXPECT_EQ(support.persistentLoopArgsVersion, 0); + EXPECT_TRUE(support.gsu); + EXPECT_FALSE(support.wgm); + EXPECT_TRUE(support.staggerU); + EXPECT_TRUE(support.useUniversalArgs); + EXPECT_FALSE(support.useSFC); + + auto const& tasks = conjunction(solution->taskPredicate); + ASSERT_EQ(tasks.value.size(), 2); + EXPECT_EQ(tasks.value[0]->type(), "WorkspaceCheck"); + EXPECT_EQ(tasks.value[1]->type(), "LaunchLimits"); + + auto const& hardware + = dynamic_cast const&>(*solution->hardwarePredicate); + auto const& processor + = dynamic_cast(*hardware.value); + EXPECT_EQ(processor.value, AMDGPU::Processor::gfx942); + + auto const* language + = onlyDirect(solution->problemPredicate); + ASSERT_NE(language, nullptr); + EXPECT_EQ(language->value, KernelLanguage::Assembly); + + auto const* xcc = onlyDirect( + solution->problemPredicate); + ASSERT_NE(xcc, nullptr); + EXPECT_EQ(xcc->value, (std::array{4, 8})); + // Unevaluated. The count is filled on the first check, not frozen at 0. + EXPECT_EQ(xcc->cuCount, size_t{0}); + + EXPECT_EQ(onlyDirect(solution->problemPredicate), nullptr); +} + +TEST(SolutionState, DictValuedAssertSizeEqual) +{ + using namespace Predicates::Contraction; + auto solution = build(with([](StateOptions& options) { options.dimAssertions = true; })); + + expectIndexValue(onlyDirect(solution->problemPredicate), 0, 32); + expectIndexValue(onlyDirect(solution->problemPredicate), 1, 8); + expectIndexValue(onlyDirect(solution->problemPredicate), 3, 16385); + expectIndexValue(onlyDirect(solution->problemPredicate), 0, 1); + expectIndexValue(onlyDirect(solution->problemPredicate), 1, 4); + expectIndexValue(onlyDirect(solution->problemPredicate), 2, 7); + + auto const* strides = onlyDirect(solution->problemPredicate); + ASSERT_NE(strides, nullptr); + ASSERT_EQ(strides->value.size(), 2); + auto const& stride0 = dynamic_cast(*strides->value[0]); + auto const& stride1 = dynamic_cast(*strides->value[1]); + EXPECT_EQ(stride0.index, 0); + EXPECT_EQ(stride0.value, 1); + EXPECT_EQ(stride1.index, 1); + EXPECT_EQ(stride1.value, 64); + + EXPECT_EQ(onlyDirect( + build(with([](StateOptions& options) { options.emptySizeEqual = true; })) + ->problemPredicate), + nullptr); + EXPECT_EQ(onlyDirect( + build(with([](StateOptions& options) { options.skippedSizeEqual = true; })) + ->problemPredicate), + nullptr); + + auto scalar + = derivedState(with([](StateOptions& options) { options.assertSizeAsScalar = true; })) + .unpack(); + try + { + solutionFromState(scalar.get()); + FAIL() << "a scalar AssertSizeEqual was accepted"; + } + catch(std::runtime_error const& error) + { + EXPECT_NE(std::string(error.what()).find("AssertSizeEqual must be a dict of {index: value}"), + std::string::npos); + } +} + +TEST(SolutionState, NamesAMirroredSummation) +{ + StateOptions options; + options.mirrorSummationA = true; + auto solution = build(options); + // Batch index 2 and summation 3 mirrored on A. The default fixture is the + // unmirrored Contraction_l_Ailk_Bljk_Cijk_Dijk. + EXPECT_EQ(solution->problemType.operationIdentifier, "Contraction_l_AiLk_Bljk_Cijk_Dijk"); +} + +TEST(SolutionState, NamesSummationOrder) +{ + StateOptions options; + options.swappedSummation = true; + auto solution = build(options); + EXPECT_EQ(solution->problemType.operationIdentifier, "Contraction_lm_AiLmk_BljMk_Cijk_Dijk"); +} + +TEST(SolutionState, BuildsWithoutADevice) +{ + // Constructing the record must not call hipGetDevice. On a machine with a + // GPU that call would store a non-zero count here. The test constructor + // also leaves the count at 0, but it marks the query done, so evaluation + // rejects the kernel instead of reading the device. + auto solution = build(); + auto const* xcc = onlyDirect( + solution->problemPredicate); + ASSERT_NE(xcc, nullptr); + EXPECT_EQ(xcc->value, (std::array{4, 8})); + EXPECT_EQ(xcc->cuCount, size_t{0}); + + auto problem = ContractionProblemGemm::GEMM( + false, false, 1024, 1024, 1024, 1024, 1024, 1024, 1.0, false, 1); + int deviceCount = 0; + hipError_t deviceErr = hipGetDeviceCount(&deviceCount); + if(deviceErr != hipSuccess || deviceCount == 0) + { + EXPECT_THROW((*xcc)(problem), std::runtime_error); + return; + } + EXPECT_TRUE((*xcc)(problem)); + EXPECT_NE(xcc->cuCount, size_t{0}); +} + +TEST(SolutionState, ClusterStreamKAcceptsPersistentLoopArgsVersion2) +{ + StateOptions options; + options.tileStrategy = "StreamK"; + options.workAssignment = "StaticGrid"; + options.clusterDim = {2, 1}; + options.kernArgsVersion = 3; + options.persistentLoopArgsVersion = 2; + auto solution = build(options); + EXPECT_EQ(solution->sizeMapping.tileProcessingStrategy, TileProcessingStrategy::StreamK); + EXPECT_EQ(solution->sizeMapping.workAssignment, WorkAssignment::StaticGrid); + EXPECT_EQ(solution->sizeMapping.clusterDim, TensileLite::dim3(2, 1, 1)); + EXPECT_EQ(solution->internalArgsSupport.persistentLoopArgsVersion, 2); + EXPECT_EQ(solution->internalArgsSupport.version, 3); +} + +TEST(SolutionState, MixedMxOperandsUseTheirOwnComputeInputs) +{ + StateOptions options; + options.dataType = "F8"; + options.dataTypeA = "F8"; + options.dataTypeB = "B8"; + auto solution = build(options); + EXPECT_EQ(solution->problemType.aType, rocisa::DataType::Float8); + EXPECT_EQ(solution->problemType.bType, rocisa::DataType::BFloat8); + EXPECT_EQ(solution->problemType.computeInputTypeA, rocisa::DataType::Float8); + EXPECT_EQ(solution->problemType.computeInputTypeB, rocisa::DataType::BFloat8); +} + +TEST(SolutionState, Int4ARecordsGroupScalePredicates) +{ + using namespace Predicates::Contraction; + StateOptions options; + options.dataType = "I4"; + options.dataTypeA = "I4"; + options.scaleBlockSizesA = {128, 32}; + auto solution = build(options); + auto const& root = conjunction(solution->problemPredicate); + Predicates::Or const* either = nullptr; + for(auto const& child : root.value) + if(auto cast = std::dynamic_pointer_cast>(child)) + either = cast.get(); + ASSERT_NE(either, nullptr); + ASSERT_EQ(either->value.size(), 2u); + auto first = std::dynamic_pointer_cast(either->value[0]); + auto second = std::dynamic_pointer_cast(either->value[1]); + ASSERT_NE(first, nullptr); + ASSERT_NE(second, nullptr); + EXPECT_EQ(first->value, 32); + EXPECT_EQ(second->value, 128); + auto const* zero = onlyDirect(solution->problemPredicate); + ASSERT_NE(zero, nullptr); + EXPECT_FALSE(zero->value); + auto const* encoding = onlyDirect(solution->problemPredicate); + ASSERT_NE(encoding, nullptr); + EXPECT_EQ(encoding->value, ContractionProblemGemm::Int4Encoding::Signed); +} + +TEST(SolutionState, StochasticRoundingIsRead) +{ + StateOptions options; + options.stochasticRounding = true; + EXPECT_TRUE(build(options)->problemType.stochasticRounding); +} + +TEST(SolutionState, CustomKernelSizeBoundsArePredicates) +{ + using namespace Predicates::Contraction; + StateOptions options; + options.sizeBounds = true; + auto solution = build(options); + expectIndexValue(onlyDirect(solution->problemPredicate), 0, 31); + expectIndexValue(onlyDirect(solution->problemPredicate), 0, 129); +} + +TEST(SolutionState, CustomKernelUniformSummationOrderFollowsTheState) +{ + StateOptions options; + options.customKernel = true; + + options.customUniformSummationOrder = 1; + auto declared = build(options); + EXPECT_EQ(declared->customKernel.name, "hand_written"); + EXPECT_TRUE(declared->customKernel.uniformSummationOrder); + EXPECT_FALSE(declared->customKernel.generated); + + options.customUniformSummationOrder = 0; + EXPECT_FALSE(build(options)->customKernel.uniformSummationOrder); + + // Contractions.CustomKernel.FromOriginalState defaults it to false. + options.customUniformSummationOrder = -1; + EXPECT_FALSE(build(options)->customKernel.uniformSummationOrder); +} + +TEST(SolutionState, SizeMappingOptionalsFollowTheState) +{ + StateOptions options; + options.oneLdsBuffer = true; + options.bufferLoadOff = true; + auto solution = build(options); + auto const& size = solution->sizeMapping; + EXPECT_TRUE(size.OneLDSBuffer); + EXPECT_FALSE(size.BufferLoad); + EXPECT_TRUE(size.BufferStore); + EXPECT_EQ(size.WavefrontSize, 64); +} + +TEST(SolutionState, IdealsKeyNamesTheFailure) +{ + StateOptions options; + options.badIdealKey = true; + try + { + build(options); + FAIL() << "a non-integer Ideals key was accepted"; + } + catch(std::runtime_error const& error) + { + EXPECT_NE(std::string(error.what()).find("Ideals key"), std::string::npos); + EXPECT_NE(std::string(error.what()).find("nope"), std::string::npos); + } +} + +TEST(SolutionState, Gfx950DeviceNameSelectsTheChip) +{ + StateOptions options; + options.isa = {9, 5, 0}; + options.deviceName = "Device 75a0"; + auto solution = build(options); + auto const& hardware + = dynamic_cast const&>(*solution->hardwarePredicate); + auto const* all = dynamic_cast const*>(hardware.value.get()); + ASSERT_NE(all, nullptr); + bool sawProcessor = false; + bool sawChip = false; + for(auto const& child : all->value) + { + if(auto processor = std::dynamic_pointer_cast(child)) + { + EXPECT_EQ(processor->value, AMDGPU::Processor::gfx950); + sawProcessor = true; + } + if(auto chip = std::dynamic_pointer_cast(child)) + { + EXPECT_EQ(chip->value, 0x75a0); + sawChip = true; + } + } + EXPECT_TRUE(sawProcessor); + EXPECT_TRUE(sawChip); +} + +TEST(SolutionState, RejectsAMissingKey) +{ + auto state = derivedState(with([](StateOptions& options) { options.batched = false; })).unpack(); + try + { + solutionFromState(state.get()); + FAIL() << "a state without ProblemType.Batched was accepted"; + } + catch(std::runtime_error const& error) + { + EXPECT_STREQ(error.what(), "Solution state has no ProblemType.Batched"); + } +}