diff --git a/.github/workflows/hipdnn-superbuild-ci.yml b/.github/workflows/hipdnn-superbuild-ci.yml index 80349f4bafb3..161ef5f6e320 100644 --- a/.github/workflows/hipdnn-superbuild-ci.yml +++ b/.github/workflows/hipdnn-superbuild-ci.yml @@ -120,6 +120,7 @@ jobs: -DROCM_LIBS_ENABLE_ROOT_CTEST=ON \ -DENABLE_CLANG_FORMAT=OFF \ -DHIPKERNELPROVIDER_ENABLE_ROCKE=ON \ + -DHIPDNN_ENABLE_KERNEL_INGESTOR=ON \ -DENABLE_CLANG_TIDY=OFF - name: Build @@ -280,11 +281,26 @@ jobs: -DENABLE_CLANG_FORMAT=OFF \ -DENABLE_CLANG_TIDY=ON \ -DHIPKERNELPROVIDER_ENABLE_ROCKE=ON \ + -DHIPDNN_ENABLE_KERNEL_INGESTOR=ON \ -DCMAKE_CXX_CLANG_TIDY=/usr/bin/clang-tidy-20 - name: Build run: cmake --build build + - name: Verify UMD op-schema registry is in sync + # The build regenerates op_schema_registry_generated.hpp in-tree from the + # .fbs schemas (HIPDNN_GENERATE_SDK_HEADERS defaults ON). The pre-commit + # flatc-hipdnn hook only refreshes the --cpp data-object headers, not this + # registry, so a schema change committed via pre-commit can leave the + # committed registry stale. This step is the authoritative drift guard. + run: | + if ! git diff --exit-code -- \ + projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/umd/op_schema_registry_generated.hpp; then + echo "::error::UMD op-schema registry is stale: a schema change was committed without regenerating op_schema_registry_generated.hpp." + echo "Rebuild hipDNN with -DHIPDNN_GENERATE_SDK_HEADERS=ON (the default) and commit the regenerated header." + exit 1 + fi + - name: Run tests run: ctest --test-dir build --output-on-failure diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 416c8ddceb3a..876c45249d66 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,8 +44,10 @@ exclude: | \.github/workflows/.*\.lock\.yml$| # Binary files projects/hipdnn/.*\.bin$| - # Generated hipDNN FlatBuffers headers + # Generated hipDNN FlatBuffers headers (flatc output) projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/data_objects/.*_generated\.h| + # umd_registry_gen-derived op-schema registry (keep OpSchemaRegistry.hpp formatted) + projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/umd/.*_generated\.hpp| # Reference data (auto-generated) dnn-providers/integration-tests/integration-test-bundles/.*| # Generated files (see rocrand/tools/ _generator.cpp files) @@ -93,6 +95,13 @@ repos: - repo: local hooks: + # Regenerates only the --cpp data-object headers (data_objects/*_generated.h) + # from the edited .fbs schemas. The UMD op-schema registry + # (umd/op_schema_registry_generated.hpp) is NOT refreshed here: it is derived + # from a C++ tool (umd_registry_gen) that must be compiled, which the + # pre-commit environment cannot do. That header is regenerated by the CMake + # build (HIPDNN_GENERATE_SDK_HEADERS, default ON) and enforced in CI by the + # "Verify UMD op-schema registry is in sync" step in hipdnn-superbuild-ci.yml. - id: flatc-hipdnn name: flatc (hipDNN) entry: python projects/hipdnn/scripts/run_flatc.py diff --git a/dnn-providers/hip-kernel-provider/CMakeLists.txt b/dnn-providers/hip-kernel-provider/CMakeLists.txt index 3ef9034dd82b..861d4c7a68f3 100644 --- a/dnn-providers/hip-kernel-provider/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/CMakeLists.txt @@ -50,6 +50,10 @@ option(HIPKERNELPROVIDER_ENABLE_ROCKE "Build rocKE engine and smoke tests" OFF) option(ENABLE_ASM_SDPA_ENGINE "Build the ASM SDPA engine" ON) option(ENABLE_HIP_MLOPS_ENGINE "Build the HIP kernel engine" ON) +# Gates the generic kernel ingestor (RFC 0017). The ingestor engine compiles its kernel +# through the HIP kernel engine's compiler, so it only builds alongside that engine. +option(HIPDNN_ENABLE_KERNEL_INGESTOR "Build the generic kernel ingestor (RFC 0017)" OFF) + option(BUILD_ADDRESS_SANITIZER "Build with Address Sanitizer enabled" OFF) option(ENABLE_CLANG_FORMAT "Enable Clang-Format checks" ON) if(WIN32) diff --git a/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/JsonDataSource.hpp b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/JsonDataSource.hpp new file mode 100644 index 000000000000..7faab6798bab --- /dev/null +++ b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/JsonDataSource.hpp @@ -0,0 +1,310 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +// JsonDataSource.hpp - a sample JsonLogic data source backed by nlohmann::json. +// +// All names below live in namespace hip_kernel_provider_common::jsonlogic; +// these examples assume `namespace jlogic = hip_kernel_provider_common::jsonlogic;`. +// +// JsonDataSource wraps an nlohmann::json document and satisfies the JsonLogic +// data-source contract (Value getData(const std::string&) const), so a compiled +// jlogic::Expression can evaluate directly against a JSON document: +// +// jlogic::JsonDataSource src(nlohmann::json{{"q", {{"dims", {8, 16}}}}}); +// auto expr = jlogic::compile(rule); +// jlogic::Value r = expr(src); +// +// It also offers the inverse, setData, which writes a jlogic::Value back into +// the document at a path, creating intermediate objects and arrays as needed: +// +// src.setData("$q.dims[0]", 2); // -> {"q":{"dims":[2, 16]}} +// +// Path syntax (shared by getData and setData): +// - dotted keys: a.b.c +// - [N] array subscripts: arr[0], rows[2].name, grid[0][1] +// - dot-form array indices: arr.1 (resolves as an index only against an +// existing array; use arr[1] to force array creation) +// - an optional leading variable sigil ('$' by default) is stripped, so both +// "q.dims[0]" and "$q.dims[0]" address the same location +// - the empty path (or a bare sigil) addresses the whole document +// +// getData follows the JsonLogic convention: an unresolved path reads as null +// (Value()). setData is a mutation and reports a malformed path or an +// incompatible index by throwing std::invalid_argument. +// +// This is a *sample* accessor: objects and null in the document convert to +// jlogic::Value null (Value has no object alternative), matching Value's +// scalar/array-only model. + +#include "hip_kernel_provider_common/JsonLogic.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace hip_kernel_provider_common::jsonlogic +{ +class JsonDataSource +{ +public: + JsonDataSource() = default; + explicit JsonDataSource(nlohmann::json doc, char sigil = '$') + : _doc(std::move(doc)) + , _sigil(sigil) + { + } + + /// The backing document, for inspection or bulk replacement. + const nlohmann::json& document() const + { + return _doc; + } + nlohmann::json& document() + { + return _doc; + } + + /// JsonLogic data-source contract: resolve a variable path to a Value. + /// An empty path returns the whole document; an unresolved path (missing + /// key, out-of-range or non-numeric index, subscript on a non-array) + /// returns null so a `var` default, if any, takes over. + Value getData(const std::string& path) const + { + std::vector segs; + if(!tokenize(path, _sigil, segs)) + { + return {}; // malformed subscript -> not found + } + const nlohmann::json* cur = &_doc; + for(const auto& seg : segs) + { + if(seg.subscript || cur->is_array()) + { + if(!cur->is_array()) + { + return {}; // subscript on a non-array + } + std::size_t idx = 0; + if(!parseIndex(seg.text, idx) || idx >= cur->size()) + { + return {}; + } + cur = &(*cur)[idx]; + } + else if(cur->is_object()) + { + const auto it = cur->find(seg.text); + if(it == cur->end()) + { + return {}; + } + cur = &*it; + } + else + { + return {}; + } + } + return toValue(*cur); + } + + /// Write a Value into the document at a path, creating intermediate + /// containers as needed: a `[N]` subscript (or a dot-form index against an + /// existing array) grows an array, filling gaps with null; any other key + /// creates or descends into an object. An empty path replaces the whole + /// document. Throws std::invalid_argument on a malformed path or a + /// non-numeric index applied to an array. + void setData(const std::string& path, const Value& value) + { + std::vector segs; + if(!tokenize(path, _sigil, segs)) + { + throw std::invalid_argument("JsonDataSource::setData: malformed path '" + path + "'"); + } + if(segs.empty()) + { + _doc = toJson(value); // whole-document assignment + return; + } + nlohmann::json* cur = &_doc; + for(std::size_t i = 0; i < segs.size(); ++i) + { + const Segment& seg = segs[i]; + nlohmann::json* target = nullptr; + if(seg.subscript) + { + std::size_t idx = 0; + if(!parseIndex(seg.text, idx)) + { + throw std::invalid_argument("JsonDataSource::setData: bad array index '" + + seg.text + "' in path '" + path + "'"); + } + if(!cur->is_array()) + { + *cur = nlohmann::json::array(); + } + target = &(*cur)[idx]; // grows the array, filling gaps with null + } + else if(cur->is_array()) + { + std::size_t idx = 0; + if(!parseIndex(seg.text, idx)) + { + throw std::invalid_argument("JsonDataSource::setData: non-numeric key '" + + seg.text + "' on array in path '" + path + "'"); + } + target = &(*cur)[idx]; + } + else + { + if(!cur->is_object()) + { + *cur = nlohmann::json::object(); + } + target = &(*cur)[seg.text]; + } + if(i + 1 == segs.size()) + { + *target = toJson(value); + } + else + { + cur = target; + } + } + } + +private: + struct Segment + { + bool subscript; // true for [N] forms, false for dotted/leading keys + std::string text; + }; + + /// Split a path into segments. Strips one optional leading sigil. Returns + /// false on a malformed subscript (missing ']'). + static bool tokenize(const std::string& raw, char sigil, std::vector& out) + { + std::size_t pos = 0; + if(!raw.empty() && raw[0] == sigil) + { + ++pos; // strip the variable sigil + } + while(pos < raw.size()) + { + if(raw[pos] == '[') + { + const std::size_t close = raw.find(']', pos + 1); + if(close == std::string::npos) + { + return false; + } + out.push_back({true, raw.substr(pos + 1, close - pos - 1)}); + pos = close + 1; + } + else + { + if(raw[pos] == '.') + { + ++pos; // skip the key separator + } + const std::size_t next = raw.find_first_of(".[", pos); + const std::size_t end = (next == std::string::npos) ? raw.size() : next; + out.push_back({false, raw.substr(pos, end - pos)}); + pos = end; + } + } + return true; + } + + /// Parse a non-negative decimal index. Rejects empty, negative, and + /// non-numeric (with trailing garbage) text. + static bool parseIndex(const std::string& s, std::size_t& idx) + { + if(s.empty()) + { + return false; + } + char* endp = nullptr; + const long val = std::strtol(s.c_str(), &endp, 10); + if(*endp != '\0' || val < 0) + { + return false; + } + idx = static_cast(val); + return true; + } + + static Value toValue(const nlohmann::json& j) + { + if(j.is_boolean()) + { + return {j.get()}; + } + if(j.is_number_integer() || j.is_number_unsigned()) + { + return {j.get()}; + } + if(j.is_number_float()) + { + return {j.get()}; + } + if(j.is_string()) + { + return {j.get()}; + } + if(j.is_array()) + { + Value::Array a; + a.reserve(j.size()); + for(const auto& e : j) + { + a.push_back(toValue(e)); + } + return {std::move(a)}; + } + return {}; // null or object -> not representable, treated as null + } + + static nlohmann::json toJson(const Value& v) + { + if(v.isBool()) + { + return v.asBool(); + } + if(v.isInt()) + { + return v.asInt(); + } + if(v.isDouble()) + { + return v.asDouble(); + } + if(v.isString()) + { + return v.asString(); + } + if(v.isArray()) + { + nlohmann::json a = nlohmann::json::array(); + for(const auto& e : v.asArray()) + { + a.push_back(toJson(e)); + } + return a; + } + return nullptr; // null + } + + nlohmann::json _doc; + char _sigil = '$'; +}; + +} // namespace hip_kernel_provider_common::jsonlogic diff --git a/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/JsonLogic.hpp b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/JsonLogic.hpp new file mode 100644 index 000000000000..2fc572774582 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/JsonLogic.hpp @@ -0,0 +1,1288 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +// JsonLogic.hpp - single-file JsonLogic expression compiler. +// +// All names below live in namespace hip_kernel_provider_common::jsonlogic; +// these examples assume `namespace jlogic = hip_kernel_provider_common::jsonlogic;`. +// +// A JsonLogic expression (an nlohmann::json value) is *compiled once* into a +// reusable jlogic::Expression, then evaluated many times against different +// data sources: +// +// struct MyData { // your data source +// jlogic::Value getData(const std::string& path) const; +// }; +// +// auto expr = jlogic::compile(rule); // parse + build tree once +// jlogic::Value r1 = expr(dataA); // evaluate, no re-parse +// jlogic::Value r2 = expr(dataB); // reuse for other data +// +// The runtime value type (jlogic::Value) is a small standalone variant that does +// NOT depend on nlohmann/json; nlohmann is used only to express the rule being +// compiled. The data source is any C++ type exposing +// Value getData(const std::string&) +// which resolves a variable path to a Value; a null return means "not found", +// triggering a var default if one is given. Variable paths are always +// non-empty (whole-document references are not supported). +// +// Inline variables (extension over stock JsonLogic) +// ------------------------------------------------- +// A string prefixed with a sigil (default '$') is a variable reference, so a +// variable can appear anywhere a literal can: +// {"+": ["$x", "$y"]} == {"+": [{"var": "x"}, {"var": "y"}]} +// "$a.b" nested path "$$x" literal "$x" +// Strings without the sigil remain literals, so stock rules keep working. +// +// Scope: core operator set (data access, logic, comparison, arithmetic), the +// membership operator `in`, the `value_or_default` fallback, the `present` / +// `not_present` resolution predicates, and the value-core math extensions the +// hipDNN UMD needs (ceil_div, abs, pow, log2, rsqrt). +// Collection/string operators (map/reduce/filter/cat/substr/...) are not +// included. +// +// Null is "unresolved", not a value (extension over stock JsonLogic) +// ----------------------------------------------------------------- +// Every operator except `present`, `not_present`, and `value_or_default` +// propagates a null argument instead of coercing it, and `and`/`or` are +// three-valued: a definite false still decides an `and`, a definite true a +// `or`. The consumer is the hipDNN UMD matcher, where an unresolved path is an +// absent optional operand; a coerced null would read as false/0/not-equal and +// make a narrowing criterion silently PASS on data the pack never saw. A null +// root is falsy, so an undecided expression declines the match. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hip_kernel_provider_common::jsonlogic +{ +/// Thrown when a rule cannot be compiled (unknown operator, bad arity, ...). +class JsonLogicCompileError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; + +// =========================================================================== +// Value - standalone runtime value (json-like, no nlohmann dependency) +// =========================================================================== +class Value +{ +public: + using Array = std::vector; + + Value() + : _v(nullptr) + { + } + Value(std::nullptr_t) + : _v(nullptr) + { + } + Value(bool b) + : _v(b) + { + } + Value(int i) + : _v(static_cast(i)) + { + } + Value(std::int64_t i) + : _v(i) + { + } + Value(double d) + : _v(d) + { + } + Value(const char* s) + : _v(std::string(s)) + { + } + Value(std::string s) + : _v(std::move(s)) + { + } + Value(Array a) + : _v(std::move(a)) + { + } + + /// Build a numeric value, storing an integer when the double is exactly + /// integral and representable, so integer inputs yield integer output. + static Value number(double d) + { + if(std::isfinite(d)) + { + const double t = std::trunc(d); + if(t == d && d >= -9.007199254740992e15 && d <= 9.007199254740992e15) + { + return {static_cast(t)}; + } + } + return {d}; + } + + bool isNull() const + { + return std::holds_alternative(_v); + } + bool isBool() const + { + return std::holds_alternative(_v); + } + bool isInt() const + { + return std::holds_alternative(_v); + } + bool isDouble() const + { + return std::holds_alternative(_v); + } + bool isNumber() const + { + return isInt() || isDouble(); + } + bool isString() const + { + return std::holds_alternative(_v); + } + bool isArray() const + { + return std::holds_alternative(_v); + } + + bool asBool() const + { + return std::get(_v); + } + std::int64_t asInt() const + { + return std::get(_v); + } + double asDouble() const + { + return std::get(_v); + } + const std::string& asString() const + { + return std::get(_v); + } + const Array& asArray() const + { + return std::get(_v); + } + + /// JsonLogic truthiness: false, 0, "", null and the empty array are falsy. + bool truthy() const + { + switch(_v.index()) + { + case 1: + return std::get(_v); + case 2: + return std::get(_v) != 0; + case 3: + return std::get(_v) != 0.0; + case 4: + return !std::get(_v).empty(); + case 5: + return !std::get(_v).empty(); + default: + return false; // null + } + } + + /// JS Number() coercion. Non-numeric strings and multi-element arrays yield + /// NaN; empty string / empty array / null yield 0. + double toNumber() const + { + switch(_v.index()) + { + case 1: + return std::get(_v) ? 1.0 : 0.0; + case 2: + return static_cast(std::get(_v)); + case 3: + return std::get(_v); + case 4: + return stringToNumber(std::get(_v)); + case 5: + { + const auto& a = std::get(_v); + if(a.empty()) + { + return 0.0; + } + if(a.size() == 1) + { + return a.front().toNumber(); + } + return std::nan(""); + } + default: + return 0.0; // null + } + } + + /// Structural equality (== / !=). Integers and doubles of equal value + /// compare equal; differing kinds do not. + bool operator==(const Value& o) const + { + if(isNumber() && o.isNumber()) + { + return toNumber() == o.toNumber(); + } + if(_v.index() != o._v.index()) + { + return false; + } + switch(_v.index()) + { + case 0: + return true; // null == null + case 1: + return std::get(_v) == std::get(o._v); + case 4: + return std::get(_v) == std::get(o._v); + case 5: + return std::get(_v) == std::get(o._v); + default: + return false; + } + } + bool operator!=(const Value& o) const + { + return !(*this == o); + } + + /// Three-way compare for ordering. Returns -1/0/1, or 2 (incomparable, NaN) + /// which makes every ordering test false. Two strings compare lexically. + static int compare(const Value& a, const Value& b) + { + if(a.isString() && b.isString()) + { + const auto& x = a.asString(); + const auto& y = b.asString(); + if(x < y) + { + return -1; + } + return x > y ? 1 : 0; + } + const double x = a.toNumber(); + const double y = b.toNumber(); + if(std::isnan(x) || std::isnan(y)) + { + return 2; + } + if(x < y) + { + return -1; + } + return x > y ? 1 : 0; + } + + /// Human-readable rendering, mainly for diagnostics and tests. + std::string dump() const + { + switch(_v.index()) + { + case 1: + return std::get(_v) ? "true" : "false"; + case 2: + return std::to_string(std::get(_v)); + case 3: + { + std::string s = std::to_string(std::get(_v)); + return s; + } + case 4: + return "\"" + std::get(_v) + "\""; + case 5: + { + std::string s = "["; + const auto& a = std::get(_v); + for(std::size_t i = 0; i < a.size(); ++i) + { + s += ((i != 0u) ? "," : "") + a[i].dump(); + } + return s + "]"; + } + default: + return "null"; + } + } + + /// Stream rendering (used by GoogleTest value printing and diagnostics). + friend std::ostream& operator<<(std::ostream& os, const Value& v) + { + return os << v.dump(); + } + +private: + static double stringToNumber(const std::string& s) + { + std::size_t b = 0; + std::size_t e = s.size(); + while(b < e && (std::isspace(static_cast(s[b])) != 0)) + { + ++b; + } + while(e > b && (std::isspace(static_cast(s[e - 1])) != 0)) + { + --e; + } + if(b == e) + { + return 0.0; // JS Number("") == 0 + } + const std::string t = s.substr(b, e - b); + const char* first = t.c_str(); + char* last = nullptr; + const double d = std::strtod(first, &last); + if(last != first + t.size()) + { + return std::nan(""); // trailing garbage -> NaN + } + return d; + } + + // null, bool, int64, double, string, array - indices used by the switches. + std::variant _v; +}; + +namespace detail +{ +// ---- data-source capability detection ------------------------------------ +template +struct HasGetData : std::false_type +{ +}; +template +struct HasGetData< + T, + std::void_t().getData(std::declval()))>> + : std::true_type +{ +}; + +// ---- type-erased data source --------------------------------------------- +// The compiled node tree evaluates against this abstract source rather than a +// concrete DataT, so the tree itself carries no template parameter (and thus +// no per-DataT virtual member instantiation). Expression wraps the +// caller's data object in a DataSourceAdapter at evaluation time. +struct IDataSource +{ + virtual ~IDataSource() = default; + virtual Value getData(const std::string& path) const = 0; +}; + +template +struct DataSourceAdapter final : IDataSource +{ + static_assert(HasGetData::value, "Data source must provide Value getData(std::string)."); + const DataT& data; + explicit DataSourceAdapter(const DataT& d) + : data(d) + { + } + Value getData(const std::string& path) const override + { + return data.getData(path); + } +}; + +// ---- compiled node tree --------------------------------------------------- +enum class Op +{ + ADD, + SUB, + MUL, + DIV, + MOD, + MIN, + MAX, + LT, + LE, + GT, + GE, + EQ, + NEQ, + NOT, + NOT_NOT, + IF, + AND, + OR, + // Spelled MEMBERSHIP, not IN: defines IN as an empty + // annotation macro, so an enumerator named IN fails to compile on Windows + // in any translation unit that also pulls in windows.h. The JsonLogic + // operator key stays "in". + MEMBERSHIP, + CEIL_DIV, + ABS, + POW, + LOG2, + RSQRT, + VALUE_OR_DEFAULT, + PRESENT, + NOT_PRESENT +}; + +struct Node +{ + virtual ~Node() = default; + virtual Value eval(const IDataSource& data) const = 0; + virtual const std::string* variable() const + { + return nullptr; + } + virtual void pushChildren(std::vector& /*unused*/) const {} +}; + +using NodePtr = std::unique_ptr; + +struct LiteralNode final : Node +{ + Value value; + explicit LiteralNode(Value v) + : value(std::move(v)) + { + } + Value eval(const IDataSource& /*unused*/) const override + { + return value; + } +}; + +struct ArrayNode final : Node +{ + std::vector items; + Value eval(const IDataSource& data) const override + { + Value::Array a; + a.reserve(items.size()); + for(const auto& it : items) + { + a.push_back(it->eval(data)); + } + return {std::move(a)}; + } + void pushChildren(std::vector& stack) const override + { + for(auto it = items.rbegin(); it != items.rend(); ++it) + { + stack.push_back(it->get()); + } + } +}; + +struct VarNode final : Node +{ + std::string path; + NodePtr defaultExpr; // optional fallback when the lookup returns null + + Value eval(const IDataSource& data) const override + { + Value r = data.getData(path); + if(r.isNull() && defaultExpr) + { + return defaultExpr->eval(data); + } + return r; + } + + const std::string* variable() const override + { + return &path; + } + void pushChildren(std::vector& stack) const override + { + if(defaultExpr) + { + stack.push_back(defaultExpr.get()); + } + } +}; + +struct OpNode final : Node +{ + Op op; + std::vector args; + + // Null is "unresolved", not a value: an absent optional operand's field + // read must neither pass nor fail a criterion (RFC 0017 §5, RFC 0018 A.4). + // Every operator below therefore propagates a null operand instead of + // coercing it (null would otherwise read as 0 / false / not-equal, so a + // narrowing check on an absent operand would silently PASS). `present`, + // `not_present`, `value_or_default`, and the short-circuit halves of + // `and`/`or` are the deliberate exceptions: they answer "did this + // resolve?", so they always yield a real value. The root null declines the + // match, because Value::truthy() reads null as false. + static bool anyNull(const std::vector& v) + { + for(const Value& x : v) + { + if(x.isNull()) + { + return true; + } + } + return false; + } + + // Evaluate every argument once, so a null is detected before the operator + // coerces it and so each argument is evaluated exactly once. + std::vector evalArgs(const IDataSource& d) const + { + std::vector out; + out.reserve(args.size()); + for(const auto& c : args) + { + out.push_back(c->eval(d)); + } + return out; + } + + Value eval(const IDataSource& d) const override + { + switch(op) + { + case Op::ADD: + { + const std::vector v = evalArgs(d); + if(anyNull(v)) + { + return {}; + } + double a = 0.0; + for(const Value& x : v) + { + a += x.toNumber(); + } + return Value::number(a); + } + case Op::MUL: + { + const std::vector v = evalArgs(d); + if(anyNull(v)) + { + return {}; + } + double a = 1.0; + for(const Value& x : v) + { + a *= x.toNumber(); + } + return Value::number(a); + } + case Op::SUB: + { + const std::vector v = evalArgs(d); + if(anyNull(v)) + { + return {}; + } + if(v.size() == 1) + { + return Value::number(-v[0].toNumber()); + } + return Value::number(v[0].toNumber() - v[1].toNumber()); + } + case Op::DIV: + case Op::MOD: + case Op::CEIL_DIV: + { + const std::vector v = evalArgs(d); + if(anyNull(v)) + { + return {}; + } + const double num = v[0].toNumber(); + const double den = v[1].toNumber(); + if(den == 0.0) + { + return {}; // zero divisor declines rather than yielding inf/NaN + } + if(op == Op::DIV) + { + return Value::number(num / den); + } + if(op == Op::MOD) + { + return Value::number(std::fmod(num, den)); + } + return Value::number(std::ceil(num / den)); + } + case Op::MIN: + case Op::MAX: + { + const std::vector v = evalArgs(d); + if(anyNull(v)) + { + return {}; + } + double best = std::nan(""); + for(const Value& x : v) + { + const double n = x.toNumber(); + if(std::isnan(best) || (op == Op::MIN ? n < best : n > best)) + { + best = n; + } + } + return Value::number(best); + } + case Op::LT: + case Op::LE: + case Op::GT: + case Op::GE: + { + const std::vector v = evalArgs(d); + if(anyNull(v)) + { + return {}; + } + const auto ordered = [this](const Value& x, const Value& y) { + const int c = Value::compare(x, y); + switch(op) + { + case Op::LT: + return c == -1; + case Op::LE: + return c == -1 || c == 0; + case Op::GT: + return c == 1; + default: + return c == 1 || c == 0; + } + }; + // The 3-arg form is the between-chain: a < b < c. + if(v.size() >= 3) + { + return {ordered(v[0], v[1]) && ordered(v[1], v[2])}; + } + return {ordered(v[0], v[1])}; + } + case Op::EQ: + case Op::NEQ: + { + const std::vector v = evalArgs(d); + // Two unresolved references are not "equal"; the question is + // unanswerable, so it declines instead of accepting. + if(anyNull(v)) + { + return {}; + } + return {op == Op::EQ ? v[0] == v[1] : v[0] != v[1]}; + } + case Op::NOT: + case Op::NOT_NOT: + { + const Value a = args[0]->eval(d); + if(a.isNull()) + { + return {}; // negating an unresolved reference stays unresolved + } + return {op == Op::NOT ? !a.truthy() : a.truthy()}; + } + case Op::IF: + { + std::size_t i = 0; + for(; i + 1 < args.size(); i += 2) + { + const Value cond = args[i]->eval(d); + if(cond.isNull()) + { + return {}; // an unresolved condition picks no branch + } + if(cond.truthy()) + { + return args[i + 1]->eval(d); + } + } + return i < args.size() ? args[i]->eval(d) : Value(); + } + case Op::AND: + { + // Kleene `and`: a definite false short-circuits even when another + // argument is unresolved, so `and`-ing an inapplicable check beside + // a failing one still declines. Otherwise a null makes the whole + // conjunction unresolved. + Value cur(true); + bool sawNull = false; + for(const auto& c : args) + { + cur = c->eval(d); + if(cur.isNull()) + { + sawNull = true; + continue; + } + if(!cur.truthy()) + { + return cur; // definite false + } + } + return sawNull ? Value() : cur; + } + case Op::OR: + { + // Kleene `or`: a definite true short-circuits past an unresolved + // argument, which is what lets + // `{"or": [{"not_present": ["$bias"]}, {"==": ["$bias.dtype", ...]}]}` + // accept a bias-free graph even though the second arm cannot run. + Value cur; + bool sawNull = false; + for(const auto& c : args) + { + cur = c->eval(d); + if(cur.isNull()) + { + sawNull = true; + continue; + } + if(cur.truthy()) + { + return cur; // definite true + } + } + return sawNull ? Value() : cur; + } + case Op::MEMBERSHIP: + { + const std::vector v = evalArgs(d); + if(anyNull(v)) + { + return {}; + } + const Value& needle = v[0]; + const Value& hay = v[1]; + if(hay.isArray()) + { + for(const auto& e : hay.asArray()) + { + if(e == needle) + { + return {true}; + } + } + return {false}; + } + if(hay.isString()) + { + const std::string n = needle.isString() ? needle.asString() : needle.dump(); + return {hay.asString().find(n) != std::string::npos}; + } + return {false}; + } + case Op::ABS: + case Op::LOG2: + case Op::RSQRT: + { + const Value a = args[0]->eval(d); + if(a.isNull()) + { + return {}; + } + const double n = a.toNumber(); + if(op == Op::ABS) + { + return Value::number(std::fabs(n)); + } + if(n <= 0.0) + { + return {}; // log2/rsqrt decline on a non-positive argument + } + return Value::number(op == Op::LOG2 ? std::log2(n) : 1.0 / std::sqrt(n)); + } + case Op::POW: + { + const std::vector v = evalArgs(d); + if(anyNull(v)) + { + return {}; + } + return Value::number(std::pow(v[0].toNumber(), v[1].toNumber())); + } + case Op::VALUE_OR_DEFAULT: + { + // First arg is a variable reference; a null result means the path + // did not resolve in the data source, so fall back to the default. + Value const v = args[0]->eval(d); + return v.isNull() ? args[1]->eval(d) : v; + } + case Op::PRESENT: + case Op::NOT_PRESENT: + { + // Presence keys on *existence*, the same mechanism as + // VALUE_OR_DEFAULT above: an unresolved path reads null. Unlike + // every other operator these do not propagate that null -- asking + // "was this supplied?" always yields a real boolean. Both fold with + // `and` over their arguments, so one call decides a whole list. + const bool wantNull = (op == Op::NOT_PRESENT); + for(const auto& c : args) + { + if(c->eval(d).isNull() != wantNull) + { + return {false}; + } + } + return {true}; + } + default: + break; + } + return {}; + } + + void pushChildren(std::vector& stack) const override + { + for(auto it = args.rbegin(); it != args.rend(); ++it) + { + stack.push_back(it->get()); + } + } +}; + +inline const Op* lookupOp(const std::string& key) +{ + static const std::array, 28> s_table + = {{{"+", Op::ADD}, + {"-", Op::SUB}, + {"*", Op::MUL}, + {"/", Op::DIV}, + {"%", Op::MOD}, + {"min", Op::MIN}, + {"max", Op::MAX}, + {"<", Op::LT}, + {"<=", Op::LE}, + {">", Op::GT}, + {">=", Op::GE}, + {"==", Op::EQ}, + {"!=", Op::NEQ}, + {"!", Op::NOT}, + {"!!", Op::NOT_NOT}, + {"if", Op::IF}, + {"?:", Op::IF}, + {"and", Op::AND}, + {"or", Op::OR}, + {"in", Op::MEMBERSHIP}, + {"ceil_div", Op::CEIL_DIV}, + {"abs", Op::ABS}, + {"pow", Op::POW}, + {"log2", Op::LOG2}, + {"rsqrt", Op::RSQRT}, + {"value_or_default", Op::VALUE_OR_DEFAULT}, + {"present", Op::PRESENT}, + {"not_present", Op::NOT_PRESENT}}}; + for(const auto& e : s_table) + { + if(key == e.first) + { + return &e.second; + } + } + return nullptr; +} + +inline void checkArity(Op op, std::size_t n, const std::string& key) +{ + const auto require = [&](bool ok) { + if(!ok) + { + throw JsonLogicCompileError("operator '" + key + + "' got wrong argument count: " + std::to_string(n)); + } + }; + switch(op) + { + case Op::ADD: + case Op::MUL: + break; // any arity + case Op::SUB: + require(n == 1 || n == 2); + break; + case Op::DIV: + case Op::MOD: + case Op::EQ: + case Op::NEQ: + case Op::GT: + case Op::GE: + require(n == 2); + break; + case Op::LT: + case Op::LE: + require(n == 2 || n == 3); + break; + case Op::MIN: + case Op::MAX: + case Op::AND: + case Op::OR: + require(n >= 1); + break; + case Op::NOT: + case Op::NOT_NOT: + require(n == 1); + break; + case Op::IF: + require(n >= 2); + break; + case Op::MEMBERSHIP: + case Op::POW: + case Op::CEIL_DIV: + case Op::VALUE_OR_DEFAULT: + require(n == 2); + break; + case Op::PRESENT: + case Op::NOT_PRESENT: + require(n >= 1); + break; + case Op::ABS: + case Op::LOG2: + case Op::RSQRT: + require(n == 1); + break; + default: + break; + } +} + +inline Value jsonScalarToValue(const nlohmann::json& j) +{ + if(j.is_boolean()) + { + return {j.get()}; + } + if(j.is_number_integer() || j.is_number_unsigned()) + { + return {j.get()}; + } + if(j.is_number_float()) + { + return {j.get()}; + } + return {}; // null +} + +inline NodePtr compileNode(const nlohmann::json& j, char sigil); + +inline NodePtr compileVar(const nlohmann::json& val, char sigil) +{ + auto n = std::make_unique(); + const auto setStaticPath = [&](const nlohmann::json& p) { + if(p.is_string()) + { + std::string s = p.get(); + if(s.empty()) + { + throw JsonLogicCompileError("whole-document variable reference is not supported"); + } + n->path = std::move(s); + } + else if(p.is_number()) + { + n->path = std::to_string(p.get()); + } + else if(p.is_null()) + { + throw JsonLogicCompileError("whole-document variable reference is not supported"); + } + else + { + throw JsonLogicCompileError("computed variable keys are not supported"); + } + }; + if(val.is_array()) + { + if(val.empty()) + { + throw JsonLogicCompileError("whole-document variable reference is not supported"); + } + setStaticPath(val.at(0)); + if(val.size() > 1) + { + n->defaultExpr = compileNode(val.at(1), sigil); + } + } + else + { + setStaticPath(val); + } + return n; +} + +inline NodePtr compileObject(const nlohmann::json& j, char sigil) +{ + if(j.size() != 1) + { + throw JsonLogicCompileError("expression object must have exactly one operator key"); + } + const auto it = j.begin(); + const std::string& key = it.key(); + const nlohmann::json& val = it.value(); + if(key == "var") + { + return compileVar(val, sigil); + } + + const Op* op = lookupOp(key); + if(op == nullptr) + { + throw JsonLogicCompileError("unrecognized operation: " + key); + } + + auto node = std::make_unique(); + node->op = *op; + if(val.is_array()) + { + node->args.reserve(val.size()); + for(const auto& e : val) + { + node->args.push_back(compileNode(e, sigil)); + } + } + else + { + node->args.push_back(compileNode(val, sigil)); + } + checkArity(*op, node->args.size(), key); + return node; +} + +inline NodePtr compileNode(const nlohmann::json& j, char sigil) +{ + if(j.is_object()) + { + return compileObject(j, sigil); + } + if(j.is_array()) + { + auto n = std::make_unique(); + n->items.reserve(j.size()); + for(const auto& e : j) + { + n->items.push_back(compileNode(e, sigil)); + } + return n; + } + if(j.is_string()) + { + const auto& s = j.get_ref(); + if(s.empty() || s[0] != sigil) + { + return std::make_unique(Value(s)); + } + if(s.size() >= 2 && s[1] == sigil) + { + return std::make_unique(Value(s.substr(1))); // "$$x" -> "$x" + } + if(s.size() == 1) + { + throw JsonLogicCompileError("whole-document variable reference is not supported"); + } + auto n = std::make_unique(); + n->path = s.substr(1); + return n; + } + return std::make_unique(jsonScalarToValue(j)); +} + +// ---- variable iteration --------------------------------------------------- +// Lazily yields, in pre-order, a reference to every variable path referenced +// by a compiled node tree. References point into the live VarNode::path, so no +// strings are copied; duplicates are yielded as they occur (build a std::set +// from the range if you need the unique, sorted set). +class VarIterator +{ +public: + using value_type = std::string; + using reference = const std::string&; + using pointer = const std::string*; + using difference_type = std::ptrdiff_t; + using iterator_category = std::input_iterator_tag; + + VarIterator() = default; // end + explicit VarIterator(const Node* root) + { + if(root != nullptr) + { + _stack.push_back(root); + } + advance(); + } + + reference operator*() const + { + return *_cur; + } + pointer operator->() const + { + return _cur; + } + + VarIterator& operator++() + { + advance(); + return *this; + } + VarIterator operator++(int) + { + VarIterator tmp = *this; + advance(); + return tmp; + } + + bool operator==(const VarIterator& o) const + { + return _cur == nullptr && o._cur == nullptr; + } + bool operator!=(const VarIterator& o) const + { + return !(*this == o); + } + +private: + void advance() + { + while(!_stack.empty()) + { + const Node* n = _stack.back(); + _stack.pop_back(); + n->pushChildren(_stack); + if(const std::string* p = n->variable()) + { + _cur = p; + return; + } + } + _cur = nullptr; + } + + std::vector _stack; + const std::string* _cur = nullptr; +}; + +class VarRange +{ +public: + explicit VarRange(const Node* root) + : _begin(root) + { + } + const VarIterator& begin() const + { + return _begin; + } + const VarIterator& end() const + { + return _end; + } + +private: + VarIterator _begin; + VarIterator _end; +}; + +} // namespace detail + +// =========================================================================== +// Expression - a compiled, reusable JsonLogic expression +// =========================================================================== +template +class Expression +{ +public: + Expression() = default; + explicit Expression(detail::NodePtr root) + : _root(std::move(root)) + { + } + + /// Evaluate against a data source. Cheap: walks the pre-compiled tree. + Value operator()(const DataT& data) const + { + if(!_root) + { + return {}; + } + const detail::DataSourceAdapter source(data); + return _root->eval(source); + } + Value evaluate(const DataT& data) const + { + return (*this)(data); + } + + explicit operator bool() const + { + return static_cast(_root); + } + + /// A lazy, pre-order range over every variable path referenced in the + /// expression. References point into the live tree, so the range must not + /// outlive this Expression. Duplicates are yielded as they occur; + /// construct a std::set from the range for the unique, sorted set. + detail::VarRange variables() const + { + return detail::VarRange(_root.get()); + } + +private: + detail::NodePtr _root; +}; + +/// Compile a JsonLogic rule into a reusable Expression bound to data source +/// type DataT. Throws JsonLogicCompileError on malformed rules. +template +Expression compile(const nlohmann::json& rule, char varSigil = '$') +{ + return Expression(detail::compileNode(rule, varSigil)); +} + +/// Convenience one-shot: compile and evaluate in a single call. Prefer +/// compile() + reuse when evaluating the same rule repeatedly. +template +Value evaluate(const nlohmann::json& rule, const DataT& data, char varSigil = '$') +{ + return compile(rule, varSigil)(data); +} + +/// True when any variable referenced by `expr` has `root` as its first path +/// token (the segment before the first '.'/'[' separator). The paths yielded by +/// Expression::variables() are already sigil-stripped, so `root` is given +/// without the sigil (e.g. "kernel"). Short-circuits on the first match. +template +bool referencesVariableRoot(const Expression& expr, std::string_view root) +{ + for(const std::string& path : expr.variables()) + { + const std::size_t end = path.find_first_of(".["); + const std::string_view first(path.data(), end == std::string::npos ? path.size() : end); + if(first == root) + { + return true; + } + } + return false; +} + +} // namespace hip_kernel_provider_common::jsonlogic diff --git a/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/JsonLogic.md b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/JsonLogic.md new file mode 100644 index 000000000000..3584a2e9038c --- /dev/null +++ b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/JsonLogic.md @@ -0,0 +1,180 @@ +# JsonLogic + +Single-header [JsonLogic](https://jsonlogic.com/) **expression compiler** +(`JsonLogic.hpp`). A rule written as an `nlohmann::json` value is compiled once +into a reusable `Expression`, then evaluated many times against different +data sources. The runtime value type (`Value`) is a small standalone variant +that does **not** depend on nlohmann/json — nlohmann is used only to express the +rule being compiled. + +All names live in namespace `hip_kernel_provider_common::jsonlogic`; the examples +below assume `namespace jlogic = hip_kernel_provider_common::jsonlogic;`. + +```cpp +#include "hip_kernel_provider_common/JsonLogic.hpp" + +namespace jlogic = hip_kernel_provider_common::jsonlogic; + +struct MyData { // your data source + jlogic::Value getData(const std::string& path) const; +}; + +nlohmann::json rule = {{"+", {"$x", "$y"}}}; +auto expr = jlogic::compile(rule); // parse + build tree once +jlogic::Value a = expr(dataA); // evaluate - no re-parse +jlogic::Value b = expr(dataB); // reuse for other data +``` + +## Data source + +The evaluator is templated on your data type, which must expose: + +```cpp +jlogic::Value getData(const std::string& path) const; +``` + +The compiled expression passes the variable path (e.g. `"a.b.c"`) straight to +this accessor; your type owns path resolution. Conventions: + +- `""` is the whole-document request. +- returning `Value()` (null) means "not found" and triggers a `var` default if + the rule supplies one. + +### Sample: `JsonDataSource` + +`JsonDataSource.hpp` provides a ready-made data source backed by an +`nlohmann::json` document, so you can evaluate rules against JSON without writing +an accessor: + +```cpp +#include "hip_kernel_provider_common/JsonDataSource.hpp" + +jlogic::JsonDataSource src(nlohmann::json{{"q", {{"dims", {8, 16}}}}}); +auto expr = jlogic::compile(rule); +jlogic::Value r = expr(src); +``` + +It resolves dotted keys (`a.b.c`), `[N]` array subscripts (`arr[0]`, +`rows[2].name`, `grid[0][1]`), and dot-form indices (`arr.1`). A leading +variable sigil is stripped, so `"$q.dims[0]"` and `"q.dims[0]"` address the same +location. It also offers the inverse, `setData`, which writes a `Value` back into +the document and creates intermediate objects and arrays on demand: + +```cpp +src.setData("$q.dims[0]", 2); // -> {"q":{"dims":[2, 16]}} +``` + +A `[N]` subscript grows an array (filling gaps with null); any other key creates +or descends into an object; the empty path replaces the whole document. +`setData` throws `std::invalid_argument` on a malformed path or a non-numeric +index applied to an array. Objects and null in the document read back as `Value` +null, matching `Value`'s scalar/array-only model. + +## `Value` + +A json-like tagged value with no external dependency. Alternatives: null, bool, +`int64_t`, `double`, `std::string`, and `Array` (`std::vector`). Numeric +results are stored as integers when exactly integral (so `1 + 1` is `2`, not +`2.0`). Key members: the `is*()` / `as*()` inspectors, `truthy()`, `toNumber()`, +`dump()`, strict `operator==`, and the static `compare`. + +There is intentionally no object alternative — nested structure is reached +through the data source's path accessor, not carried in a `Value`. + +## Inline variables + +Stock JsonLogic references data with `{"var": "path"}` and treats every bare +string as a literal. This implementation additionally lets a variable appear +anywhere a literal can, by prefixing it with a sigil (`$` by default): + +| Inline | Equivalent | Meaning | +| ------------ | ------------------- | ----------------------- | +| `"$x"` | `{"var": "x"}` | top-level key | +| `"$a.b.c"` | `{"var": "a.b.c"}` | nested path | +| `"$arr[0]"` | `{"var": "arr[0]"}` | array index (subscript) | +| `"$"` | `{"var": ""}` | whole document | +| `"$$text"` | `"$text"` | escaped string literal | + +Strings without the sigil stay literals, so stock rules keep working. Pass a +different sigil as the second argument to `compile` / `evaluate` if your keys +begin with `$`. + +## Supported operators + +Data access: `var` (dotted paths, `[N]` array subscripts, `""` whole-document, +`[path, default]` fallback, and computed `[expr, default]` paths); +`value_or_default` (`{"value_or_default": ["$x", fallback]}`) returns the +variable's value when the path resolves and the fallback otherwise. It keys on +*existence* (an unresolved path reads as `null`), not truthiness, so a present +`0`, `""`, or `false` is returned rather than the fallback. The fallback is any +expression, evaluated lazily, so `["$a", "$b"]` reads "this field, else that +one". + +Logic / control: `if` / `?:`, `and`, `or`, `!`, `!!` (short-circuit, lazy +branches). + +Comparison: `==`, `!=`, `<`, `<=`, `>`, `>=` (`<`/`<=` accept a third argument +for between-tests). `==`/`!=` are strict: no type coercion, so `1 == "1"` is +false (integers and doubles of equal value still compare equal). Ordering +(`<`/`<=`/`>`/`>=`) still uses numeric coercion. + +Arithmetic: `+`, `-` (binary and unary negation), `*`, `/`, `%`, `min`, `max`. +`/`, `%`, and `ceil_div` decline (yield `null`) on a zero divisor rather than +producing `inf`/`NaN`. + +Math extensions (value-core, for hipDNN dispatch/constraint formulas): `ceil_div` +(2-arg ceiling division), `abs`, `pow`, `log2`, `rsqrt` (`1/sqrt(x)`). `log2` and +`rsqrt` decline on a non-positive argument. + +Membership: `in`. `{"in": [needle, array]}` is true when `array` contains +`needle` (strict element equality); `{"in": [needle, string]}` is a substring +test. + +Presence: `present`, `not_present`. `{"present": ["$a", "$b"]}` is true when +every listed path resolves to a non-null value; `{"not_present": [...]}` is true +when every listed path resolves to null. Both are `and`-folds, so one call +decides a whole list. They key on the same *existence* mechanism as +`value_or_default`, and, with `value_or_default`, are the only operators that do +not propagate a null: an unresolved path yields `false`/`true` rather than +`null`, because answering "was this supplied?" is their whole job. + +## Null is unknown, and it propagates + +`null` means *unresolved*, not a value. Every operator except `present`, +`not_present`, and `value_or_default` returns `null` when any argument is +`null`, rather than coercing it to `false`/`0`/not-equal. This matters because +the consumer is hipDNN's UMD matcher, where an unresolved path is an absent +optional operand: if `null` coerced, a narrowing check such as +`{"!=": ["$bias.dtype", "BFLOAT16"]}` would evaluate **true** on a graph with no +bias and the matcher would accept input it cannot serve. Two `null`s are not +equal to each other either — the question is unanswerable, so `==` and `!=` both +decline. + +`and` and `or` are three-valued (Kleene): a definite `false` still decides an +`and` and a definite `true` still decides an `or`, even beside a `null` +argument; otherwise the result is `null`. That is what lets +`{"or": [{"not_present": ["$bias"]}, {"and": [{"present": ["$bias"]}, ...]}]}` +accept an absent operand whose field reads cannot run. + +Once every argument resolves, value semantics follow JsonLogic/JS: `false`, `0`, +`""`, `null` and the empty array are falsy; `Number()`-style coercion drives +arithmetic and ordering. A `null` root is falsy, so an undecided expression +declines. + +Malformed rules (unknown operator, wrong argument count, a non-operator object) +raise `JsonLogicCompileError` at `compile` time, so evaluation stays on the fast +path. + +Not included: collection/string operators (`map`, `reduce`, `filter`, `all`, +`some`, `none`, `merge`, `cat`, `substr`, `missing`, `missing_some`). + +## Tests + +Unit tests live in `src/tests/core/TestJsonLogic.cpp` and build into the +`hip_kernel_provider_tests` GTest binary. Run them with: + +```bash +ctest -R hip_kernel_provider_tests +# or, filtered directly on the binary: +./hip_kernel_provider_tests --gtest_filter=TestJsonLogic.* +``` diff --git a/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/BindingContext.hpp b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/BindingContext.hpp new file mode 100644 index 000000000000..ca28531e7960 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/BindingContext.hpp @@ -0,0 +1,460 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +// BindingContext.hpp - the queryable "bindings object" for RFC 0018 (UMD). +// +// All names below live in namespace hip_kernel_provider_common::umd; these +// examples assume `namespace umd = hip_kernel_provider_common::umd;`. +// +// After a UniversalGraphMatcher structurally matches a graph, it builds one +// BindingContext per match. The context is both: +// +// 1. the JsonLogic data source the criteria expression evaluates against +// (it satisfies the `Value getData(const std::string&) const` contract), and +// 2. the queryable bindings object the caller inspects post-match +// (`ctx.get("$q.head_size")`). +// +// It resolves the RFC's five `$`-namespaces over a live flatbuffer graph +// (RFC 0018 §4): +// - Tensor `$q`, `$q.uid`, `$q.rank`, `$q.dtype`, `$q.dims[i]`, +// `$q.strides[i]`, `$q.`, `$q.stride_order`, +// `$q.packed`, `$q.virtual`, `$q.present`, +// `$q.is_runtime_pass_by_value`, `$q.value_f32` +// - Graph `$graph.node_count`, `$graph.is_override_shape_enabled` +// - Attributes `$.`, `$..present` +// - Kernel `$kernel.` (from caller-supplied, fully-resolved KMD +// metadata; an unbound document is a defensive fallback that +// reads every field as null) +// - Device `$device.` +// +// The two resolvers (RFC 0018 §4, "binding architecture"): edge (name->UID-> +// tensor) and scalar-attribute reads use the Phase 0 *generated* typed +// accessors carried on the op-schema registry entry; the Tensor-namespace path +// resolver (dims/strides/rank/stride_order/packed/virtual/uid) is hand-written +// once over the single TensorAttributes shape. jlogic::Value is the sole +// type-erasure boundary. +// +// Fail-closed (RFC 0018 §4/§14): every unresolved reference -- an unknown root, +// an out-of-range `dims[i]`, an unknown dim-name, or a read of a field on an +// absent optional operand -- resolves to null, which makes the enclosing +// criterion false, declining the match rather than matching on a wrong value. + +#include "hip_kernel_provider_common/JsonDataSource.hpp" +#include "hip_kernel_provider_common/JsonLogic.hpp" +#include "hip_kernel_provider_common/umd/UmdPathParse.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace hip_kernel_provider_common::umd +{ + +namespace jlogic = hip_kernel_provider_common::jsonlogic; + +// A tensor pattern-variable bound to a concrete graph tensor. `tensor` is null +// only for an optional operand the graph omitted; `dimNames` points at the +// UMD-static name->index map the compiler's `shape` lowering produced (owned by +// the CompiledUmd, which outlives every BindingContext it spawns). +struct BoundTensor +{ + const hipdnn_flatbuffers_sdk::data_objects::TensorAttributes* tensor = nullptr; + bool optional = false; + const std::unordered_map* dimNames = nullptr; +}; + +// A pattern node bound to a concrete graph node's attributes, resolving the +// Attributes namespace `$.` for that node. +struct BoundNode +{ + const hipdnn_flatbuffers_sdk::umd::OpSchemaEntry* schema = nullptr; + const void* attributes = nullptr; +}; + +class BindingContext +{ +public: + using TensorAttributes = hipdnn_flatbuffers_sdk::data_objects::TensorAttributes; + using IGraph = hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph; + using OpSchemaEntry = hipdnn_flatbuffers_sdk::umd::OpSchemaEntry; + + BindingContext() = default; + + BindingContext(const IGraph* graph, std::unordered_map device) + : _graph(graph) + , _device(std::move(device)) + { + } + + // Bind a pattern node id (`$sdpa_fwd`) to a graph node's attributes. A UMD + // may declare several nodes, each resolving its own Attributes namespace. + void bindNode(const std::string& nodeId, const OpSchemaEntry* schema, const void* attributes) + { + _nodes[nodeId] = BoundNode{schema, attributes}; + } + + // Bind a tensor pattern variable (`$q`) to a graph tensor. A null `tensor` + // records an absent optional operand (`$q.present` reads false; any other + // field read declines). + void bindTensor(const std::string& tvar, + const TensorAttributes* tensor, + bool optional, + const std::unordered_map* dimNames) + { + _tensors[tvar] = BoundTensor{tensor, optional, dimNames}; + } + + // Bind the kernel metadata the criteria's `$kernel.` references + // resolve against (RFC 0018 §4). The document is a UKD's fully-resolved KMD + // values (optional fields already filled with schema defaults, so every + // field a referenced criterion needs is present). An unset or empty + // document is a defensive fallback in which every kernel field reads null. + void bindKernelMetadata(nlohmann::json metadata) + { + _kernel = jlogic::JsonDataSource(std::move(metadata)); + } + + // JsonLogic data-source contract: resolve a sigil-stripped variable path + // (`q.head_size`, `graph.node_count`, `sdpa_fwd.dropout_probability.present`) + // to a Value. Unresolved -> null (fail closed). + jlogic::Value getData(const std::string& varPath) const + { + std::string root; + std::string rest; + path::splitRoot(varPath, root, rest); + + if(root == "graph") + { + return resolveGraph(rest); + } + if(root == "device") + { + return resolveDevice(rest); + } + if(root == "kernel") + { + return _kernel.getData(rest); + } + const auto nit = _nodes.find(root); + if(nit != _nodes.end()) + { + return resolveAttr(nit->second, rest); + } + const auto it = _tensors.find(root); + if(it != _tensors.end()) + { + return resolveTensor(it->second, rest); + } + return {}; // unknown root -> decline + } + + // Query helper for post-match inspection: accepts a leading `$` sigil. + jlogic::Value get(const std::string& ref) const + { + if(!ref.empty() && ref.front() == '$') + { + return getData(ref.substr(1)); + } + return getData(ref); + } + + // The tensor pattern variables bound in this match, for symbol-table + // enumeration (RFC 0018 §15). `tensor` is null for an absent optional. + const std::unordered_map& boundTensors() const + { + return _tensors; + } + + // The pattern nodes bound in this match, keyed by node id. + const std::unordered_map& boundNodes() const + { + return _nodes; + } + +private: + static std::vector toVector(const ::flatbuffers::Vector* v) + { + if(v == nullptr) + { + return {}; + } + // NOLINTNEXTLINE(modernize-return-braced-init-list) - iterator-range ctor, not initializer_list + return std::vector(v->begin(), v->end()); + } + + jlogic::Value resolveGraph(const std::string& rest) const + { + if(_graph == nullptr) + { + return {}; + } + if(rest == "node_count") + { + return {static_cast(_graph->nodeCount())}; + } + if(rest == "is_override_shape_enabled") + { + // The graph's own state, read from the same accessor the matcher's + // override-shape gate uses. Distinct from the descriptor's + // `allow_override_shape` key, which is the matcher's opt-in. + return {_graph->getGraph().is_override_shape_enabled()}; + } + return {}; + } + + jlogic::Value resolveDevice(const std::string& rest) const + { + const auto it = _device.find(rest); + return it != _device.end() ? it->second : jlogic::Value(); + } + + static jlogic::Value resolveAttr(const BoundNode& node, const std::string& rest) + { + if(node.schema == nullptr || node.attributes == nullptr) + { + return {}; + } + if(rest.empty()) + { + return {}; // a bare node id is not a value reference (compiler-rejected) + } + + std::string attr = rest; + const bool wantPresent = path::stripPresentSuffix(attr); + + const hipdnn_flatbuffers_sdk::umd::AttrBinding* binding = nullptr; + for(std::size_t i = 0; i < node.schema->attributeCount; ++i) + { + if(node.schema->attributes[i].name == attr) + { + binding = &node.schema->attributes[i]; + break; + } + } + if(binding == nullptr || binding->read == nullptr) + { + return {}; + } + + const hipdnn_flatbuffers_sdk::umd::ScalarValue sv = binding->read(node.attributes); + if(wantPresent) + { + return {sv.present}; + } + if(!sv.present) + { + return {}; // absent optional attribute read -> decline + } + switch(sv.type) + { + case hipdnn_flatbuffers_sdk::umd::AttrType::INT: + return {sv.i}; + case hipdnn_flatbuffers_sdk::umd::AttrType::FLOAT: + return {sv.f}; + case hipdnn_flatbuffers_sdk::umd::AttrType::BOOL: + return {sv.b}; + case hipdnn_flatbuffers_sdk::umd::AttrType::DTYPE: + return {std::string(sv.dtype != nullptr ? sv.dtype : "")}; + default: + break; + } + return {}; + } + + static jlogic::Value resolveTensor(const BoundTensor& bt, const std::string& rest) + { + if(rest == "present") + { + if(!bt.optional) + { + return {}; // present on a required operand is refused (compiler-gated) + } + return {bt.tensor != nullptr}; + } + if(bt.tensor == nullptr) + { + return {}; // field read on an absent optional operand -> decline + } + const TensorAttributes* t = bt.tensor; + + if(rest.empty() || rest == "uid") + { + return {static_cast(t->uid())}; + } + if(rest == "rank") + { + return {static_cast(t->dims() != nullptr ? t->dims()->size() : 0)}; + } + if(rest == "dtype") + { + return {std::string( + hipdnn_flatbuffers_sdk::data_objects::EnumNameDataType(t->data_type()))}; + } + if(rest == "virtual") + { + return {t->virtual_()}; + } + if(rest == "packed") + { + return {hipdnn_data_sdk::utilities::isTensorPacked(toVector(t->dims()), + toVector(t->strides()))}; + } + if(rest == "is_runtime_pass_by_value") + { + return {t->is_runtime_pass_by_value()}; + } + if(rest == "value_f32") + { + return resolveValueF32(*t); + } + if(rest == "stride_order") + { + // Published in the RFC 0017 §5 form: logical dimension indices + // ordered outermost (largest-stride) first, so `[0,2,3,1]` over an + // (n,c,h,w) dim order spells N,H,W,C. extractStrideOrder returns + // the inverse -- entry `d` is dimension `d`'s stride rank, higher + // meaning slower-varying -- so invert it here, once, at the one + // place a descriptor-visible value is minted. + const std::vector ranks + = hipdnn_data_sdk::utilities::extractStrideOrder(toVector(t->strides())); + jlogic::Value::Array arr(ranks.size(), jlogic::Value(std::int64_t{0})); + const auto rank = static_cast(ranks.size()); + for(std::size_t dim = 0; dim < ranks.size(); ++dim) + { + // A dim of stride rank r sits at physical position rank-1-r. + const std::int64_t position = rank - 1 - ranks[dim]; + if(position < 0 || position >= rank) + { + return {}; // malformed rank vector -> decline (fail closed) + } + arr[static_cast(position)] + = jlogic::Value(static_cast(dim)); + } + return {std::move(arr)}; + } + + std::size_t idx = 0; + if(path::parseSubscript(rest, "dims", idx)) + { + const auto* dims = t->dims(); + if(dims == nullptr || idx >= dims->size()) + { + return {}; + } + return { + static_cast(dims->Get(static_cast<::flatbuffers::uoffset_t>(idx)))}; + } + if(path::parseSubscript(rest, "strides", idx)) + { + const auto* strides = t->strides(); + if(strides == nullptr || idx >= strides->size()) + { + return {}; + } + return {static_cast( + strides->Get(static_cast<::flatbuffers::uoffset_t>(idx)))}; + } + + // Named dim introduced by a `shape` short-hand. + if(bt.dimNames != nullptr) + { + const auto dit = bt.dimNames->find(rest); + if(dit != bt.dimNames->end()) + { + const auto* dims = t->dims(); + if(dims == nullptr || dit->second >= dims->size()) + { + return {}; + } + return {static_cast( + dims->Get(static_cast<::flatbuffers::uoffset_t>(dit->second)))}; + } + } + return {}; // unknown tensor field -> decline + } + + // Coerce whichever arm of the tensor's `value` union is set to f32 and + // publish it as the single `$q.value_f32` token (RFC 0017 §5). A tensor + // carrying no compile-time value resolves to null, so a criterion over it + // declines rather than reading a zero that would satisfy a comparison. + static jlogic::Value resolveValueF32(const TensorAttributes& t) + { + namespace data = hipdnn_flatbuffers_sdk::data_objects; + switch(t.value_type()) + { + case data::TensorValue::Float32Value: + return {static_cast(t.value_as_Float32Value()->value())}; + case data::TensorValue::Float16Value: + return {static_cast(t.value_as_Float16Value()->value())}; + case data::TensorValue::BFloat16Value: + return {static_cast(t.value_as_BFloat16Value()->value())}; + case data::TensorValue::Float64Value: + return {t.value_as_Float64Value()->value()}; + case data::TensorValue::Int32Value: + return {static_cast(t.value_as_Int32Value()->value())}; + case data::TensorValue::Int64Value: + return {static_cast(t.value_as_Int64Value()->value())}; + case data::TensorValue::BoolValue: + return {t.value_as_BoolValue()->value() ? 1.0 : 0.0}; + case data::TensorValue::Float8Value: + return fp8ValueToF32(t.value_as_Float8Value()->value(), t.data_type()); + default: + break; + } + return {}; // TensorValue::NONE -> no compile-time value -> decline + } + + // Float8Value stores raw bits; the tensor's data_type says which 8-bit + // format they encode. An arm/data_type pairing that names no 8-bit format + // resolves to null (fail closed) rather than reinterpreting the bits. + static jlogic::Value fp8ValueToF32(std::uint8_t bits, + hipdnn_flatbuffers_sdk::data_objects::DataType dtype) + { + namespace data = hipdnn_flatbuffers_sdk::data_objects; + namespace types = hipdnn_data_sdk::types; + switch(dtype) + { + case data::DataType::FP8_E4M3: + return {static_cast(static_cast(types::fp8_e4m3::from_bits(bits)))}; + case data::DataType::FP8_E5M2: + return {static_cast(static_cast(types::fp8_e5m2::from_bits(bits)))}; + case data::DataType::FP8_E4M3_FNUZ: + return {static_cast(static_cast(types::fp8_e4m3_fnuz::from_bits(bits)))}; + case data::DataType::FP8_E5M2_FNUZ: + return {static_cast(static_cast(types::fp8_e5m2_fnuz::from_bits(bits)))}; + case data::DataType::FP8_E8M0: + return {static_cast(static_cast(types::fp8_e8m0::from_bits(bits)))}; + case data::DataType::UINT8: + return {static_cast(bits)}; + case data::DataType::INT8: + // Reinterpret the raw byte as signed: an INT8 -1 is stored as 0xFF + // and must read as -1.0, not 255.0. + return {static_cast(static_cast(bits))}; + default: + break; + } + return {}; + } + + const IGraph* _graph = nullptr; + std::unordered_map _device; + std::unordered_map _nodes; + std::unordered_map _tensors; + jlogic::JsonDataSource _kernel; // empty until bindKernelMetadata() +}; + +} // namespace hip_kernel_provider_common::umd diff --git a/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/UmdCompiler.hpp b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/UmdCompiler.hpp new file mode 100644 index 000000000000..cd21b04e9963 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/UmdCompiler.hpp @@ -0,0 +1,1382 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +// UmdCompiler.hpp - compiles a Universal Match Descriptor (RFC 0018) into a +// reusable CompiledUmd. +// +// All names below live in namespace hip_kernel_provider_common::umd. +// +// The compiler turns a `hipdnn.umd/v1` descriptor (a nlohmann::json document) +// into a CompiledUmd the UniversalGraphMatcher runs against graphs. It: +// 1. validates the descriptor against Appendix A.10 (schema/id, node object, +// registry-resolved opcodes and names, `?`<->optionality, reserved roots, +// single producer); +// 2. lowers the JSON criteria to pure JsonLogic: expands the `shape` +// short-hand (A.5) to a rank check while recording each named dim on its +// tensor, and expands layout aliases (A.8) to stride-order arrays; +// 3. statically type-checks the lowered criteria against the Phase 0 +// generated registry (A.10 §9) -- the payoff of the strongly-typed +// codegen; a type mismatch is a compile error, not a runtime decline; +// 4. compiles the criteria to a jlogic::Expression and +// publishes the referenced bound-symbol set (RFC 0018 §4). +// +// A descriptor that violates any check throws UmdCompileError (RFC 0018 §10 / +// A.10: refuse at compile, never match by default). + +#include "hip_kernel_provider_common/JsonLogic.hpp" +#include "hip_kernel_provider_common/umd/BindingContext.hpp" +#include "hip_kernel_provider_common/umd/UmdPathParse.hpp" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace hip_kernel_provider_common::umd +{ + +namespace reg = hipdnn_flatbuffers_sdk::umd; + +// Thrown when a descriptor fails any Appendix A.10 compile-time check. +class UmdCompileError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; + +// One operand/result binding slot of a pattern node: the op-schema tensor name +// (`q`) and the pattern variable it binds (`q`), with the generated typed UID +// reader for that node's opcode. +struct EdgeSlot +{ + std::string schemaName; // op-schema registry tensor name, e.g. "q" / "attn_mask" + std::string tvar; // pattern variable, e.g. "q" + bool optional = false; + bool isOutput = false; // operand vs result + const reg::UidReader* reader = nullptr; // generated typed UID reader +}; + +// A pattern node: its id, opcode/op-schema, and the operand/result edges that +// bind pattern variables. A descriptor may declare several nodes; edges connect +// them when two nodes share a pattern variable (RFC 0018 §3 / A.3). +struct NodeSpec +{ + std::string id; + std::string opcode; + const reg::OpSchemaEntry* opSchema = nullptr; + std::vector edges; +}; + +// A pattern variable (`$q`) shared across the descriptor's nodes. `dimNames` is +// populated by the `shape` lowering and read by BindingContext. `optional` is +// true only when every binding slot referencing it is optional (its `.present` +// is then legal). +struct TensorVarSpec +{ + std::string tvar; + bool optional = false; + std::unordered_map dimNames; // shape short-hand (A.5) + std::size_t pinnedRank = 0; // 0 == not pinned by a shape op +}; + +// A compiled descriptor: heap-stable (owned via unique_ptr by the matcher) so +// BindingContext may reference its TensorVarSpec::dimNames maps for the match's +// lifetime. +struct CompiledUmd +{ + // The descriptor's identity (RFC 0018 §13): a UUID, held as the 128-bit + // value hipDNN carries every descriptor and graph id in, not as text. + hipdnn_flatbuffers_sdk::utilities::UuidBytes id{}; + std::string name; + // Matcher format version and the hipDNN graph schema version this matcher + // was authored against; empty when the descriptor declares neither + // (RFC 0017 §4). + std::string version; + std::string sdkVersion; + bool allowOverrideShape = false; + std::vector nodes; + std::vector tvars; + jlogic::Expression criteria; + std::set boundSymbols; + + // The `$kernel.` names the criteria expression reads, without the + // `kernel.` prefix (`$kernel.tile_m` contributes "tile_m"). Set once by + // UmdCompiler::run() and immutable thereafter; empty for a graph-only + // matcher, which is what routes between the metadata and non-metadata + // match overloads. A KDP/KMD loader checks each name exists in the engine's + // KMD and uses the set as the per-kernel memoization key (RFC 0017 §5). + std::set kernelFields; + + const TensorVarSpec* findTvar(const std::string& tvar) const + { + for(const auto& t : tvars) + { + if(t.tvar == tvar) + { + return &t; + } + } + return nullptr; + } + + TensorVarSpec* findTvarMut(const std::string& tvar) + { + for(auto& t : tvars) + { + if(t.tvar == tvar) + { + return &t; + } + } + return nullptr; + } + + const NodeSpec* findNode(const std::string& nodeId) const + { + for(const auto& n : nodes) + { + if(n.id == nodeId) + { + return &n; + } + } + return nullptr; + } +}; + +class UmdCompiler +{ +public: + static CompiledUmd compile(const nlohmann::json& descriptor) + { + UmdCompiler c; + return c.run(descriptor); + } + + // The matcher format version this compiler implements. Gating is a + // CEILING, not a floor (RFC 0017 §4): a descriptor whose major differs, or + // whose minor is newer than this runtime's, is refused, because it carries + // features this runtime cannot understand. An older minor within the same + // major always loads -- a file stamped "1.0" loads on a "1.1" runtime. + static constexpr const char* K_RUNTIME_VERSION = "1.0"; + + // The hipDNN graph schema (SDK) version this compiler understands. Unlike + // the format version, `sdk_version` is gated against a floor the GRAPH + // sets, not a constant: a graph reports the schema version its own + // contents require, and a matcher declaring less is declined before it runs + // (RFC 0017 §4). This constant is only the ceiling half -- the upper bound + // on what a matcher may claim to understand. The per-graph floor is applied + // by UniversalGraphMatcher via graphRequiredSdkVersion(). + static constexpr const char* K_RUNTIME_SDK_VERSION = "1.2"; + + // The version an omitted `version` / `sdk_version` key means: the lowest + // this format ever had, which is what a descriptor authored before either + // key existed implies (RFC 0018 A.1). + static constexpr const char* K_DEFAULT_VERSION = "1.0"; + + // Parsed ".". A version that does not parse is not ordered + // against anything; callers treat that as a refusal (fail closed). + struct SemVer + { + std::size_t major = 0; + std::size_t minor = 0; + + bool operator<(const SemVer& o) const + { + return major != o.major ? major < o.major : minor < o.minor; + } + bool operator<=(const SemVer& o) const + { + return !(o < *this); + } + }; + + // Parse "."; false when the string is not exactly two + // digit-only components. + static bool parseSemVer(const std::string& s, SemVer& out) + { + return parseVersion(s, out.major, out.minor); + } + + // True when `version` is at or above `floor`. Compared numerically on + // (major, minor), never lexicographically, so "1.10" ranks above "1.9". A + // version that does not parse is below every floor (fail closed). + static bool versionAtLeast(const std::string& version, const std::string& floor) + { + SemVer v; + SemVer f; + if(!parseSemVer(version, v) || !parseSemVer(floor, f)) + { + return false; + } + return f <= v; + } + + // True when a descriptor stamped `version` is loadable on a runtime + // implementing `runtime`: same major, and a minor no newer than the + // runtime's (RFC 0017 §4). An unparseable version loads nowhere. + static bool versionLoadableOn(const std::string& version, const std::string& runtime) + { + SemVer v; + SemVer r; + if(!parseSemVer(version, v) || !parseSemVer(runtime, r)) + { + return false; + } + return v.major == r.major && v.minor <= r.minor; + } + +private: + // Static type domain for A.10 §9. Numeric unifies Int and Float; ARRAY + // carries its element kind for `in` / IntArray equality. DYNAMIC is a + // runtime-resolved scalar wildcard: `$kernel.` values come from a + // UKD's KMD, unavailable at UMD-compile time (RFC 0018 §4/§10), so they are + // not statically type-checked here. A referenced kernel field is assumed + // validated against the engine's KMD schema at load (a typo'd field is + // caught there, not here); the metadata supplied to match() is fully + // resolved, so every referenced field is present at match time. + enum class Kind + { + INT, + FLOAT, + BOOL, + DTYPE, + ARRAY, + TENSOR, + JSON_NULL, + DYNAMIC, + UNKNOWN + }; + struct TypeInfo + { + Kind kind = Kind::UNKNOWN; + Kind elem = Kind::UNKNOWN; // element kind when kind == ARRAY + }; + + static bool isNumeric(Kind k) + { + return k == Kind::INT || k == Kind::FLOAT; + } + + // Kind rendered for a diagnostic (A.10 §9). + static std::string kindName(Kind k) + { + switch(k) + { + case Kind::INT: + return "Int"; + case Kind::FLOAT: + return "Float"; + case Kind::BOOL: + return "Bool"; + case Kind::DTYPE: + return "Dtype"; + case Kind::ARRAY: + return "Array"; + case Kind::TENSOR: + return "Tensor"; + case Kind::JSON_NULL: + return "Null"; + case Kind::DYNAMIC: + return "Dynamic"; + default: + break; + } + return "Unknown"; + } + + CompiledUmd run(const nlohmann::json& d) + { + CompiledUmd out; + validateTopLevel(d); + out.id = parseId(d.at("id")); + out.name = d.at("name").get(); + // Both version keys are optional. An omitted key means "1.0", the + // version every descriptor authored against this revision implies + // (RFC 0018 A.1), so the published value is never empty and the + // per-graph SDK floor has a concrete version to compare against. + out.version = d.contains("version") ? d.at("version").get() + : std::string(K_DEFAULT_VERSION); + out.sdkVersion = d.contains("sdk_version") ? d.at("sdk_version").get() + : std::string(K_DEFAULT_VERSION); + out.allowOverrideShape + = d.contains("allow_override_shape") ? d.at("allow_override_shape").get() : false; + + parseNodes(d.at("nodes"), out); + + nlohmann::json const lowered = lowerCriteria(d.at("criteria"), out); + const TypeInfo t = inferType(lowered, &out); + if(t.kind != Kind::BOOL && t.kind != Kind::DYNAMIC) + { + throw UmdCompileError("criteria must have static type Bool (A.10 §10)"); + } + + out.criteria = jlogic::compile(lowered); + for(const std::string& sym : out.criteria.variables()) + { + // Validate each reference resolves (A.10 §8); varTypeOf throws if not. + varTypeOf(sym, out); + out.boundSymbols.insert(sym); + std::string root; + std::string rest; + path::splitRoot(sym, root, rest); + if(root == "kernel") + { + out.kernelFields.insert(rest); + } + } + return out; + } + + // ---- A.1 / A.10 top level ------------------------------------------- + static void validateTopLevel(const nlohmann::json& d) + { + if(!d.is_object()) + { + throw UmdCompileError("descriptor must be a JSON object"); + } + static const std::set s_allowed = {"schema", + "version", + "sdk_version", + "id", + "name", + "allow_override_shape", + "nodes", + "criteria"}; + for(const auto& [key, value] : d.items()) + { + if(s_allowed.find(key) == s_allowed.end()) + { + throw UmdCompileError("unknown top-level key: " + key); + } + } + for(const char* required : {"schema", "id", "name", "nodes", "criteria"}) + { + if(!d.contains(required)) + { + throw UmdCompileError(std::string("missing required key: ") + required); + } + } + if(!d.at("schema").is_string() || d.at("schema").get() != "hipdnn.umd/v1") + { + throw UmdCompileError("schema must equal \"hipdnn.umd/v1\" (A.10 §1)"); + } + if(!d.at("id").is_string()) + { + throw UmdCompileError("id must be a string (A.10 §1)"); + } + if(!d.at("name").is_string()) + { + throw UmdCompileError("name must be a string"); + } + if(d.contains("allow_override_shape") && !d.at("allow_override_shape").is_boolean()) + { + throw UmdCompileError("allow_override_shape must be a boolean"); + } + validateVersionKey(d, "version", K_RUNTIME_VERSION); + validateVersionKey(d, "sdk_version", K_RUNTIME_SDK_VERSION); + if(!d.at("nodes").is_array() || d.at("nodes").empty()) + { + throw UmdCompileError("nodes must be a non-empty array (A.1)"); + } + } + + // A.10 §1: the id must be a well-formed UUID. Parsing it here is the check + // -- a malformed id is refused at compile rather than surviving as an + // unusable string. + static hipdnn_flatbuffers_sdk::utilities::UuidBytes parseId(const nlohmann::json& id) + { + try + { + return hipdnn_flatbuffers_sdk::utilities::parseUuid(id.get()); + } + catch(const std::invalid_argument&) + { + throw UmdCompileError("id must be a well-formed UUID (A.10 §1)"); + } + } + + // Both version keys are optional; when present each must be a + // "." string this runtime can honor. The gate is a ceiling + // (RFC 0017 §4): same major, minor no newer than the runtime's. An older + // minor always loads, so a descriptor stays loadable on the oldest runtime + // that can serve it. The per-graph `sdk_version` FLOOR is a separate, + // match-time check (UniversalGraphMatcher), because only a graph can say + // which schema version its own contents require. + static void validateVersionKey(const nlohmann::json& d, const char* key, const char* runtime) + { + if(!d.contains(key)) + { + return; + } + if(!d.at(key).is_string()) + { + throw UmdCompileError(std::string(key) + + " must be a \".\" string (A.10 §1)"); + } + const std::string declared = d.at(key).get(); + SemVer parsed; + if(!parseSemVer(declared, parsed)) + { + throw UmdCompileError(std::string(key) + " \"" + declared + + "\" is not a \".\" string (A.10 §1)"); + } + if(!versionLoadableOn(declared, runtime)) + { + throw UmdCompileError(std::string(key) + " " + declared + + " is not supported by this runtime, which implements " + runtime + + " (A.10 §1)"); + } + } + + static bool parseVersion(const std::string& s, std::size_t& major, std::size_t& minor) + { + const std::size_t dot = s.find('.'); + if(dot == std::string::npos || s.find('.', dot + 1) != std::string::npos) + { + return false; + } + return parseVersionComponent(s.substr(0, dot), major) + && parseVersionComponent(s.substr(dot + 1), minor); + } + + // A version component is a run of decimal digits: no sign, no whitespace, + // no exponent. The length cap keeps the accumulation below overflow. + static bool parseVersionComponent(const std::string& s, std::size_t& out) + { + if(s.empty() || s.size() > 9) + { + return false; + } + std::size_t value = 0; + for(const char c : s) + { + if(std::isdigit(static_cast(c)) == 0) + { + return false; + } + value = (value * 10) + static_cast(c - '0'); + } + out = value; + return true; + } + + static bool isReservedRoot(const std::string& s) + { + return s == "graph" || s == "kernel" || s == "device"; + } + + // ---- A.2 / A.3 node object ------------------------------------------ + static void parseNodes(const nlohmann::json& nodes, CompiledUmd& out) + { + for(const auto& node : nodes) + { + parseNode(node, out); + } + + // A.10 §5: node ids unique. + for(std::size_t i = 0; i < out.nodes.size(); ++i) + { + for(std::size_t j = i + 1; j < out.nodes.size(); ++j) + { + if(out.nodes[i].id == out.nodes[j].id) + { + throw UmdCompileError("duplicate node id (A.10 §5): " + out.nodes[i].id); + } + } + } + + // A.10 §5: pattern variables disjoint from node ids and reserved roots. + for(const TensorVarSpec& t : out.tvars) + { + if(out.findNode(t.tvar) != nullptr) + { + throw UmdCompileError("pattern variable collides with a node id (A.10 §5): " + + t.tvar); + } + if(isReservedRoot(t.tvar)) + { + throw UmdCompileError("pattern variable uses a reserved root (A.10 §5): " + t.tvar); + } + } + } + + static void parseNode(const nlohmann::json& node, CompiledUmd& out) + { + if(!node.is_object()) + { + throw UmdCompileError("node must be a JSON object"); + } + static const std::set s_allowed = {"kind", "id", "op", "operands", "results"}; + for(const auto& [key, value] : node.items()) + { + if(s_allowed.find(key) == s_allowed.end()) + { + throw UmdCompileError("unknown node key: " + key); + } + } + if(!node.contains("kind") || node.at("kind").get() != "op") + { + throw UmdCompileError("node kind must be \"op\" (A.2)"); + } + if(!node.contains("id") || !node.at("id").is_string()) + { + throw UmdCompileError("node must have a string id (A.2)"); + } + NodeSpec spec; + spec.id = node.at("id").get(); + if(isReservedRoot(spec.id)) + { + throw UmdCompileError("node id must not be a reserved root (A.10 §5): " + spec.id); + } + if(!node.contains("op") || !node.at("op").is_string()) + { + // one_of / "any" op-selectors are deferred (opcode-string PoC scope). + throw UmdCompileError("node op must be an opcode string (PoC scope)"); + } + spec.opcode = node.at("op").get(); + spec.opSchema = reg::lookupOpByName(spec.opcode); + if(spec.opSchema == nullptr) + { + throw UmdCompileError("opcode does not resolve in the op-schema registry: " + + spec.opcode); + } + + if(node.contains("operands")) + { + parseNameMap(node.at("operands"), out, spec, /*isOutput=*/false); + } + if(node.contains("results")) + { + parseNameMap(node.at("results"), out, spec, /*isOutput=*/true); + } + out.nodes.push_back(std::move(spec)); + } + + static void + parseNameMap(const nlohmann::json& map, CompiledUmd& out, NodeSpec& spec, bool isOutput) + { + if(!map.is_object()) + { + throw UmdCompileError("operands/results must be an object (A.2)"); + } + for(const auto& [name, bindJson] : map.items()) + { + if(!bindJson.is_string()) + { + throw UmdCompileError("name binding must be a \"$var\" string: " + name); + } + std::string bind = bindJson.get(); + if(bind.empty() || bind.front() != '$') + { + throw UmdCompileError("name binding must start with '$': " + bind); + } + bool optional = false; + if(bind.back() == '?') + { + optional = true; + bind.pop_back(); + } + const std::string tvar = bind.substr(1); + if(tvar.empty()) + { + throw UmdCompileError("empty pattern variable in binding"); + } + + // A.10 §3/§4: name resolves in registry with matching optionality. + const reg::UidReader* reader = nullptr; + bool registryOptional = false; + if(!resolveTensorName(*spec.opSchema, name, isOutput, reader, registryOptional)) + { + throw UmdCompileError("name does not resolve in the op-schema registry for " + + spec.opcode + ": " + name); + } + if(registryOptional != optional) + { + throw UmdCompileError( + "`?` suffix does not match registry optionality for name (A.10 §4): " + name); + } + // A.3: single producer -- a variable is a `results` value at most + // once across all nodes. + if(isOutput) + { + for(const NodeSpec& n : out.nodes) + { + for(const EdgeSlot& e : n.edges) + { + if(e.isOutput && e.tvar == tvar) + { + throw UmdCompileError("pattern variable produced more than once (A.3): " + + tvar); + } + } + } + for(const EdgeSlot& e : spec.edges) + { + if(e.isOutput && e.tvar == tvar) + { + throw UmdCompileError("pattern variable produced more than once (A.3): " + + tvar); + } + } + } + + spec.edges.push_back(EdgeSlot{name, tvar, optional, isOutput, reader}); + registerTvar(out, tvar, optional); + } + } + + // Register (or update) a pattern variable. A variable is optional only when + // every binding slot referencing it is optional (RFC 0018 A.4: `.present` + // is legal only for an optional binding). + static void registerTvar(CompiledUmd& out, const std::string& tvar, bool optional) + { + TensorVarSpec* existing = out.findTvarMut(tvar); + if(existing == nullptr) + { + TensorVarSpec spec; + spec.tvar = tvar; + spec.optional = optional; + out.tvars.push_back(std::move(spec)); + return; + } + existing->optional = existing->optional && optional; + } + + static bool resolveTensorName(const reg::OpSchemaEntry& e, + const std::string& name, + bool isOutput, + const reg::UidReader*& reader, + bool& optional) + { + if(isOutput) + { + for(std::size_t i = 0; i < e.outputTensorCount; ++i) + { + if(e.outputTensors[i].name == name) + { + reader = &e.outputTensors[i].read; + optional = e.outputTensors[i].optional; + return true; + } + } + return false; + } + for(std::size_t i = 0; i < e.inputTensorCount; ++i) + { + if(e.inputTensors[i].name == name) + { + reader = &e.inputTensors[i].read; + optional = e.inputTensors[i].optional; + return true; + } + } + return false; + } + + // ---- A.5 / A.8 lowering --------------------------------------------- + nlohmann::json lowerCriteria(const nlohmann::json& node, CompiledUmd& out) + { + if(node.is_object()) + { + if(node.size() == 1 && node.begin().key() == "shape") + { + return lowerShape(node.begin().value(), out); + } + nlohmann::json result = nlohmann::json::object(); + for(const auto& [key, value] : node.items()) + { + nlohmann::json lowered = lowerCriteria(value, out); + if((key == "==" || key == "!=") && lowered.is_array() && lowered.size() == 2) + { + expandLayoutAlias(lowered, out); + } + result[key] = std::move(lowered); + } + return result; + } + if(node.is_array()) + { + nlohmann::json result = nlohmann::json::array(); + for(const auto& e : node) + { + result.push_back(lowerCriteria(e, out)); + } + return result; + } + return node; + } + + static nlohmann::json lowerShape(const nlohmann::json& args, CompiledUmd& out) + { + if(!args.is_array() || args.size() != 2 || !args.at(0).is_string() + || !args.at(1).is_array()) + { + throw UmdCompileError("shape short-hand must be [\"$tensor\", [entries]] (A.5)"); + } + const std::string ref = args.at(0).get(); + if(ref.empty() || ref.front() != '$') + { + throw UmdCompileError("shape target must be a tensor reference: " + ref); + } + const std::string tvar = ref.substr(1); + TensorVarSpec* spec = out.findTvarMut(tvar); + if(spec == nullptr) + { + throw UmdCompileError("shape target is not a bound tensor: " + ref); + } + + const nlohmann::json& entries = args.at(1); + bool hasCapture = false; + std::size_t idx = 0; + for(const auto& entry : entries) + { + if(!entry.is_string()) + { + throw UmdCompileError("shape entry must be a string (A.5)"); + } + const std::string name = entry.get(); + if(hasCapture) + { + throw UmdCompileError("shape capture must be the last entry (A.5/A.10 §7)"); + } + if(name == "_") + { + ++idx; // anonymous positional + continue; + } + if(name.front() == '$') + { + hasCapture = true; // trailing capture vector; rank pinned as >= count-1 + ++idx; + continue; + } + if(isReservedTensorField(name)) + { + throw UmdCompileError( + "shape dim-name collides with a reserved tensor-field (A.10 §7): " + name); + } + if(spec->dimNames.find(name) != spec->dimNames.end()) + { + throw UmdCompileError("duplicate shape dim-name (A.5): " + name); + } + spec->dimNames[name] = idx; + ++idx; + } + + const std::string rankRef = "$" + tvar + ".rank"; + if(hasCapture) + { + spec->pinnedRank = 0; // variable rank + return nlohmann::json{ + {">=", + nlohmann::json::array({rankRef, static_cast(entries.size() - 1)})}}; + } + spec->pinnedRank = entries.size(); + return nlohmann::json{ + {"==", nlohmann::json::array({rankRef, static_cast(entries.size())})}}; + } + + static bool isReservedTensorField(const std::string& name) + { + static const std::set s_reserved = {"uid", + "rank", + "dtype", + "stride_order", + "packed", + "virtual", + "present", + "dims", + "strides", + "is_runtime_pass_by_value", + "value_f32"}; + return s_reserved.find(name) != s_reserved.end(); + } + + static void expandLayoutAlias(nlohmann::json& args, CompiledUmd& out) + { + const int strideRefIdx = strideOrderRefIndex(args); + if(strideRefIdx < 0) + { + return; + } + const std::size_t aliasIdx = strideRefIdx == 0 ? 1 : 0; + if(!args.at(aliasIdx).is_string()) + { + return; + } + const std::string alias = args.at(aliasIdx).get(); + if(alias.empty() || alias.front() == '$') + { + return; // a tensor reference, not an alias + } + const std::string tvar = tvarOfStrideOrderRef( + args.at(static_cast(strideRefIdx)).get()); + std::size_t rank = 0; + const TensorVarSpec* spec = out.findTvar(tvar); + if(spec != nullptr) + { + rank = spec->pinnedRank; + } + args[aliasIdx] = aliasToArray(alias, rank); + } + + static int strideOrderRefIndex(const nlohmann::json& args) + { + for(int i = 0; i < 2; ++i) + { + if(args.at(static_cast(i)).is_string()) + { + const std::string s = args.at(static_cast(i)).get(); + if(!s.empty() && s.front() == '$' && endsWith(s, ".stride_order")) + { + return i; + } + } + } + return -1; + } + + static std::string tvarOfStrideOrderRef(const std::string& ref) + { + // "$q.stride_order" -> "q" + const std::size_t dot = ref.find('.'); + return ref.substr(1, dot - 1); + } + + // Expand a layout alias to a stride-order array, in the RFC 0017 §5 form + // BindingContext publishes: logical dimension indices ordered outermost + // (largest-stride) first, so a contiguous rank-4 tensor is [0,1,2,3] and + // NHWC is [0,2,3,1] -- the array reads as the layout it names. This is the + // inverse of the stride-rank vector extractStrideOrder returns; + // BindingContext converts once when it binds `$q.stride_order`, so + // `stride_order == alias` holds against a live graph's strides. + // RFC 0018 A.8 tabulates these arrays. + static nlohmann::json aliasToArray(const std::string& alias, std::size_t rank) + { + nlohmann::json arr; + if(alias == "nchw" || alias == "bhsd") + { + arr = nlohmann::json::array({0, 1, 2, 3}); + } + else if(alias == "nhwc") + { + arr = nlohmann::json::array({0, 2, 3, 1}); + } + else if(alias == "ncdhw") + { + arr = nlohmann::json::array({0, 1, 2, 3, 4}); + } + else if(alias == "ndhwc") + { + arr = nlohmann::json::array({0, 2, 3, 4, 1}); + } + else if(alias == "contiguous") + { + if(rank == 0) + { + throw UmdCompileError("layout alias \"contiguous\" requires a rank pinned by a " + "shape short-hand (A.8)"); + } + arr = nlohmann::json::array(); + for(std::size_t i = 0; i < rank; ++i) + { + arr.push_back(static_cast(i)); + } + } + else + { + throw UmdCompileError("unknown layout alias (A.8): " + alias); + } + // A fixed-rank alias compared against a tensor whose shape short-hand + // pinned a different rank can never hold; refuse it at compile rather + // than silently always declining at match time (A.8). + if(rank != 0 && arr.size() != rank) + { + throw UmdCompileError( + "layout alias \"" + alias + "\" has rank " + std::to_string(arr.size()) + + " but the tensor is pinned to rank " + std::to_string(rank) + " (A.8)"); + } + return arr; + } + + static bool endsWith(const std::string& s, const std::string& suffix) + { + return s.size() >= suffix.size() + && s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0; + } + + // ---- A.10 §9 static type-check -------------------------------------- + TypeInfo inferType(const nlohmann::json& node, const CompiledUmd* out = nullptr) + { + // `out` threads the compiled context through recursion; it is set once + // by run() via a member and reused. Store it on first entry. + if(out != nullptr) + { + _ctx = out; + } + return inferTypeImpl(node); + } + + TypeInfo inferTypeImpl(const nlohmann::json& node) + { + if(node.is_object()) + { + if(node.size() != 1) + { + throw UmdCompileError("expression object must have exactly one operator key (A.6)"); + } + return inferOp(node.begin().key(), node.begin().value()); + } + if(node.is_array()) + { + TypeInfo info; + info.kind = Kind::ARRAY; + info.elem = Kind::UNKNOWN; + for(const auto& e : node) + { + const TypeInfo et = inferTypeImpl(e); + if(info.elem == Kind::UNKNOWN) + { + info.elem = et.kind; + } + else if(!sameDomain(info.elem, et.kind)) + { + throw UmdCompileError("array literal has mixed element types (A.7 `in`)"); + } + } + return info; + } + if(node.is_boolean()) + { + return {Kind::BOOL, Kind::UNKNOWN}; + } + if(node.is_number_float()) + { + return {Kind::FLOAT, Kind::UNKNOWN}; + } + if(node.is_number()) + { + return {Kind::INT, Kind::UNKNOWN}; + } + if(node.is_string()) + { + const std::string s = node.get(); + if(!s.empty() && s.front() == '$') + { + // A.4: any string beginning with `$` is a variable reference; no + // escape form exists, so a malformed ref resolves (and throws) + // through varTypeOf rather than being reinterpreted as a literal. + return varTypeOf(s.substr(1), *_ctx); + } + return {Kind::DTYPE, Kind::UNKNOWN}; // enum name / opcode literal + } + return {Kind::JSON_NULL, Kind::UNKNOWN}; + } + + static bool sameDomain(Kind a, Kind b) + { + // DYNAMIC (a runtime-resolved $kernel field) is a scalar wildcard: it + // unifies with any scalar domain, but not with ARRAY or TENSOR. Its + // concrete scalar type is unknown until match time. + if(a == Kind::DYNAMIC) + { + return b != Kind::ARRAY && b != Kind::TENSOR; + } + if(b == Kind::DYNAMIC) + { + return a != Kind::ARRAY && a != Kind::TENSOR; + } + if(isNumeric(a) && isNumeric(b)) + { + return true; + } + return a == b; + } + + TypeInfo inferOp(const std::string& op, const nlohmann::json& operand) + { + std::vector args; + if(operand.is_array()) + { + for(const auto& a : operand) + { + args.push_back(a); + } + } + else + { + args.push_back(operand); // unary sugar + } + + const auto argType = [&](std::size_t i) { return inferTypeImpl(args.at(i)); }; + // Reject arity violations at compile with a UmdCompileError (A.7/A.10 §9) + // rather than letting a fixed-index argType() throw std::out_of_range. + // `max == 0` means unbounded (n-ary). + const auto requireArity = [&](std::size_t minArgs, std::size_t maxArgs) { + if(args.size() < minArgs || (maxArgs != 0 && args.size() > maxArgs)) + { + throw UmdCompileError("operator '" + op + "' has wrong arity (A.7)"); + } + }; + const auto requireBool = [&](const TypeInfo& t) { + if(t.kind != Kind::BOOL && t.kind != Kind::DYNAMIC) + { + throw UmdCompileError("operator '" + op + "' requires Bool arguments (A.7)"); + } + }; + const auto requireNumeric = [&](const TypeInfo& t) { + if(!isNumeric(t.kind) && t.kind != Kind::DYNAMIC) + { + throw UmdCompileError("operator '" + op + "' requires numeric arguments (A.7)"); + } + }; + + if(op == "and" || op == "or") + { + requireArity(2, 0); + for(std::size_t i = 0; i < args.size(); ++i) + { + requireBool(argType(i)); + } + return {Kind::BOOL, Kind::UNKNOWN}; + } + if(op == "!" || op == "!!") + { + requireArity(1, 1); + requireBool(argType(0)); + return {Kind::BOOL, Kind::UNKNOWN}; + } + if(op == "==" || op == "!=") + { + requireArity(2, 2); + const TypeInfo a = argType(0); + const TypeInfo b = argType(1); + if(!sameDomain(a.kind, b.kind)) + { + throw UmdCompileError("'" + op + "' compares values of different types (A.7)"); + } + return {Kind::BOOL, Kind::UNKNOWN}; + } + if(op == "<" || op == "<=" || op == ">" || op == ">=") + { + requireArity(2, 2); + requireNumeric(argType(0)); + requireNumeric(argType(1)); + return {Kind::BOOL, Kind::UNKNOWN}; + } + if(op == "in") + { + requireArity(2, 2); + const TypeInfo needle = argType(0); + const TypeInfo hay = argType(1); + if(hay.kind != Kind::ARRAY) + { + throw UmdCompileError("'in' requires an array second argument (A.7)"); + } + if(hay.elem != Kind::UNKNOWN && !sameDomain(needle.kind, hay.elem)) + { + throw UmdCompileError("'in' element type mismatch (A.7)"); + } + return {Kind::BOOL, Kind::UNKNOWN}; + } + if(op == "+" || op == "*" || op == "min" || op == "max") + { + requireArity(2, 0); // n-ary + Kind k = Kind::INT; + for(std::size_t i = 0; i < args.size(); ++i) + { + const TypeInfo t = argType(i); + requireNumeric(t); + if(t.kind == Kind::FLOAT) + { + k = Kind::FLOAT; + } + } + return {k, Kind::UNKNOWN}; + } + if(op == "-") + { + requireArity(2, 2); + const TypeInfo a = argType(0); + const TypeInfo b = argType(1); + requireNumeric(a); + requireNumeric(b); + return {(a.kind == Kind::FLOAT || b.kind == Kind::FLOAT) ? Kind::FLOAT : Kind::INT, + Kind::UNKNOWN}; + } + if(op == "%" || op == "ceil_div") + { + requireArity(2, 2); + requireNumeric(argType(0)); + requireNumeric(argType(1)); + return {Kind::INT, Kind::UNKNOWN}; + } + if(op == "/" || op == "pow") + { + requireArity(2, 2); + requireNumeric(argType(0)); + requireNumeric(argType(1)); + return {Kind::FLOAT, Kind::UNKNOWN}; + } + if(op == "abs") + { + requireArity(1, 1); + const TypeInfo t = argType(0); + requireNumeric(t); + return {t.kind, Kind::UNKNOWN}; + } + if(op == "log2" || op == "rsqrt") + { + requireArity(1, 1); + requireNumeric(argType(0)); + return {Kind::FLOAT, Kind::UNKNOWN}; + } + if(op == "value_or_default") + { + // A.7: the fallback is any expression of the same type, not just a + // literal, so both arms are checked and unified. DYNAMIC carries no + // concrete type, so the other arm supplies the result kind. + requireArity(2, 2); + const TypeInfo a = argType(0); + const TypeInfo b = argType(1); + if(!sameDomain(a.kind, b.kind)) + { + throw UmdCompileError("'value_or_default' arms have different types (A.7): " + + kindName(a.kind) + " vs " + kindName(b.kind)); + } + return a.kind == Kind::DYNAMIC ? b : a; + } + if(op == "present" || op == "not_present") + { + // A.7: n-ary resolution predicates. Each argument is a variable + // reference resolved as itself -- never through the `.present` + // field form -- so a required operand is legal here, and the result + // is always a real Bool because the operator inspects resolution + // rather than a value. + requireArity(1, 0); + for(std::size_t i = 0; i < args.size(); ++i) + { + const nlohmann::json& a = args.at(i); + if(!a.is_string() || a.get_ref().size() < 2 + || a.get_ref().front() != '$') + { + throw UmdCompileError("operator '" + op + + "' requires variable-reference arguments (A.7)"); + } + argType(i); // the reference must resolve (A.10 §8) + } + return {Kind::BOOL, Kind::UNKNOWN}; + } + if(op == "if") + { + return inferIf(args); + } + throw UmdCompileError("unrecognized operator: " + op); + } + + TypeInfo inferIf(const std::vector& args) + { + if(args.size() < 3 || args.size() % 2 == 0) + { + throw UmdCompileError("'if' requires 3 or 2n+1 arguments (A.7)"); + } + TypeInfo result; + bool haveResult = false; + for(std::size_t i = 0; i + 1 < args.size(); i += 2) + { + const Kind cond = inferTypeImpl(args.at(i)).kind; + if(cond != Kind::BOOL && cond != Kind::DYNAMIC) + { + throw UmdCompileError("'if' condition must be Bool (A.7)"); + } + const TypeInfo branch = inferTypeImpl(args.at(i + 1)); + result = unify(result, branch, haveResult); + haveResult = true; + } + if(args.size() % 2 == 1) + { + result = unify(result, inferTypeImpl(args.back()), haveResult); + } + return result; + } + + static TypeInfo unify(const TypeInfo& acc, const TypeInfo& next, bool haveAcc) + { + if(!haveAcc) + { + return next; + } + if(!sameDomain(acc.kind, next.kind)) + { + throw UmdCompileError("'if' branch results must share a type (A.7)"); + } + return acc; + } + + // Resolve the static type of a `$`-stripped variable path against the + // registry and shape maps; throws if the reference does not resolve + // (A.10 §8). + static TypeInfo varTypeOf(const std::string& varPath, const CompiledUmd& out) + { + std::string root; + std::string rest; + path::splitRoot(varPath, root, rest); + + if(root == "graph") + { + if(rest == "node_count") + { + return {Kind::INT, Kind::UNKNOWN}; + } + if(rest == "is_override_shape_enabled") + { + return {Kind::BOOL, Kind::UNKNOWN}; + } + throw UmdCompileError("unknown graph field: " + rest); + } + if(root == "device") + { + if(rest.empty()) + { + throw UmdCompileError("device reference needs a field"); + } + return {Kind::INT, Kind::UNKNOWN}; // device scalars are numeric + } + if(root == "kernel") + { + if(rest.empty()) + { + throw UmdCompileError("kernel reference needs a field"); + } + // Kernel field types come from a UKD's KMD, unavailable at + // UMD-compile time; resolve their domain at match time (DYNAMIC). + // Field existence is validated against the engine's KMD schema at + // load, so an undeclared field is rejected before match runs. + return {Kind::DYNAMIC, Kind::UNKNOWN}; + } + const NodeSpec* node = out.findNode(root); + if(node != nullptr) + { + return attrType(rest, *node); + } + const TensorVarSpec* spec = out.findTvar(root); + if(spec != nullptr) + { + return tensorFieldType(rest, *spec); + } + throw UmdCompileError("unresolved reference (A.10 §8): $" + varPath); + } + static TypeInfo attrType(const std::string& rest, const NodeSpec& node) + { + if(rest.empty()) + { + // A bare node id is not a value reference (A.4 attr-ref needs a field). + throw UmdCompileError("node id $" + node.id + " is not a value reference (A.4)"); + } + std::string attr = rest; + const bool present = path::stripPresentSuffix(attr); + for(std::size_t i = 0; i < node.opSchema->attributeCount; ++i) + { + if(node.opSchema->attributes[i].name == attr) + { + if(present) + { + if(!node.opSchema->attributes[i].optional) + { + throw UmdCompileError(".present on a required attribute is refused (A.4): " + + attr); + } + return {Kind::BOOL, Kind::UNKNOWN}; + } + switch(node.opSchema->attributes[i].type) + { + case reg::AttrType::INT: + return {Kind::INT, Kind::UNKNOWN}; + case reg::AttrType::FLOAT: + return {Kind::FLOAT, Kind::UNKNOWN}; + case reg::AttrType::BOOL: + return {Kind::BOOL, Kind::UNKNOWN}; + case reg::AttrType::DTYPE: + return {Kind::DTYPE, Kind::UNKNOWN}; + default: + break; + } + } + } + throw UmdCompileError("unknown attribute for " + node.opcode + ": " + attr); + } + + static TypeInfo tensorFieldType(const std::string& rest, const TensorVarSpec& spec) + { + if(rest == "present") + { + if(!spec.optional) + { + throw UmdCompileError(".present on a required operand is refused (A.4): " + + spec.tvar); + } + return {Kind::BOOL, Kind::UNKNOWN}; + } + if(rest.empty() || rest == "uid") + { + return rest.empty() ? TypeInfo{Kind::TENSOR, Kind::UNKNOWN} + : TypeInfo{Kind::INT, Kind::UNKNOWN}; + } + if(rest == "rank") + { + return {Kind::INT, Kind::UNKNOWN}; + } + if(rest == "dtype") + { + return {Kind::DTYPE, Kind::UNKNOWN}; + } + if(rest == "stride_order") + { + return {Kind::ARRAY, Kind::INT}; + } + if(rest == "packed" || rest == "virtual" || rest == "is_runtime_pass_by_value") + { + return {Kind::BOOL, Kind::UNKNOWN}; + } + if(rest == "value_f32") + { + // The schema layer coerces whichever arm of the tensor's `value` + // union is set to f32 and publishes it as one typed token; it reads + // null when the tensor carries no compile-time value (RFC 0017 §5). + return {Kind::FLOAT, Kind::UNKNOWN}; + } + for(const std::string_view prefix : {std::string_view("dims"), std::string_view("strides")}) + { + if(path::isSubscriptOf(rest, prefix)) + { + std::size_t idx = 0; + if(!path::parseSubscript(rest, prefix, idx)) + { + throw UmdCompileError("malformed subscript $" + spec.tvar + "." + rest + + " (A.4 requires a non-negative integer index)"); + } + return {Kind::INT, Kind::UNKNOWN}; + } + } + if(spec.dimNames.find(rest) != spec.dimNames.end()) + { + return {Kind::INT, Kind::UNKNOWN}; + } + throw UmdCompileError("unknown tensor field $" + spec.tvar + "." + rest); + } + + const CompiledUmd* _ctx = nullptr; +}; + +} // namespace hip_kernel_provider_common::umd diff --git a/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/UmdPathParse.hpp b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/UmdPathParse.hpp new file mode 100644 index 000000000000..d9a5439bc079 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/UmdPathParse.hpp @@ -0,0 +1,103 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +// UmdPathParse.hpp - shared parsing of a sigil-stripped UMD variable path +// (RFC 0018 A.4), used by both the runtime resolver (BindingContext) and the +// compile-time resolver (UmdCompiler). The two MUST agree on how a path splits +// into namespace root and field, so the logic lives in one place rather than +// being duplicated where it could drift. + +#include +#include +#include + +namespace hip_kernel_provider_common::umd::path +{ + +// Split a path into its first segment (`root`) and the remainder (`rest`). A +// leading '[' subscript stays in `rest` so a bare-tensor path like `q` and a +// subscripted `q.dims[0]` both split on the first '.'/'['. +inline void splitRoot(const std::string& path, std::string& root, std::string& rest) +{ + const std::size_t p = path.find_first_of(".["); + if(p == std::string::npos) + { + root = path; + rest.clear(); + } + else if(path[p] == '.') + { + root = path.substr(0, p); + rest = path.substr(p + 1); + } + else + { + root = path.substr(0, p); + rest = path.substr(p); // keep '[' for the subscript + } +} + +// Parse `[N]` -> N where N is a non-negative decimal integer. Returns +// false when `rest` is not exactly that form (wrong prefix, missing/misplaced +// brackets, empty, non-numeric, negative, or trailing characters). +inline bool parseSubscript(const std::string& rest, std::string_view prefix, std::size_t& idx) +{ + const std::size_t plen = prefix.size(); + if(rest.size() < plen + 2 || rest.compare(0, plen, prefix) != 0 || rest[plen] != '[') + { + return false; + } + const std::size_t close = rest.find(']', plen + 1); + if(close == std::string::npos || close != rest.size() - 1) + { + return false; + } + const std::string num = rest.substr(plen + 1, close - plen - 1); + if(num.empty()) + { + return false; + } + std::size_t consumed = 0; + long value = 0; + try + { + value = std::stol(num, &consumed); + } + catch(const std::exception&) + { + return false; + } + if(consumed != num.size() || value < 0) + { + return false; + } + idx = static_cast(value); + return true; +} + +// If `s` ends with the reserved `.present` suffix, strip it and return true; +// otherwise leave `s` unchanged and return false. +inline bool stripPresentSuffix(std::string& s) +{ + static constexpr std::string_view K_PRESENT = ".present"; + if(s.size() > K_PRESENT.size() + && s.compare(s.size() - K_PRESENT.size(), K_PRESENT.size(), K_PRESENT) == 0) + { + s.resize(s.size() - K_PRESENT.size()); + return true; + } + return false; +} + +// True when `rest` begins with `[`, i.e. it is (well-formed or not) a +// subscript of that prefix. Used to distinguish a subscript attempt from an +// unrelated field before validating it with parseSubscript. +inline bool isSubscriptOf(const std::string& rest, std::string_view prefix) +{ + return rest.size() > prefix.size() && rest.compare(0, prefix.size(), prefix) == 0 + && rest[prefix.size()] == '['; +} + +} // namespace hip_kernel_provider_common::umd::path diff --git a/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/UniversalGraphMatcher.hpp b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/UniversalGraphMatcher.hpp new file mode 100644 index 000000000000..5e6e4455991a --- /dev/null +++ b/dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/umd/UniversalGraphMatcher.hpp @@ -0,0 +1,389 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +// UniversalGraphMatcher.hpp - RFC 0018 (UMD) graph matcher. +// +// All names below live in namespace hip_kernel_provider_common::umd. +// +// A UniversalGraphMatcher owns exactly one CompiledUmd -- there is a 1:1 +// relationship between a matcher and its UMD JSON descriptor -- and matches a +// live flatbuffer graph against it. Matching does double duty (RFC 0018 §4): it +// decides applicability and binds the descriptor's named variables into a +// queryable BindingContext. +// +// match(): +// 1. structural -- locate a distinct graph node per pattern node (by opcode), +// honor the `allow_override_shape` gate, and bind each operand/result name +// to a graph tensor via the Phase 0 generated UID readers, enforcing that +// a shared pattern variable resolves to one tensor across nodes (implicit +// edges); decline on a missing required tensor or an edge conflict; +// 2. criteria -- construct the BindingContext and evaluate the compiled +// criteria expression; a non-true result declines; +// 3. return a MatchResult carrying the queryable bindings and the descriptor id. +// +// Fail closed: any structural failure or a false criterion yields no match; the +// matcher never matches a graph by default (RFC 0018 §14). + +#include "hip_kernel_provider_common/JsonLogic.hpp" +#include "hip_kernel_provider_common/umd/BindingContext.hpp" +#include "hip_kernel_provider_common/umd/UmdCompiler.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hip_kernel_provider_common::umd +{ + +// The device-property scalars a criterion may read via `$device.` +// (RFC 0018 §4). Populated from the runtime device (hipDeviceProp_t +// sharedMemPerBlock / warpSize) and surfaced by name in the BindingContext. +struct DeviceProperties +{ + std::int64_t ldsSize = 0; + std::int64_t warpSize = 0; +}; + +struct MatchResult +{ + bool matched = false; + hipdnn_flatbuffers_sdk::utilities::UuidBytes umdId{}; + BindingContext bindings; // valid when matched; queryable post-match +}; + +class UniversalGraphMatcher +{ +public: + using IGraph = hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph; + + // Compile a descriptor into this matcher. There is a 1:1 relationship + // between a matcher and its UMD; a matcher matches graphs against exactly + // one descriptor. Throws UmdCompileError on an invalid descriptor + // (RFC 0018 A.10). + explicit UniversalGraphMatcher(const nlohmann::json& descriptor) + : _umd(std::make_unique(UmdCompiler::compile(descriptor))) + { + } + + // Adopt an already-compiled descriptor. + explicit UniversalGraphMatcher(CompiledUmd umd) + : _umd(std::make_unique(std::move(umd))) + { + } + + // The id of the descriptor this matcher matches against. + const hipdnn_flatbuffers_sdk::utilities::UuidBytes& umdId() const + { + return _umd->id; + } + + const CompiledUmd& descriptor() const + { + return *_umd; + } + + // True when this matcher's descriptor references `$kernel.`. Such a + // descriptor must be matched with the kernel-metadata overload of match(); + // the two-argument overload throws for it. + bool referencesKernelMetadata() const + { + return !_umd->kernelFields.empty(); + } + + // The `$kernel.` names this matcher reads, without the `kernel.` + // prefix. A KDP/KMD loader checks each name exists in the engine's KMD and + // uses the set as the key that memoizes this matcher per distinct kernel + // field tuple, rather than per kernel (RFC 0017 §5). + const std::set& kernelFields() const + { + return _umd->kernelFields; + } + + // The variable paths the descriptor's criteria bind, so a pack's dispatch + // descriptor can be checked against the fields its matchers publish + // (RFC 0017 §6). + const std::set& boundSymbols() const + { + return _umd->boundSymbols; + } + + // Match a graph against this matcher's single descriptor. + // + // Throws std::logic_error if the descriptor references `$kernel.`: + // resolving those needs kernel metadata, so the caller must use the + // three-argument overload. Use referencesKernelMetadata() to route. + MatchResult match(const DeviceProperties& device, const IGraph& graph) const + { + if(referencesKernelMetadata()) + { + throw std::logic_error( + "UMD '" + hipdnn_flatbuffers_sdk::utilities::formatUuid(_umd->id) + + "' references $kernel metadata; call the kernel-metadata match overload"); + } + return matchImpl(device, graph, nullptr); + } + + // Match a graph against this matcher's single descriptor, resolving + // `$kernel.` references against the supplied kernel metadata (a UKD's + // KMD values). An unresolved kernel field reads null (fail closed). + MatchResult match(const DeviceProperties& device, + const IGraph& graph, + const nlohmann::json& kernelMetadata) const + { + return matchImpl(device, graph, &kernelMetadata); + } + +private: + using INodeWrapper = hipdnn_flatbuffers_sdk::flatbuffer_utilities::INodeWrapper; + + MatchResult matchImpl(const DeviceProperties& device, + const IGraph& graph, + const nlohmann::json* kernel) const + { + // Fail closed on an invalid or nodeless graph. nodeCount() tolerates a + // null `nodes` field; nodeWrappers() would throw on it, so this guard + // also keeps match() exception-free for a graph with no nodes. + if(!graph.isValid() || graph.nodeCount() == 0) + { + return {}; + } + std::unordered_map deviceMap; + deviceMap.emplace("lds_size", jlogic::Value(device.ldsSize)); + deviceMap.emplace("warp_size", jlogic::Value(device.warpSize)); + + return tryMatch(*_umd, graph, graph.nodeWrappers(), deviceMap, kernel); + } + + // The hipDNN graph schema (SDK) version this graph's own contents require, + // as "." (RFC 0017 §4). hipDNN already computes exactly this + // floor from the optional features a graph uses and stamps it into the + // serialized graph as `min_required_engine_api_version` (RFC 0008 + // override shapes at 1.1, RFC 0016 runtime pass-by-value at 1.2), so the + // matcher reads that rather than inventing a second source of truth. An + // unstamped graph (a hand-built fixture, or one written before the field + // existed) reads as the 1.0 baseline, matching the plugin SDK's own + // null-tolerant accessor. + static std::string graphRequiredSdkVersion(const IGraph& graph) + { + const auto* required = graph.getGraph().min_required_engine_api_version(); + if(required == nullptr) + { + return {UmdCompiler::K_DEFAULT_VERSION}; + } + return std::to_string(required->major()) + "." + std::to_string(required->minor()); + } + + MatchResult tryMatch(const CompiledUmd& umd, + const IGraph& graph, + const std::vector>& wrappers, + const std::unordered_map& deviceMap, + const nlohmann::json* kernel) const + { + // Per-graph SDK floor (RFC 0017 §4). A graph reports the schema version + // its own contents require; a matcher declaring less was authored + // before a feature this graph uses existed, so it is skipped rather + // than asked -- it would otherwise match on the fields it knows and + // silently ignore one that changes what the graph means. + if(!UmdCompiler::versionAtLeast(umd.sdkVersion, graphRequiredSdkVersion(graph))) + { + return {}; + } + + // allow_override_shape gate (RFC 0018 §3): decline override-shape graphs + // unless the descriptor opts in. + if(graph.getGraph().is_override_shape_enabled() && !umd.allowOverrideShape) + { + return {}; + } + + // Structural: candidate graph nodes per pattern node (same opcode). + std::vector> candidates(umd.nodes.size()); + for(std::size_t i = 0; i < umd.nodes.size(); ++i) + { + for(const auto& w : wrappers) + { + if(w->attributesType() == umd.nodes[i].opSchema->attributesType + && w->attributes() != nullptr) + { + candidates[i].push_back(w.get()); + } + } + if(candidates[i].empty()) + { + return {}; + } + } + + // Assign each pattern node to a distinct graph node (injective), then + // enforce edges and evaluate criteria. Patterns are tiny, but a hostile + // graph could present many same-opcode candidates; a step budget caps + // the search so an adversarial input declines rather than stalls + // (RFC 0018 §14). Exhausting the budget fails closed. + std::vector assignment(umd.nodes.size(), nullptr); + std::vector used; + std::size_t steps = 0; + MatchResult result; + if(assignNode( + umd, graph, deviceMap, kernel, candidates, 0, assignment, used, steps, result)) + { + return result; + } + return {}; + } + + // Upper bound on backtracking steps (candidate placements attempted). Real + // fusion patterns need a handful; the cap only fires on pathological input. + static constexpr std::size_t K_MAX_SEARCH_STEPS = 100000; + + bool assignNode(const CompiledUmd& umd, + const IGraph& graph, + const std::unordered_map& deviceMap, + const nlohmann::json* kernel, + const std::vector>& candidates, + std::size_t index, + std::vector& assignment, + std::vector& used, + std::size_t& steps, + MatchResult& result) const + { + if(index == umd.nodes.size()) + { + return finalize(umd, graph, deviceMap, kernel, assignment, result); + } + for(const INodeWrapper* cand : candidates[index]) + { + if(++steps > K_MAX_SEARCH_STEPS) + { + return false; // search budget exhausted -> fail closed (RFC 0018 §14) + } + if(std::find(used.begin(), used.end(), cand) != used.end()) + { + continue; // a graph node backs at most one pattern node + } + assignment[index] = cand; + used.push_back(cand); + if(assignNode(umd, + graph, + deviceMap, + kernel, + candidates, + index + 1, + assignment, + used, + steps, + result)) + { + return true; + } + used.pop_back(); + assignment[index] = nullptr; + } + return false; + } + + // Resolve every edge for a full node assignment, enforcing that a shared + // pattern variable resolves to one tensor across all nodes (implicit edges, + // RFC 0018 §3/A.3), then evaluate the criteria over the BindingContext. + static bool finalize(const CompiledUmd& umd, + const IGraph& graph, + const std::unordered_map& deviceMap, + const nlohmann::json* kernel, + const std::vector& assignment, + MatchResult& result) + { + struct VarBinding + { + bool present = false; + std::int64_t uid = 0; + }; + std::unordered_map binds; + + for(std::size_t i = 0; i < umd.nodes.size(); ++i) + { + const void* attrs = assignment[i]->attributes(); + for(const EdgeSlot& slot : umd.nodes[i].edges) + { + // NOLINTNEXTLINE(misc-const-correctness) - reader out-parameter + std::int64_t uid = 0; + const bool present = (*slot.reader)(attrs, uid); + const auto it = binds.find(slot.tvar); + if(!present) + { + if(!slot.optional) + { + return false; // required tensor absent -> decline + } + if(it == binds.end()) + { + binds.emplace(slot.tvar, VarBinding{false, 0}); + } + else if(it->second.present) + { + return false; // absent here but present on another edge + } + continue; + } + if(it == binds.end()) + { + binds.emplace(slot.tvar, VarBinding{true, uid}); + } + else if(!it->second.present || it->second.uid != uid) + { + return false; // edge conflict: same variable, different tensor + } + } + } + + BindingContext ctx(&graph, deviceMap); + if(kernel != nullptr) + { + ctx.bindKernelMetadata(*kernel); + } + for(std::size_t i = 0; i < umd.nodes.size(); ++i) + { + ctx.bindNode(umd.nodes[i].id, umd.nodes[i].opSchema, assignment[i]->attributes()); + } + + const auto& tensorMap = graph.getTensorMap(); + for(const TensorVarSpec& spec : umd.tvars) + { + const auto it = binds.find(spec.tvar); + if(it == binds.end() || !it->second.present) + { + ctx.bindTensor(spec.tvar, nullptr, spec.optional, &spec.dimNames); + continue; + } + const auto t = tensorMap.find(it->second.uid); + if(t == tensorMap.end()) + { + return false; // dangling UID -> decline (malformed graph) + } + ctx.bindTensor(spec.tvar, t->second, spec.optional, &spec.dimNames); + } + + if(!umd.criteria(ctx).truthy()) + { + return false; + } + result.matched = true; + result.umdId = umd.id; + result.bindings = std::move(ctx); + return true; + } + + std::unique_ptr _umd; +}; + +} // namespace hip_kernel_provider_common::umd diff --git a/dnn-providers/hip-kernel-provider/src/CMakeLists.txt b/dnn-providers/hip-kernel-provider/src/CMakeLists.txt index 4eaab18e33ae..87eebc706c0e 100644 --- a/dnn-providers/hip-kernel-provider/src/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/src/CMakeLists.txt @@ -46,6 +46,10 @@ if(ENABLE_HIP_MLOPS_ENGINE) target_compile_definitions(hip_kernel_provider_private PUBLIC HIPDNN_ENGINE_HIP_MLOPS=1) endif() +if(HIPDNN_ENABLE_KERNEL_INGESTOR) + target_compile_definitions(hip_kernel_provider_private PUBLIC HIPDNN_ENABLE_KERNEL_INGESTOR=1) +endif() + # hip_kernel_provider - SHARED library (the actual plugin) add_library(hip_kernel_provider SHARED $) target_link_libraries(hip_kernel_provider PRIVATE hipdnn_data_sdk hipdnn_flatbuffers_sdk hipdnn_plugin_sdk hiprtc::hiprtc hip::host) diff --git a/dnn-providers/hip-kernel-provider/src/core/Container.cpp b/dnn-providers/hip-kernel-provider/src/core/Container.cpp index 50ae143c18cc..d420da0ab748 100644 --- a/dnn-providers/hip-kernel-provider/src/core/Container.cpp +++ b/dnn-providers/hip-kernel-provider/src/core/Container.cpp @@ -20,6 +20,11 @@ #include "engines/asm_sdpa_engine/plans/SdpaFwdPlanBuilder.hpp" #endif +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR +#include "engines/kernel_ingestor_engine/KernelIngestorEngine.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddPack.hpp" +#endif + #include #include #include @@ -64,6 +69,18 @@ const std::vector& Container::getEngineDefinitions( engine->addPlanBuilder(std::make_unique()); return engine; }}, +#endif +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + // The kernel ingestor's engine (RFC 0017). Registered statically here; runtime + // loading of engine descriptors will replace this entry with one built per + // installed UED file. + {kernel_ingestor_engine::pointwiseAddEngineId(), + [](const device::IDevicePropertyProvider& /*devicePropertyProvider*/) + -> std::unique_ptr> { + // Device facts are resolved per call from the handle, not from the + // construction-time provider. + return kernel_ingestor_engine::makePointwiseAddEngine(); + }}, #endif }; @@ -97,6 +114,14 @@ Container::Container() { HIPDNN_PLUGIN_LOG_INFO("Creating Container"); +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + // Every ingestor pack's native matchers, scorers, and dispatch handlers must be + // registered before any descriptor-backed engine below can resolve the symbols its + // UMDs, UHDs, and UDDs name. Safe to call on every Container construction: it + // registers exactly once per process (see SharedContainerManager). + kernel_ingestor_engine::registerNativeIngestorSymbols(); +#endif + _engineManager = std::make_unique>(); diff --git a/dnn-providers/hip-kernel-provider/src/core/Settings.hpp b/dnn-providers/hip-kernel-provider/src/core/Settings.hpp index 4561db434349..87dd5c13f980 100644 --- a/dnn-providers/hip-kernel-provider/src/core/Settings.hpp +++ b/dnn-providers/hip-kernel-provider/src/core/Settings.hpp @@ -7,6 +7,10 @@ #include +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR +#include +#endif + /** * @brief HIP kernel provider plugin-specific execution settings. * @@ -18,4 +22,11 @@ struct Settings /// Accumulator precision for backward SDPA dQ gradient. /// nullopt means no user preference (default: A32). std::optional accumulatorType; + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + /// The kernel ingestor's per-call knob filter (RFC 0017 §4's catalog -> filter -> + /// rank ordering). Populated by initializeExecutionSettings(); getMaxWorkspaceSize() + /// reads it from here since it receives only this Settings object. + hipdnn_plugin_sdk::ingestor::KnobFilter ingestorKnobFilter; +#endif }; diff --git a/dnn-providers/hip-kernel-provider/src/engines/CMakeLists.txt b/dnn-providers/hip-kernel-provider/src/engines/CMakeLists.txt index 959a2a9169f7..4bc2a01cd5fe 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/src/engines/CMakeLists.txt @@ -8,3 +8,7 @@ endif() if(ENABLE_HIP_MLOPS_ENGINE) add_subdirectory(hip_mlops_engine) endif() + +if(HIPDNN_ENABLE_KERNEL_INGESTOR) + add_subdirectory(kernel_ingestor_engine) +endif() diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/CMakeLists.txt b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/CMakeLists.txt new file mode 100644 index 000000000000..2dfb0423e0c3 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/CMakeLists.txt @@ -0,0 +1,20 @@ +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +# Embed the kernel sources this engine's packs compile at plan-build time. +include("${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/KernelEmbedding.cmake") + +add_kernels_for_embedding( + TARGET hip_kernel_provider_impl FILES ${CMAKE_CURRENT_SOURCE_DIR}/kernels/PointwiseAdd.cpp +) + +target_sources(hip_kernel_provider_impl PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/KernelIngestorEngine.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/packs/PointwiseAddDispatchHandler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/packs/PointwiseAddMatchers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/packs/PointwiseAddPack.cpp +) + +# Sources above are entirely inside `#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR`; the +# definition must reach this object library, not only the static library tests link. +target_compile_definitions(hip_kernel_provider_impl PUBLIC HIPDNN_ENABLE_KERNEL_INGESTOR=1) diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/HandleDeviceResolver.hpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/HandleDeviceResolver.hpp new file mode 100644 index 000000000000..169540098655 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/HandleDeviceResolver.hpp @@ -0,0 +1,106 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include + +#include +#include +#include + +#include "core/Handle.hpp" + +namespace hip_kernel_provider::kernel_ingestor_engine +{ + +/** + * @brief Resolves a call's device from the stream its handle carries. + * + * A handle can be rebound between calls, so the device is read per call from the + * handle's stream rather than the calling thread's current device. + * + * Device properties are cached for this resolver's lifetime; the cache is never + * invalidated. Only successful queries are cached. + * + * One instance per process (see KernelIngestorEngine.cpp's deviceResolver()). + */ +class HandleDeviceResolver : public hipdnn_plugin_sdk::ingestor::IDeviceResolver +{ +public: + hipdnn_plugin_sdk::ingestor::DeviceId deviceId(const Handle& handle) const override + { + int deviceId = 0; + + // A null stream means the default stream, which belongs to the current device. + if(handle.getStream() != nullptr) + { + if(hipStreamGetDevice(handle.getStream(), &deviceId) == hipSuccess) + { + return deviceId; + } + } + + if(hipGetDevice(&deviceId) != hipSuccess) + { + // Neither 0 nor a throw: deviceId() runs under isApplicable(), which + // EngineManager walks with no try/catch. Matchers decline on NO_DEVICE. + return hipdnn_plugin_sdk::ingestor::NO_DEVICE; + } + return deviceId; + } + + const hipDeviceProp_t& + deviceProperties(hipdnn_plugin_sdk::ingestor::DeviceId deviceId) const override + { + // Inert values for a call with no resolvable device; MatchContext binds + // properties before any matcher runs, so the caller receives this regardless. + if(deviceId == hipdnn_plugin_sdk::ingestor::NO_DEVICE) + { + static const hipDeviceProp_t s_noDevice{}; + return s_noDevice; + } + + const std::lock_guard lock(_mutex); + + auto it = _properties.find(deviceId); + if(it != _properties.end()) + { + return it->second; + } + + hipDeviceProp_t properties{}; + const auto status = queryDeviceProperties(&properties, deviceId); + if(status != hipSuccess) + { + // Never cached; see the class doc. + throw hipdnn_plugin_sdk::HipdnnPluginException( + HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR, + "hipGetDeviceProperties failed for device " + std::to_string(deviceId) + ": " + + hipGetErrorString(status)); + } + return _properties.emplace(deviceId, properties).first->second; + } + +protected: + /// Seam for tests that need to grow the cache without that many real devices; + /// overriding it lets a test supply successful answers for ids this machine + /// does not have. + virtual hipError_t queryDeviceProperties(hipDeviceProp_t* properties, + hipdnn_plugin_sdk::ingestor::DeviceId deviceId) const + { + return hipGetDeviceProperties(properties, deviceId); + } + +private: + mutable std::mutex _mutex; + mutable std::unordered_map _properties; +}; + +} // namespace hip_kernel_provider::kernel_ingestor_engine + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/KernelIngestorEngine.cpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/KernelIngestorEngine.cpp new file mode 100644 index 000000000000..e128b97e28f2 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/KernelIngestorEngine.cpp @@ -0,0 +1,89 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "engines/kernel_ingestor_engine/KernelIngestorEngine.hpp" + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include + +#include + +#include "engines/hip_mlops_engine/HipMlopsKernelCompiler.hpp" +#include "engines/kernel_ingestor_engine/HandleDeviceResolver.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddDispatchHandler.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddPack.hpp" + +namespace hip_kernel_provider::kernel_ingestor_engine +{ + +namespace +{ + +/** + * @brief The dispatch handler this pack's UDD resolves to. + * + * Process-lifetime: DispatchRegistry stores a non-owning pointer to it, while a + * provider's Container is created and destroyed per handle (see + * SharedContainerManager). The compiler it holds is a static for the same reason. + */ +const PointwiseAddDispatchHandler& dispatchHandler() +{ + static const HipMlopsKernelCompiler s_kernelCompiler; + static const PointwiseAddDispatchHandler s_dispatchHandler(s_kernelCompiler); + return s_dispatchHandler; +} + +/** + * @brief The device resolver every descriptor-backed engine in this provider shares. + * + * Container/process-lifetime, not per-engine: it is a device-property cache with no + * engine-specific state (see HandleDeviceResolver's doc). + */ +const HandleDeviceResolver& deviceResolver() +{ + static const HandleDeviceResolver s_deviceResolver; + return s_deviceResolver; +} + +} // namespace + +/// Runs the body of registerNativeIngestorSymbols() at most once per process; broken +/// out so a test can drive it directly and observe rollback on a forced conflict. +void registerNativeIngestorSymbolsOnce() +{ + registerPointwiseAddMatchers(); + try + { + registerPointwiseAddDispatch(dispatchHandler()); + } + catch(...) + { + unregisterPointwiseAddMatchers(); + throw; + } +} + +void registerNativeIngestorSymbols() +{ + // call_once leaves the flag unset on a throw, so a retry re-runs the body. + static std::once_flag s_registered; + std::call_once(s_registered, registerNativeIngestorSymbolsOnce); +} + +std::unique_ptr> makePointwiseAddEngine() +{ + auto set = buildPointwiseAddDescriptorSet(); + // Moved out of `set` in its own statement, fully sequenced before the move of the + // remainder below. + auto engine = std::move(set.engine); + return std::make_unique>( + std::move(engine), makePointwiseAddStateManager(std::move(set)), deviceResolver()); +} + +} // namespace hip_kernel_provider::kernel_ingestor_engine + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/KernelIngestorEngine.hpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/KernelIngestorEngine.hpp new file mode 100644 index 000000000000..58ca2a189c8c --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/KernelIngestorEngine.hpp @@ -0,0 +1,44 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include + +#include + +#include "core/Context.hpp" +#include "core/Handle.hpp" +#include "core/Settings.hpp" + +namespace hip_kernel_provider::kernel_ingestor_engine +{ + +/** + * @brief Registers every native matcher, scorer, and dispatch handler this provider's + * ingestor packs resolve through, exactly once for the life of the process. + * + * Called from Container's constructor, so a UED loaded from a descriptor file with no + * compile-time link to the symbol's translation unit still finds it. Idempotent. + */ +void registerNativeIngestorSymbols(); + +/// The once_flag-guarded body of registerNativeIngestorSymbols(), callable directly so +/// a test can force and observe a partial-failure/rollback cycle. +void registerNativeIngestorSymbolsOnce(); + +/** + * @brief Builds the descriptor-backed pointwise-add engine. + * + * The device resolver and dispatch handler its kernels need are process-lifetime + * statics (see KernelIngestorEngine.cpp), not members. + * + * Requires registerNativeIngestorSymbols() to have run first. + */ +std::unique_ptr> makePointwiseAddEngine(); + +} // namespace hip_kernel_provider::kernel_ingestor_engine + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/kernels/PointwiseAdd.cpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/kernels/PointwiseAdd.cpp new file mode 100644 index 000000000000..51d0e45d87be --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/kernels/PointwiseAdd.cpp @@ -0,0 +1,20 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +// Skeleton kernel behind the pointwise-add descriptor pack: elementwise add over a +// single-element tensor. +// +// HIP_PLUGIN_POINTWISE_ADD_TYPE is the element type from the kernel descriptor's dtype +// metadata. HIP_PLUGIN_POINTWISE_ADD_BLOCK_SIZE is the descriptor's block size; unused +// by the kernel itself (one element needs one thread), but must reach the compiler for +// ranking and knob reporting. + +extern "C" __global__ void PointwiseAdd(const HIP_PLUGIN_POINTWISE_ADD_TYPE* a, + const HIP_PLUGIN_POINTWISE_ADD_TYPE* b, + HIP_PLUGIN_POINTWISE_ADD_TYPE* c) +{ + if(blockIdx.x == 0 && threadIdx.x == 0) + { + c[0] = a[0] + b[0]; + } +} diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddDispatchHandler.cpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddDispatchHandler.cpp new file mode 100644 index 000000000000..225229a84507 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddDispatchHandler.cpp @@ -0,0 +1,160 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "engines/kernel_ingestor_engine/packs/PointwiseAddDispatchHandler.hpp" + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include + +#include +#include +#include + +#include "compilation/KernelCompileOptions.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddSymbols.hpp" + +namespace hip_kernel_provider::kernel_ingestor_engine +{ + +using namespace hipdnn_plugin_sdk::ingestor; +namespace data_objects = hipdnn_flatbuffers_sdk::data_objects; + +namespace +{ + +/// Scratch reported by the larger-block kernel. Arbitrary: its only job is to make the +/// engine's max-across-survivors answer observably non-zero. +constexpr size_t LARGE_BLOCK_WORKSPACE_BYTES = 1024; +constexpr int64_t LARGE_BLOCK_SIZE = 256; + +/// The compiled kernel plus the operand uids it launches with: everything read from the +/// graph, resolved once, owning nothing that points back into it. +class PreparedPointwiseAdd : public PreparedDispatch +{ +public: + PreparedPointwiseAdd(std::unique_ptr program, + std::unique_ptr kernel, + PointwiseAddBinding binding) + : _program(std::move(program)) + , _kernel(std::move(kernel)) + , _binding(binding) + { + } + + const compilation::IRunnableKernel& kernel() const + { + return *_kernel; + } + + const PointwiseAddBinding& binding() const + { + return _binding; + } + +private: + // The runnable kernel is a view into its program's module, so the program must + // outlive it; both are held here for the plan's lifetime. + std::unique_ptr _program; + std::unique_ptr _kernel; + PointwiseAddBinding _binding; +}; + +/// The C++ type the kernel is compiled for, from the kernel's dtype metadata. +std::string elementTypeFor(const KernelDefinition& kernel) +{ + const auto& dtype = kernel.getStringMetadata(std::string(DTYPE_FIELD)); + if(dtype == "FLOAT") + { + return "float"; + } + if(dtype == "HALF") + { + return "_Float16"; + } + + // Unreachable via matching, which admits only dtypes this pack declares. + throw hipdnn_plugin_sdk::HipdnnPluginException( + HIPDNN_PLUGIN_STATUS_BAD_PARAM, + "kernel '" + toString(kernel.kernelId) + "' declares unsupported dtype '" + dtype + "'"); +} + +const data_objects::TensorAttributes& firstInput(const MatchContext& context, + const PointwiseAddBinding& binding) +{ + const auto& tensors = context.graph.getTensorMap(); + auto it = tensors.find(binding.inputA); + if(it == tensors.end() || it->second == nullptr) + { + throw hipdnn_plugin_sdk::HipdnnPluginException( + HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR, + "matched pointwise add has no tensor for uid " + std::to_string(binding.inputA)); + } + return *it->second; +} + +} // namespace + +size_t PointwiseAddDispatchHandler::workspaceBytes(const MatchContext& /*context*/, + const BoundTokens& /*bound*/, + const KernelDefinition& kernel) const +{ + return kernel.getIntMetadata(std::string(BLOCK_SIZE_FIELD)) == LARGE_BLOCK_SIZE + ? LARGE_BLOCK_WORKSPACE_BYTES + : 0; +} + +std::unique_ptr PointwiseAddDispatchHandler::prepare( + const MatchContext& context, const BoundTokens& bound, const KernelDefinition& kernel) const +{ + // Reads the operand uids the matcher bound rather than re-deriving them. + const auto binding = pointwiseAddBinding(bound); + + const auto blockSize + = static_cast(kernel.getIntMetadata(std::string(BLOCK_SIZE_FIELD))); + + compilation::KernelCompileOptions options(&firstInput(context, binding), + context.deviceProperties); + options.add("HIP_PLUGIN_POINTWISE_ADD_TYPE", elementTypeFor(kernel)); + options.add("HIP_PLUGIN_POINTWISE_ADD_BLOCK_SIZE", blockSize); + + // The only KernelSourceKind this dispatch handler knows how to load. + auto program = _kernelCompiler.compile(kernel.source.sourceFile, options); + auto runnableKernel = program->getKernel(kernel.source.entryPoint); + + // One element, so one workgroup; block size comes from kernel metadata. + runnableKernel->setBlockSize(blockSize, 1, 1); + runnableKernel->setGridSize(1, 1, 1); + + return std::make_unique( + std::move(program), std::move(runnableKernel), binding); +} + +void PointwiseAddDispatchHandler::launch(const Handle& handle, + const PreparedDispatch& prepared, + const hipdnnPluginDeviceBuffer_t* deviceBuffers, + uint32_t numDeviceBuffers, + void* /*workspace*/) const +{ + const auto& preparedAdd = dynamic_cast(prepared); + const auto& binding = preparedAdd.binding(); + + const auto inputA + = hipdnn_plugin_sdk::findDeviceBuffer(binding.inputA, deviceBuffers, numDeviceBuffers); + const auto inputB + = hipdnn_plugin_sdk::findDeviceBuffer(binding.inputB, deviceBuffers, numDeviceBuffers); + const auto output + = hipdnn_plugin_sdk::findDeviceBuffer(binding.output, deviceBuffers, numDeviceBuffers); + + preparedAdd.kernel().launch(handle.getStream(), inputA.ptr, inputB.ptr, output.ptr); +} + +void registerPointwiseAddDispatch(const PointwiseAddDispatchHandler& handler) +{ + DispatchRegistry::registerSymbol(std::string(DISPATCH_SYMBOL), &handler); +} + +} // namespace hip_kernel_provider::kernel_ingestor_engine + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddDispatchHandler.hpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddDispatchHandler.hpp new file mode 100644 index 000000000000..5ee95aad905b --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddDispatchHandler.hpp @@ -0,0 +1,77 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include + +#include +#include + +#include "compilation/IKernelCompiler.hpp" +#include "core/Handle.hpp" + +namespace hip_kernel_provider::kernel_ingestor_engine +{ + +/** + * @brief The native dispatch behind this pack's UDD: sizes and launches a pointwise add. + * + * Holds the kernel compiler a launch needs. + * + * Splits per RFC 0017 §8.5: everything derived from the graph and chosen kernel + * resolves once at plan build; execute only resolves device pointers by uid and + * launches. A plan may execute concurrently from several threads, so nothing here + * mutates after preparation. + */ +class PointwiseAddDispatchHandler + : public hipdnn_plugin_sdk::ingestor::IKernelDispatchHandler +{ +public: + /// @param kernelCompiler Owned by the engine, which outlives this handler. + /// + /// Device properties are not held; they arrive per call on the MatchContext, so a + /// kernel is compiled for the device the call is actually for. + explicit PointwiseAddDispatchHandler(const compilation::IKernelCompiler& kernelCompiler) + : _kernelCompiler(kernelCompiler) + { + } + + /** + * @brief Scratch this kernel requires. + * + * A one-element add needs none. The 256-block kernel reports a non-zero + * requirement so the engine's max-across-survivors is observably a maximum. + */ + size_t + workspaceBytes(const hipdnn_plugin_sdk::ingestor::MatchContext& context, + const hipdnn_plugin_sdk::ingestor::BoundTokens& bound, + const hipdnn_plugin_sdk::ingestor::KernelDefinition& kernel) const override; + + std::unique_ptr + prepare(const hipdnn_plugin_sdk::ingestor::MatchContext& context, + const hipdnn_plugin_sdk::ingestor::BoundTokens& bound, + const hipdnn_plugin_sdk::ingestor::KernelDefinition& kernel) const override; + + void launch(const Handle& handle, + const hipdnn_plugin_sdk::ingestor::PreparedDispatch& prepared, + const hipdnnPluginDeviceBuffer_t* deviceBuffers, + uint32_t numDeviceBuffers, + void* workspace) const override; + +private: + const compilation::IKernelCompiler& _kernelCompiler; +}; + +/// @brief Registers @p handler under this pack's dispatch symbol. +/// +/// The handler is registered by pointer and must outlive every plan built from it. +void registerPointwiseAddDispatch(const PointwiseAddDispatchHandler& handler); + +} // namespace hip_kernel_provider::kernel_ingestor_engine + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.cpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.cpp new file mode 100644 index 000000000000..c56c4713009a --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.cpp @@ -0,0 +1,259 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.hpp" + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "core/Utils.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddSymbols.hpp" + +namespace hip_kernel_provider::kernel_ingestor_engine +{ + +using namespace hipdnn_plugin_sdk::ingestor; +namespace data_objects = hipdnn_flatbuffers_sdk::data_objects; + +namespace +{ + +const data_objects::TensorAttributes* findTensor(const MatchContext& context, int64_t uid) +{ + const auto& tensors = context.graph.getTensorMap(); + auto it = tensors.find(uid); + return it == tensors.end() ? nullptr : it->second; +} + +/// The tensor ranks this pack accepts; compile options derive layout from the tensor +/// and reject anything outside this range. +constexpr uint32_t MIN_SUPPORTED_RANK = 4; +constexpr uint32_t MAX_SUPPORTED_RANK = 5; + +/// True when the tensor's stride order is channel-first or channel-last, the only +/// orders the dispatch path's compile options can classify. +bool hasSupportedLayout(const data_objects::TensorAttributes& tensor) +{ + try + { + static_cast(core::utils::isChannelLastLayout(&tensor)); + return true; + } + catch(const hipdnn_plugin_sdk::HipdnnPluginException&) + { + return false; + } +} + +/// True when the tensor is a supported rank and layout holding exactly one element. +bool isSingleElement(const data_objects::TensorAttributes& tensor) +{ + const auto* dims = tensor.dims(); + if(dims == nullptr || dims->size() < MIN_SUPPORTED_RANK || dims->size() > MAX_SUPPORTED_RANK) + { + return false; + } + + int64_t elements = 1; + for(const auto dim : *dims) + { + elements *= dim; + } + if(elements != 1) + { + return false; + } + + return hasSupportedLayout(tensor); +} + +/// The graph's element type, from the first input; the matcher below requires every +/// operand to agree, so any of them would answer the same. +std::optional graphDataType(const MatchContext& context) +{ + if(context.graph.nodeCount() != 1) + { + return std::nullopt; + } + + const auto& node = context.graph.getNodeWrapper(0); + if(node.attributesType() != data_objects::NodeAttributes::PointwiseAttributes) + { + return std::nullopt; + } + + const auto& attributes = node.attributesAs(); + const auto* input = findTensor(context, attributes.in_0_tensor_uid()); + if(input == nullptr) + { + return std::nullopt; + } + return input->data_type(); +} + +std::string dataTypeName(data_objects::DataType dataType) +{ + return data_objects::EnumNameDataType(dataType); +} + +} // namespace + +bool pointwiseAddGraphMatches(const MatchContext& context, BoundTokens& bound) +{ + // No device, no launch: every fact this matcher would select on -- including the + // device properties the compile is configured from -- is meaningless without one. + if(context.deviceId == hipdnn_plugin_sdk::ingestor::NO_DEVICE) + { + return false; + } + + // Exactly one node: this pack's kernel serves one complete graph. + if(context.graph.nodeCount() != 1) + { + return false; + } + + const auto& node = context.graph.getNodeWrapper(0); + if(node.attributesType() != data_objects::NodeAttributes::PointwiseAttributes) + { + return false; + } + + const auto& attributes = node.attributesAs(); + if(attributes.operation() != data_objects::PointwiseMode::ADD) + { + return false; + } + + // Binary add: a second operand is required, a third would be a different operation. + if(!attributes.in_1_tensor_uid().has_value() || attributes.in_2_tensor_uid().has_value()) + { + return false; + } + + const auto* inputA = findTensor(context, attributes.in_0_tensor_uid()); + const auto* inputB = findTensor(context, attributes.in_1_tensor_uid().value()); + const auto* output = findTensor(context, attributes.out_0_tensor_uid()); + if(inputA == nullptr || inputB == nullptr || output == nullptr) + { + return false; + } + + // One element each: this pack's kernel indexes element 0 and nothing else. + if(!isSingleElement(*inputA) || !isSingleElement(*inputB) || !isSingleElement(*output)) + { + return false; + } + + // A virtual tensor has no device buffer for findDeviceBuffer to resolve at launch. + if(inputA->virtual_() || inputB->virtual_() || output->virtual_()) + { + return false; + } + + // A 1-element rank-4 tensor is also the shape of a pass-by-value scalar, whose + // variant-pack slot holds a host pointer, not a device one. + if(hipdnn_flatbuffers_sdk::utilities::isPassByValueTensor(inputA) + || hipdnn_flatbuffers_sdk::utilities::isPassByValueTensor(inputB) + || hipdnn_flatbuffers_sdk::utilities::isPassByValueTensor(output)) + { + return false; + } + + // Uniform dtype across operands; mixed precision is a different kernel. + if(inputA->data_type() != inputB->data_type() || inputA->data_type() != output->data_type()) + { + return false; + } + + // Binds operand uids for the dispatch handler to read back rather than re-deriving + // them from the graph. + bound[std::string(INPUT_A_TOKEN)] = attributes.in_0_tensor_uid(); + bound[std::string(INPUT_B_TOKEN)] = attributes.in_1_tensor_uid().value(); + bound[std::string(OUTPUT_TOKEN)] = attributes.out_0_tensor_uid(); + return true; +} + +bool pointwiseAddKernelMatches(const MatchContext& context, const KernelDefinition& kernel) +{ + const auto dataType = graphDataType(context); + if(!dataType.has_value()) + { + return false; + } + + // Pins the kernel's baked dtype against the graph's, so an f32 graph cannot reach + // an f16 kernel and get wrong numbers back. + return kernel.getStringMetadata(std::string(DTYPE_FIELD)) == dataTypeName(*dataType); +} + +double pointwiseAddScore(const KernelDefinition& kernel, const MatchContext& /*context*/) +{ + // A stand-in for a trained model: prefers the larger block size. + return static_cast(kernel.getIntMetadata(std::string(BLOCK_SIZE_FIELD))); +} + +PointwiseAddBinding pointwiseAddBinding(const BoundTokens& bound) +{ + // Every token was written by the graph matcher that admitted this graph; a missing + // one means the catalog was built by a matcher other than ours. + const auto read = [&bound](std::string_view token) { + const auto value = hipdnn_plugin_sdk::ingestor::tryGetBoundInt(bound, token); + if(!value.has_value()) + { + throw hipdnn_plugin_sdk::HipdnnPluginException( + HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR, + "pointwise add dispatch is missing bound token '" + std::string(token) + + "', or it does not hold a tensor uid"); + } + return *value; + }; + + return {read(INPUT_A_TOKEN), read(INPUT_B_TOKEN), read(OUTPUT_TOKEN)}; +} + +void registerPointwiseAddMatchers() +{ + GraphMatcherRegistry::registerSymbol(std::string(GRAPH_MATCHER_SYMBOL), + &pointwiseAddGraphMatches); + try + { + KernelMatcherRegistry::registerSymbol(std::string(KERNEL_MATCHER_SYMBOL), + &pointwiseAddKernelMatches); + } + catch(...) + { + GraphMatcherRegistry::unregisterSymbol(std::string(GRAPH_MATCHER_SYMBOL)); + throw; + } + try + { + ScoreRegistry::registerSymbol(std::string(SCORE_SYMBOL), &pointwiseAddScore); + } + catch(...) + { + GraphMatcherRegistry::unregisterSymbol(std::string(GRAPH_MATCHER_SYMBOL)); + KernelMatcherRegistry::unregisterSymbol(std::string(KERNEL_MATCHER_SYMBOL)); + throw; + } +} + +void unregisterPointwiseAddMatchers() +{ + GraphMatcherRegistry::unregisterSymbol(std::string(GRAPH_MATCHER_SYMBOL)); + KernelMatcherRegistry::unregisterSymbol(std::string(KERNEL_MATCHER_SYMBOL)); + ScoreRegistry::unregisterSymbol(std::string(SCORE_SYMBOL)); +} + +} // namespace hip_kernel_provider::kernel_ingestor_engine + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.hpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.hpp new file mode 100644 index 000000000000..26e03766edfc --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.hpp @@ -0,0 +1,63 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include + +namespace hip_kernel_provider::kernel_ingestor_engine +{ + +/** + * @brief Graph-scoped applicability: is this a single-node pointwise ADD over + * 1-element tensors? + * + * Evaluated once per (graph, device); a failure disqualifies every kernel in the pack. + */ +bool pointwiseAddGraphMatches(const hipdnn_plugin_sdk::ingestor::MatchContext& context, + hipdnn_plugin_sdk::ingestor::BoundTokens& bound); + +/** + * @brief Kernel-scoped applicability: does this kernel's dtype match the graph's? + * + * Evaluated once per candidate kernel. Without it, an f32 graph could reach an f16 + * binary and return wrong numbers rather than failing. + */ +bool pointwiseAddKernelMatches(const hipdnn_plugin_sdk::ingestor::MatchContext& context, + const hipdnn_plugin_sdk::ingestor::KernelDefinition& kernel); + +/// @brief Ranks kernels for a pointwise add. +double pointwiseAddScore(const hipdnn_plugin_sdk::ingestor::KernelDefinition& kernel, + const hipdnn_plugin_sdk::ingestor::MatchContext& context); + +/// @brief The tensor uids a matched pointwise-add graph binds, in argument order. +struct PointwiseAddBinding +{ + int64_t inputA = 0; + int64_t inputB = 0; + int64_t output = 0; +}; + +/** + * @brief Re-reads the operand bindings a match established. + * + * @throws HipdnnPluginException if the graph is not one this matcher accepts. + */ +PointwiseAddBinding pointwiseAddBinding(const hipdnn_plugin_sdk::ingestor::BoundTokens& bound); + +/// @brief Registers this pack's matchers and scorer under their symbol names. +/// +/// Called once, from the pack's translation unit. Atomic: if one symbol is already +/// taken, any symbols this call already installed are unregistered before it rethrows. +void registerPointwiseAddMatchers(); + +/// @brief Undoes registerPointwiseAddMatchers(), for a caller rolling this pack back +/// after a later registration step of its own fails. +void unregisterPointwiseAddMatchers(); + +} // namespace hip_kernel_provider::kernel_ingestor_engine + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddPack.cpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddPack.cpp new file mode 100644 index 000000000000..40061be8de0f --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddPack.cpp @@ -0,0 +1,136 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "engines/kernel_ingestor_engine/packs/PointwiseAddPack.hpp" + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include + +#include +#include +#include +#include + +#include "engines/kernel_ingestor_engine/packs/PointwiseAddSymbols.hpp" + +namespace hip_kernel_provider::kernel_ingestor_engine +{ + +using namespace hipdnn_plugin_sdk::ingestor; +using hipdnn_flatbuffers_sdk::utilities::parseUuid; + +namespace +{ + +// Descriptor ids are stable GUIDs, minted once for this pack and never regenerated. +// Authored descriptors will carry the same values as text, parsed with parseUuid(). +const DescriptorId ENGINE_ID = parseUuid("8f1a6c30-7d24-4a0e-9b51-2c6d84f0a911"); +const DescriptorId SCHEMA_ID = parseUuid("b27e4d15-3f89-4c62-8a07-51de93b4c7a2"); +const DescriptorId HEURISTIC_ID = parseUuid("4c9b0e72-16a5-4d38-b6f4-8e20a7c15d63"); +const DescriptorId GRAPH_MATCHER_ID = parseUuid("e5d38a04-9c71-4b2f-83a6-7f14b0d29e58"); +const DescriptorId KERNEL_MATCHER_ID = parseUuid("1a7f52c8-b036-4e91-95d2-6c83af407b14"); +const DescriptorId DISPATCH_ID = parseUuid("93b6c1e7-2058-4fa3-8d17-4b95e2c60fa8"); +const DescriptorId PACK_ID = parseUuid("7d024fb9-5e13-4c86-a92b-08f37d1c4e50"); + +KernelDescriptor makeKernel(const DescriptorId& id, + const std::string& variant, + int64_t blockSize, + const std::string& dtype, + int64_t priority) +{ + KernelDescriptor kernel; + kernel.id = id; + kernel.name = "pointwise_add." + variant; + // The only KernelSourceKind this pack implements; see KernelSource's doc for the others. + kernel.source.sourceFile = "PointwiseAdd.cpp"; + kernel.source.entryPoint = "PointwiseAdd"; + kernel.metadata = {{std::string(BLOCK_SIZE_FIELD), MetadataValue{blockSize}}, + {std::string(DTYPE_FIELD), MetadataValue{dtype}}}; + kernel.priority = priority; + return kernel; +} + +} // namespace + +PointwiseAddDescriptorSet buildPointwiseAddDescriptorSet() +{ + PointwiseAddDescriptorSet set; + + set.schema.id = SCHEMA_ID; + set.schema.name = "pointwise add variant fields"; + // block_size defaults; dtype does not, since an omitted one would silently inherit + // another kernel's. + set.schema.fields = {{std::string(BLOCK_SIZE_FIELD), MetadataType::INT, int64_t{64}}, + {std::string(DTYPE_FIELD), MetadataType::STRING, std::nullopt}}; + + set.heuristic.id = HEURISTIC_ID; + set.heuristic.name = "pointwise add selector"; + set.heuristic.payload = SCORE_SYMBOL; + + set.engine.id = ENGINE_ID; + set.engine.name = ENGINE_NAME; + set.engine.heuristicId = HEURISTIC_ID; + set.engine.metadataSchemaId = SCHEMA_ID; + // block_size is exposed to the caller; dtype is pinned by the graph, not chosen. + set.engine.knobs = {std::string(BLOCK_SIZE_FIELD)}; + // This engine's dispatch handler compiles through hiprtc at plan build; a pack + // shipping prebuilt kernels would not declare this. + set.engine.behaviorNotes = {HIPDNN_BEHAVIOR_NOTE_RUNTIME_COMPILATION}; + + set.matchers = { + {GRAPH_MATCHER_ID, + "single-node pointwise add over 1-element tensors", + MatchScope::GRAPH, + std::string(GRAPH_MATCHER_SYMBOL)}, + {KERNEL_MATCHER_ID, + "kernel dtype matches the graph's dtype", + MatchScope::KERNEL, + std::string(KERNEL_MATCHER_SYMBOL)}, + }; + + set.dispatches = { + {DISPATCH_ID, "pointwise add dispatch", std::string(DISPATCH_SYMBOL)}, + }; + + KernelDescriptorPack pack; + pack.id = PACK_ID; + pack.name = "hipkernel:pointwise_add"; + pack.matcherIds = {GRAPH_MATCHER_ID, KERNEL_MATCHER_ID}; + pack.engineId = ENGINE_ID; + pack.dispatchId = DISPATCH_ID; + // The two FLOAT entries differ only in block size, giving ranking an order; the + // HALF entry is what the kernel-scoped matcher prunes on a FLOAT graph. + pack.kernels + = {makeKernel( + parseUuid("2f8c17d6-4a90-4b53-9e18-c05a63b782d4"), "f32_block64", 64, "FLOAT", 0), + makeKernel( + parseUuid("a41b93e5-6d72-4f08-b3c9-15e847d0629f"), "f32_block256", 256, "FLOAT", 0), + makeKernel( + parseUuid("c6e20a48-8b31-4d97-a052-9f4173e8b5c1"), "f16_block64", 64, "HALF", 0)}; + set.packs = {std::move(pack)}; + + return set; +} + +int64_t pointwiseAddEngineId() +{ + // Registers the engine name against its id on first call; see the header for why + // this is a function-local static. + static const hipdnn_data_sdk::utilities::EngineRegistrar s_registrar{ENGINE_NAME}; + return hipdnn_data_sdk::utilities::engineNameToId(ENGINE_NAME); +} + +std::unique_ptr> + makePointwiseAddStateManager(PointwiseAddDescriptorSet set) +{ + return std::make_unique>(std::move(set.schema), + std::move(set.matchers), + std::move(set.dispatches), + std::move(set.packs), + makeKernelHeuristic(set.heuristic)); +} + +} // namespace hip_kernel_provider::kernel_ingestor_engine + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddPack.hpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddPack.hpp new file mode 100644 index 000000000000..e3563010e824 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddPack.hpp @@ -0,0 +1,66 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include + +#include +#include + +#include "core/Handle.hpp" + +namespace hip_kernel_provider::kernel_ingestor_engine +{ + +/** + * @file PointwiseAddPack.hpp + * @brief The pointwise-add descriptor set, built in memory. + * + * Stands in for what a loader will produce from installed files: one engine (UED), + * its metadata schema (KMD) and heuristic (UHD), two matchers (UMDs), one dispatch + * descriptor (UDD), and one pack (KDP) binding them over three kernels (UKDs). + */ + +/// A pack plus every descriptor it references by id, ready to construct a state manager. +struct PointwiseAddDescriptorSet +{ + hipdnn_plugin_sdk::ingestor::EngineDescriptor engine; + hipdnn_plugin_sdk::ingestor::MetadataSchema schema; + std::vector matchers; + std::vector dispatches; + std::vector packs; + hipdnn_plugin_sdk::ingestor::HeuristicDescriptor heuristic; +}; + +/// @brief Builds this pack's descriptor set. +PointwiseAddDescriptorSet buildPointwiseAddDescriptorSet(); + +/** + * @brief This engine's hipDNN engine id, registering its name on first call. + * + * The id is the UED name hashed into hipDNN's engine-id space, registered at run time + * since the engine is defined by data. A function-local static, not a namespace-scope + * object: the registrar throws on a collision, which from a global constructor during + * dlopen() would terminate the process. + */ +int64_t pointwiseAddEngineId(); + +/** + * @brief Builds the state manager backing this pack, from an already-built + * descriptor set. + * + * Takes @p set by value so a caller building both the engine and its state manager + * calls buildPointwiseAddDescriptorSet() once. @p set's UED is ignored; the engine + * owns it. + */ +std::unique_ptr> + makePointwiseAddStateManager(PointwiseAddDescriptorSet set); + +} // namespace hip_kernel_provider::kernel_ingestor_engine + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddSymbols.hpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddSymbols.hpp new file mode 100644 index 000000000000..b5d97730bc81 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/PointwiseAddSymbols.hpp @@ -0,0 +1,57 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include + +namespace hip_kernel_provider::kernel_ingestor_engine +{ + +/** + * @file PointwiseAddSymbols.hpp + * @brief The symbol names this pack's descriptors resolve through. + * + * Each constant is a placeholder for a declarative descriptor field once the + * follow-up RFCs land. + */ + +/// Graph-scoped applicability: a single-node pointwise ADD over 1-element tensors. +inline constexpr std::string_view GRAPH_MATCHER_SYMBOL = "hipkernel.pointwise_add.graph_match"; + +/// Kernel-scoped applicability: the kernel's dtype must be the graph's dtype. +inline constexpr std::string_view KERNEL_MATCHER_SYMBOL = "hipkernel.pointwise_add.kernel_match"; + +/// Kernel-selection score. +inline constexpr std::string_view SCORE_SYMBOL = "hipkernel.pointwise_add.score"; + +/// Workspace sizing and launch. +inline constexpr std::string_view DISPATCH_SYMBOL = "hipkernel.pointwise_add.dispatch"; + +/** + * @brief The engine name, hashed into hipDNN's engine-id space. + * + * Prefixed per RFC 0017's global-uniqueness rule. Absent from EngineNames.hpp's + * registry: registered at construction, not at build time. + */ +inline constexpr std::string_view ENGINE_NAME = "hipkernel:PointwiseAdd"; + +/// The KMD fields this engine's kernels vary along. +inline constexpr std::string_view BLOCK_SIZE_FIELD = "block_size"; +inline constexpr std::string_view DTYPE_FIELD = "dtype"; + +/** + * @brief Tokens the graph matcher binds for the dispatch handler to read. + * + * Native stand-in for the `$`-prefixed names a descriptor's dispatch formulas + * would reference (RFC 0017 §5). + */ +inline constexpr std::string_view INPUT_A_TOKEN = "pointwise_add.input_a.uid"; +inline constexpr std::string_view INPUT_B_TOKEN = "pointwise_add.input_b.uid"; +inline constexpr std::string_view OUTPUT_TOKEN = "pointwise_add.output.uid"; + +} // namespace hip_kernel_provider::kernel_ingestor_engine + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/integration_tests/CMakeLists.txt b/dnn-providers/hip-kernel-provider/src/integration_tests/CMakeLists.txt index 39bb59890104..3044c117c9a8 100644 --- a/dnn-providers/hip-kernel-provider/src/integration_tests/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/src/integration_tests/CMakeLists.txt @@ -49,9 +49,13 @@ if(ENABLE_ASM_SDPA_ENGINE) ${CMAKE_CURRENT_SOURCE_DIR}/../tests/engines/asm_sdpa_engine/AsmSdpaConfigHelpers.cpp asm_sdpa_engine/IntegrationGpuSdpaForward.cpp ) - # Add include path for codegen-generated headers (e.g., asm_fmha_v3_fwd_configs.hpp) - # and the shared test helpers (AsmSdpaConfigHelpers.hpp) + # AsmSdpaConfigHelpers.cpp above is shared with hip_kernel_provider_tests and + # spells its provider includes from the src root, which that target inherits from + # hip_kernel_provider_private. This target links only the SDKs, so it has to name + # the root itself. The other two entries are the codegen-generated headers (e.g. + # asm_fmha_v3_fwd_configs.hpp) and the shared test helpers (AsmSdpaConfigHelpers.hpp). target_include_directories(hip_kernel_provider_integration_tests PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR}/../engines/asm_sdpa_engine/generated ${CMAKE_CURRENT_SOURCE_DIR}/../tests/engines/asm_sdpa_engine ) @@ -70,6 +74,18 @@ if(ENABLE_HIP_MLOPS_ENGINE) hip_mlops_engine/IntegrationGpuPassByValue.cpp ) endif() + +if(HIPDNN_ENABLE_KERNEL_INGESTOR) + target_sources(hip_kernel_provider_integration_tests PRIVATE + kernel_ingestor_engine/IntegrationGpuKernelIngestor.cpp + ) + # This target links only the SDKs, not hip_kernel_provider_private, so it does not + # inherit the provider's gate and needs the definition set directly. + target_compile_definitions(hip_kernel_provider_integration_tests + PRIVATE HIPDNN_ENABLE_KERNEL_INGESTOR=1 + ) +endif() + clang_tidy_check(hip_kernel_provider_integration_tests) # ENVIRONMENT is forwarded to add_integration_test_target() rather than set diff --git a/dnn-providers/hip-kernel-provider/src/integration_tests/IntegrationGraphVerificationHarness.hpp b/dnn-providers/hip-kernel-provider/src/integration_tests/IntegrationGraphVerificationHarness.hpp index 638b34ce9477..5b8fff1cbabe 100644 --- a/dnn-providers/hip-kernel-provider/src/integration_tests/IntegrationGraphVerificationHarness.hpp +++ b/dnn-providers/hip-kernel-provider/src/integration_tests/IntegrationGraphVerificationHarness.hpp @@ -164,6 +164,12 @@ class IntegrationGraphVerificationHarness : public ::testing::TestWithParam _tensorIdToNameMap; std::unordered_map> _tensorIdToValidatorMap; diff --git a/dnn-providers/hip-kernel-provider/src/integration_tests/kernel_ingestor_engine/IntegrationGpuKernelIngestor.cpp b/dnn-providers/hip-kernel-provider/src/integration_tests/kernel_ingestor_engine/IntegrationGpuKernelIngestor.cpp new file mode 100644 index 000000000000..95295f91b1a7 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/integration_tests/kernel_ingestor_engine/IntegrationGpuKernelIngestor.cpp @@ -0,0 +1,372 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../IntegrationGraphVerificationHarness.hpp" + +using namespace hipdnn_frontend; +using namespace hipdnn_frontend::graph; +using namespace hipdnn_data_sdk::utilities; +using namespace hipdnn_test_sdk::utilities; +using namespace hip_kernel_provider::test_utilities; + +/** + * @file IntegrationGpuKernelIngestor.cpp + * @brief The kernel ingestor pack, end to end through the hipDNN frontend API. + * + * Drives the same path a real caller takes -- Graph::pointwise(), + * get_ranked_engine_ids(), get_knobs_for_engine(), get_workspace_size(), execute() -- + * so the descriptor set, the matchers, the heuristic, and the dispatch handler are + * exercised as they compose in production. + * + * Exception: `hipdnnEnginePluginGetAllEngineIds` enumerates every engine a plugin + * exports independent of any graph, and has no frontend equivalent, so that one + * assertion crosses the raw C ABI via a minimal dlopen helper. + */ +namespace hip_kernel_provider::kernel_ingestor_engine::integration +{ + +namespace +{ + +constexpr const char* ENGINE_NAME = "hipkernel:PointwiseAdd"; +constexpr const char* BLOCK_SIZE_KNOB = "block_size"; + +/// Maximum workspace across the pack's surviving kernels for a FLOAT graph. +constexpr int64_t EXPECTED_WORKSPACE_BYTES = 1024; + +std::shared_ptr makeScalarTensor(int64_t uid, const std::string& name) +{ + auto tensor = std::make_shared(); + tensor->set_uid(uid) + .set_name(name) + .set_dim({1, 1, 1, 1}) + .set_stride({1, 1, 1, 1}) + .set_data_type(DataType::FLOAT); + return tensor; +} + +/// A single pointwise-ADD node over 1-element FLOAT tensors: the one shape this pack +/// accepts. Each call returns a fresh graph, never shared build/plan state. +std::shared_ptr buildPointwiseAddGraph() +{ + auto graph = std::make_shared(); + graph->set_name("pointwise_add") + .set_io_data_type(DataType::FLOAT) + .set_intermediate_data_type(DataType::FLOAT) + .set_compute_data_type(DataType::FLOAT); + + auto a = makeScalarTensor(1, "A"); + auto b = makeScalarTensor(2, "B"); + + PointwiseAttributes attrs; + attrs.set_name("pointwise_add").set_mode(PointwiseMode::ADD); + auto c = graph->pointwise(a, b, attrs); + c->set_uid(3).set_name("C").set_output(true).set_data_type(DataType::FLOAT); + + return graph; +} + +/// A graph this pack must decline: two nodes, so no single prebuilt kernel serves it. +std::shared_ptr buildUnsupportedGraph() +{ + auto graph = std::make_shared(); + graph->set_name("two_node_pointwise") + .set_io_data_type(DataType::FLOAT) + .set_intermediate_data_type(DataType::FLOAT) + .set_compute_data_type(DataType::FLOAT); + + auto a = makeScalarTensor(1, "A"); + auto b = makeScalarTensor(2, "B"); + + PointwiseAttributes attrs1; + attrs1.set_name("add_1").set_mode(PointwiseMode::ADD); + auto intermediate = graph->pointwise(a, b, attrs1); + intermediate->set_uid(4).set_name("Intermediate").set_data_type(DataType::FLOAT); + + PointwiseAttributes attrs2; + attrs2.set_name("add_2").set_mode(PointwiseMode::ADD); + auto c = graph->pointwise(intermediate, b, attrs2); + c->set_uid(3).set_name("C").set_output(true).set_data_type(DataType::FLOAT); + + return graph; +} + +/// Execute shapes: a single call, and several reusing the same built plan. +struct ExecuteCase +{ + std::string name; + int iterations; +}; + +} // namespace + +class IntegrationGpuKernelIngestor + : public hip_kernel_provider::test_utilities::IntegrationGraphVerificationHarness +{ +protected: + static int64_t engineId() + { + return hipdnn_data_sdk::utilities::engineNameToId(ENGINE_NAME); + } + + /// Builds `graph`, pins this pack via set_preferred_engine_id_ext() before plan + /// creation, and compiles a plan with default knobs. + void buildAndCompile(Graph& graph) + { + graph.set_preferred_engine_id_ext(engineId()); + + auto result = graph.build_operation_graph(_handle); + ASSERT_EQ(result.code, ErrorCode::OK) << result.err_msg; + + result = graph.create_execution_plans(); + ASSERT_EQ(result.code, ErrorCode::OK) << result.err_msg; + + result = graph.check_support(); + ASSERT_EQ(result.code, ErrorCode::OK) << result.err_msg; + + result = graph.build_plans(); + ASSERT_EQ(result.code, ErrorCode::OK) << result.err_msg; + } + + /// Builds fresh CPU/GPU tensor bundles for `graph`, executes it once on GPU with + /// `workspace`, and verifies against CpuReferenceGraphExecutor. `seed` varies the + /// input values so repeated calls never compare against stale buffers. + void executeAndVerify(Graph& graph, void* workspace, unsigned int seed) + { + GraphTensorBundle gpuBundle; + GraphTensorBundle cpuBundle; + graph.visit([&](const INode& node) { + for(const auto& tensorAttr : node.getNodeOutputTensorAttributes()) + { + gpuBundle.addTensor(*tensorAttr, createTensorFromAttribute(*tensorAttr)); + cpuBundle.addTensor(*tensorAttr, createTensorFromAttribute(*tensorAttr)); + } + for(const auto& tensorAttr : node.getNodeInputTensorAttributes()) + { + if(gpuBundle.tensors.find(tensorAttr->get_uid()) == gpuBundle.tensors.end()) + { + gpuBundle.addTensor(*tensorAttr, createTensorFromAttribute(*tensorAttr)); + cpuBundle.addTensor(*tensorAttr, createTensorFromAttribute(*tensorAttr)); + } + } + }); + for(auto& [uid, tensor] : gpuBundle.tensors) + { + // Offset per uid: randomizeTensor seeds a fresh mt19937, so a shared seed + // makes every operand byte-identical and allClose() would pass on a+a as + // readily as a+b, no longer witnessing uid-based argument resolution. + const auto tensorSeed = seed + static_cast(uid); + gpuBundle.randomizeTensor(uid, -4.0f, 4.0f, tensorSeed); + cpuBundle.randomizeTensor(uid, -4.0f, 4.0f, tensorSeed); + } + + auto deviceVariantPack = gpuBundle.toDeviceVariantPack(); + auto result = graph.execute(_handle, deviceVariantPack, workspace); + ASSERT_EQ(result.code, ErrorCode::OK) << result.err_msg; + ASSERT_EQ(hipStreamSynchronize(_stream), hipSuccess); + + auto [serializedGraph, serErr] = graph.to_binary(); + ASSERT_TRUE(serErr.is_good()) << serErr.get_message(); + CpuReferenceGraphExecutor().execute( + serializedGraph.data(), serializedGraph.size(), cpuBundle.toHostVariantPack()); + + auto& gpuOut = gpuBundle.getTensor(3); + auto& cpuOut = cpuBundle.getTensor(3); + gpuOut.markDeviceModified(); + // Proves the full chain: matcher admission, heuristic ranking, handler compile + // and launch, and argument resolution by tensor uid. + EXPECT_TRUE(CpuFpReferenceValidation().allClose(cpuOut, gpuOut)); + } +}; + +// --------------------------------------------------------------------------- +// Direct ABI: load-time self-registration +// --------------------------------------------------------------------------- + +// The frontend never lists "every engine this plugin exports", so proving load-time +// self-registration needs the raw C ABI. +TEST(IntegrationGpuKernelIngestorDirectAbi, SelfRegistersAllEngineIds) +{ + const std::filesystem::path pluginTarget(PLUGIN_PATH); + const auto pluginFile = hipdnn_data_sdk::utilities::LIB_PREFIX + + pluginTarget.filename().string() + + hipdnn_data_sdk::utilities::SHARED_LIB_EXT; + const auto pluginPath = std::filesystem::weakly_canonical( + hipdnn_data_sdk::utilities::getCurrentExecutableDirectory() / pluginTarget.parent_path() + / pluginFile); + auto* library = hipdnn_data_sdk::utilities::openLibrary(pluginPath); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + auto* getAllEngineIds = reinterpret_cast( + hipdnn_data_sdk::utilities::getSymbol(library, "hipdnnEnginePluginGetAllEngineIds")); + ASSERT_NE(getAllEngineIds, nullptr); + + uint32_t count = 0; + ASSERT_EQ(getAllEngineIds(nullptr, 0, &count), HIPDNN_PLUGIN_STATUS_SUCCESS); + std::vector engines(count); + ASSERT_EQ(getAllEngineIds(engines.data(), count, &count), HIPDNN_PLUGIN_STATUS_SUCCESS); + + EXPECT_NE(std::find(engines.begin(), engines.end(), engineNameToId(ENGINE_NAME)), + engines.end()); +} + +// --------------------------------------------------------------------------- +// Applicability +// --------------------------------------------------------------------------- + +TEST_F(IntegrationGpuKernelIngestor, AcceptsTheGraphItsDescriptorsDescribe) +{ + auto graph = buildPointwiseAddGraph(); + + auto result = graph->build_operation_graph(_handle); + ASSERT_EQ(result.code, ErrorCode::OK) << result.err_msg; + + std::vector rankedEngineIds; + result = graph->get_ranked_engine_ids(rankedEngineIds); + ASSERT_EQ(result.code, ErrorCode::OK) << result.err_msg; + + EXPECT_NE(std::find(rankedEngineIds.begin(), rankedEngineIds.end(), engineId()), + rankedEngineIds.end()); +} + +TEST_F(IntegrationGpuKernelIngestor, DeclinesATwoNodeGraph) +{ + auto graph = buildUnsupportedGraph(); + + auto result = graph->build_operation_graph(_handle); + ASSERT_EQ(result.code, ErrorCode::OK) << result.err_msg; + + std::vector rankedEngineIds; + result = graph->get_ranked_engine_ids(rankedEngineIds); + EXPECT_EQ(result.code, ErrorCode::GRAPH_NOT_SUPPORTED); + EXPECT_TRUE(rankedEngineIds.empty()); +} + +// --------------------------------------------------------------------------- +// Engine details: knobs from the catalog +// --------------------------------------------------------------------------- + +TEST_F(IntegrationGpuKernelIngestor, ReportsAKnobWhoseValuesComeFromTheCatalog) +{ + auto graph = buildPointwiseAddGraph(); + + auto result = graph->build_operation_graph(_handle); + ASSERT_EQ(result.code, ErrorCode::OK) << result.err_msg; + + std::vector knobs; + result = graph->get_knobs_for_engine(engineId(), knobs); + ASSERT_EQ(result.code, ErrorCode::OK) << result.err_msg; + ASSERT_EQ(knobs.size(), 1U); + EXPECT_EQ(knobs[0].knobId(), BLOCK_SIZE_KNOB); + + // The pack ships three kernels; the HALF one is pruned for this FLOAT graph, so the + // knob offers exactly the two block sizes the surviving kernels implement. + const auto* constraint = dynamic_cast(knobs[0].constraint()); + ASSERT_NE(constraint, nullptr); + const auto& validValues = constraint->getValidValues(); + EXPECT_EQ(validValues, (std::unordered_set{64, 256})); + + // The default is whatever the heuristic ranked first. + const auto* defaultValue = std::get_if(&knobs[0].defaultValue()); + ASSERT_NE(defaultValue, nullptr); + EXPECT_EQ(*defaultValue, 256); +} + +// --------------------------------------------------------------------------- +// Workspace +// --------------------------------------------------------------------------- + +TEST_F(IntegrationGpuKernelIngestor, ReportsTheMaximumWorkspaceAcrossSurvivingKernels) +{ + auto graph = buildPointwiseAddGraph(); + buildAndCompile(*graph); + + // One surviving kernel declares 0 bytes and the other 1024, so this proves the + // query aggregates across the catalog rather than reporting one kernel's value. + int64_t workspaceSize = 0; + auto result = graph->get_workspace_size(workspaceSize); + ASSERT_EQ(result.code, ErrorCode::OK) << result.err_msg; + EXPECT_EQ(workspaceSize, EXPECTED_WORKSPACE_BYTES); +} + +// --------------------------------------------------------------------------- +// Plan build and execute +// --------------------------------------------------------------------------- + +// Every decision the plan makes happens at build, so execution must never depend on +// the previous call. +TEST_P(IntegrationGpuKernelIngestor, ExecutesTheSelectedKernelOnDevice) +{ + const auto& testCase = GetParam(); + + auto graph = buildPointwiseAddGraph(); + buildAndCompile(*graph); + + int64_t workspaceSize = 0; + ASSERT_EQ(graph->get_workspace_size(workspaceSize).code, ErrorCode::OK); + const hipdnn_data_sdk::utilities::Workspace workspace(static_cast(workspaceSize)); + + for(int iteration = 0; iteration < testCase.iterations; ++iteration) + { + executeAndVerify(*graph, workspace.get(), static_cast(iteration)); + } +} + +// GraphDescriptor::finalize() always synthesizes a fresh UUID, so every graph built +// through the frontend is a guaranteed catalog miss relative to every other one. Two +// independently built graphs must both match the reference executor, proving the +// recompute path is correct rather than relying on state left by a prior graph. +TEST_F(IntegrationGpuKernelIngestor, ExecutesTwoIndependentlyBuiltGraphsCorrectly) +{ + auto graphA = buildPointwiseAddGraph(); + buildAndCompile(*graphA); + int64_t workspaceSizeA = 0; + ASSERT_EQ(graphA->get_workspace_size(workspaceSizeA).code, ErrorCode::OK); + const hipdnn_data_sdk::utilities::Workspace workspaceA(static_cast(workspaceSizeA)); + executeAndVerify(*graphA, workspaceA.get(), 0); + + auto graphB = buildPointwiseAddGraph(); + buildAndCompile(*graphB); + int64_t workspaceSizeB = 0; + ASSERT_EQ(graphB->get_workspace_size(workspaceSizeB).code, ErrorCode::OK); + const hipdnn_data_sdk::utilities::Workspace workspaceB(static_cast(workspaceSizeB)); + executeAndVerify(*graphB, workspaceB.get(), 1); +} + +INSTANTIATE_TEST_SUITE_P(, + IntegrationGpuKernelIngestor, + ::testing::Values(ExecuteCase{"SingleExecute", 1}, + ExecuteCase{"RepeatedExecute", 3}), + [](const ::testing::TestParamInfo& info) { + return info.param.name; + }); + +} // namespace hip_kernel_provider::kernel_ingestor_engine::integration + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/tests/CMakeLists.txt b/dnn-providers/hip-kernel-provider/src/tests/CMakeLists.txt index 8015e97ef56a..c60e135c5288 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/src/tests/CMakeLists.txt @@ -13,6 +13,12 @@ endif() if(NOT TARGET hipdnn_plugin_sdk) find_package(hipdnn_plugin_sdk CONFIG REQUIRED) endif() +# TestJsonLogic.cpp expresses rules as nlohmann::json. Available transitively via +# the hipdnn SDKs when HIPDNN_SKIP_JSON_LIB=OFF; link explicitly so the suite also +# builds standalone and with HIPDNN_SKIP_JSON_LIB=ON. +if(NOT TARGET nlohmann_json::nlohmann_json) + find_package(nlohmann_json CONFIG REQUIRED) +endif() add_executable( hip_kernel_provider_tests @@ -22,6 +28,9 @@ add_executable( core/TestContext.cpp core/TestUtils.cpp core/TestPluginPublic.cpp + core/TestJsonLogic.cpp + core/TestJsonDataSource.cpp + core/TestUniversalGraphMatcher.cpp compilation/TestProgramAndKernel.cpp compilation/TestKernelCompileOptions.cpp compilation/TestModuleCache.cpp @@ -30,6 +39,21 @@ add_executable( # Add the dummy kernel for running the tests include("${CMAKE_CURRENT_SOURCE_DIR}/../cmake/KernelEmbedding.cmake") add_kernels_for_embedding(TARGET hip_kernel_provider_tests FILES ${CMAKE_CURRENT_SOURCE_DIR}/kernels/vector_add.cpp) + +# Test binary needs its own copy (embedding is per-target). Copied via configure_file: +# registering the same source for embedding in two directories doubles its +# CMAKE_CONFIGURE_DEPENDS entry, which Ninja rejects as two rules for one file. +if(HIPDNN_ENABLE_KERNEL_INGESTOR) + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/../engines/kernel_ingestor_engine/kernels/PointwiseAdd.cpp + ${CMAKE_CURRENT_BINARY_DIR}/kernels/PointwiseAdd.cpp + COPYONLY + ) + add_kernels_for_embedding( + TARGET hip_kernel_provider_tests + FILES ${CMAKE_CURRENT_BINARY_DIR}/kernels/PointwiseAdd.cpp + ) +endif() embed_kernel_sources( TARGET hip_kernel_provider_tests OUTPUT_SRCS_CPP ${CMAKE_CURRENT_BINARY_DIR}/engines/kernel_sources.cpp @@ -49,7 +73,7 @@ target_compile_options(hip_kernel_provider_tests PRIVATE ${HIPDNN_WARNING_COMPIL target_link_libraries( hip_kernel_provider_tests PUBLIC GTest::gtest GTest::gmock hip::host - hip_kernel_provider_private hipdnn_test_sdk hipdnn_plugin_sdk Threads::Threads + hip_kernel_provider_private hipdnn_test_sdk hipdnn_plugin_sdk nlohmann_json::nlohmann_json Threads::Threads ) # make it so tests inside folders can find files without having to use a relative path. @@ -69,6 +93,10 @@ if(ENABLE_HIP_MLOPS_ENGINE) add_subdirectory(engines/hip_mlops_engine) endif() +if(HIPDNN_ENABLE_KERNEL_INGESTOR) + add_subdirectory(engines/kernel_ingestor_engine) +endif() + clang_tidy_check(hip_kernel_provider_tests) add_unit_test_target(hip_kernel_provider_tests ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/dnn-providers/hip-kernel-provider/src/tests/core/TestContainer.cpp b/dnn-providers/hip-kernel-provider/src/tests/core/TestContainer.cpp index 7b76c00738c9..15d4983e7f94 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/core/TestContainer.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/core/TestContainer.cpp @@ -1,6 +1,7 @@ // Copyright © Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT +#include #include #include @@ -14,6 +15,11 @@ #include #include +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR +#include "engines/kernel_ingestor_engine/packs/PointwiseAddPack.hpp" +#include "tests/engines/kernel_ingestor_engine/packs/PointwiseAddTestGraphs.hpp" +#endif + using namespace hip_kernel_provider; using namespace hip_kernel_provider::core; @@ -27,6 +33,10 @@ constexpr uint32_t EXPECTED_ENGINES = 0 + 1 #endif +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + + 1 +#endif + // Add more blocks like this as more engines are implemented ; @@ -106,6 +116,30 @@ TEST(TestContainer, GetApplicableEngineIdsSdpaGraph) #endif } +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR +TEST(TestContainer, GetApplicableEngineIdsPointwiseAddGraph) +{ + // Applicability is device-resolved: with no device, matchers decline. + SKIP_IF_NO_DEVICES(); + + using namespace hip_kernel_provider::kernel_ingestor_engine; + using namespace hip_kernel_provider::kernel_ingestor_engine::testing; + + Handle handle; + Container container; + auto& engineManager = container.getEngineManager(); + + const auto graph = buildPointwiseGraph(); + const auto graphWrapper = hipdnn_flatbuffers_sdk::flatbuffer_utilities::GraphWrapper( + graph.GetBufferPointer(), graph.GetSize()); + + auto applicableEngines = engineManager.getApplicableEngineIds(handle, graphWrapper); + + EXPECT_NE(std::find(applicableEngines.begin(), applicableEngines.end(), pointwiseAddEngineId()), + applicableEngines.end()); +} +#endif + TEST(TestContainer, GetAllEngineIds) { Container container; diff --git a/dnn-providers/hip-kernel-provider/src/tests/core/TestJsonDataSource.cpp b/dnn-providers/hip-kernel-provider/src/tests/core/TestJsonDataSource.cpp new file mode 100644 index 000000000000..b6d2595835c8 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/tests/core/TestJsonDataSource.cpp @@ -0,0 +1,109 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include + +#include + +#include + +#include "hip_kernel_provider_common/JsonDataSource.hpp" +#include "hip_kernel_provider_common/JsonLogic.hpp" + +namespace jlogic = hip_kernel_provider_common::jsonlogic; + +using json = nlohmann::json; +using V = jlogic::Value; + +// --------------------------------------------------------------------------- +// JsonDataSource: the sample nlohmann::json-backed data source (getData/setData). +// --------------------------------------------------------------------------- +TEST(TestJsonDataSource, GetResolvesPathsAndSubscripts) +{ + const jlogic::JsonDataSource src{json{{"q", {{"dims", {8, 16}}}}, + {"rows", {{{"name", "a0"}}, {{"name", "a1"}}}}, + {"grid", {{1, 2}, {3, 4}}}}}; + EXPECT_EQ(src.getData("q.dims"), V(V::Array{V(8), V(16)})); + EXPECT_EQ(src.getData("q.dims[1]"), V(16)); + EXPECT_EQ(src.getData("q.dims.0"), V(8)); // dot-form index against an array + EXPECT_EQ(src.getData("rows[1].name"), V("a1")); + EXPECT_EQ(src.getData("grid[0][1]"), V(2)); + EXPECT_EQ(src.getData(""), V()); // whole document is an object -> null + // unresolved paths read as null + EXPECT_EQ(src.getData("q.nope"), V()); + EXPECT_EQ(src.getData("q.dims[9]"), V()); + EXPECT_EQ(src.getData("q.dims[x]"), V()); + EXPECT_EQ(src.getData("q.dims["), V()); // malformed subscript +} + +TEST(TestJsonDataSource, GetStripsOptionalSigil) +{ + const jlogic::JsonDataSource src{json{{"q", {{"dims", {8, 16}}}}}}; + EXPECT_EQ(src.getData("$q.dims[0]"), src.getData("q.dims[0]")); + EXPECT_EQ(src.getData("$q.dims[0]"), V(8)); +} + +TEST(TestJsonDataSource, SetScalarCreatesNestedObjects) +{ + jlogic::JsonDataSource src; + src.setData("q.dims", V(4)); + EXPECT_EQ(src.getData("q.dims"), V(4)); + EXPECT_EQ(src.document(), (json{{"q", {{"dims", 4}}}})); +} + +TEST(TestJsonDataSource, SetSubscriptCreatesAndExtendsArray) +{ + jlogic::JsonDataSource src; + // [N] forces array creation; gaps fill with null. + src.setData("q.dims[2]", V(7)); + EXPECT_EQ(src.document(), (json{{"q", {{"dims", {nullptr, nullptr, 7}}}}})); + EXPECT_EQ(src.getData("q.dims[2]"), V(7)); + EXPECT_EQ(src.getData("q.dims[0]"), V()); // gap reads as null +} + +TEST(TestJsonDataSource, SetOverwritesExistingArrayIndex) +{ + jlogic::JsonDataSource src{json{{"q", {{"dims", {8, 16}}}}}}; + src.setData("$q.dims[0]", V(2)); // the motivating example + EXPECT_EQ(src.document(), (json{{"q", {{"dims", {2, 16}}}}})); + EXPECT_EQ(src.getData("q.dims[0]"), V(2)); +} + +TEST(TestJsonDataSource, SetWholeDocument) +{ + jlogic::JsonDataSource src{json{{"x", 1}}}; + src.setData("", V(V::Array{V(1), V(2)})); + EXPECT_EQ(src.document(), (json::array({1, 2}))); + EXPECT_EQ(src.getData("[1]"), V(2)); +} + +TEST(TestJsonDataSource, SetRoundTripsValueKinds) +{ + jlogic::JsonDataSource src; + src.setData("b", V(true)); + src.setData("s", V("amd")); + src.setData("d", V(1.5)); + src.setData("a", V(V::Array{V(1), V("two"), V(false)})); + EXPECT_EQ(src.getData("b"), V(true)); + EXPECT_EQ(src.getData("s"), V("amd")); + EXPECT_EQ(src.getData("d"), V(1.5)); + EXPECT_EQ(src.getData("a"), V(V::Array{V(1), V("two"), V(false)})); +} + +TEST(TestJsonDataSource, SetThenEvaluateReflectsChange) +{ + jlogic::JsonDataSource src{json{{"q", {{"dims", {8, 16}}}}}}; + const auto expr = jlogic::compile( + json({{"*", json::array({"$q.dims[0]", "$q.dims[1]"})}})); + EXPECT_EQ(expr(src), V(128)); + src.setData("$q.dims[0]", V(2)); + EXPECT_EQ(expr(src), V(32)); +} + +TEST(TestJsonDataSource, SetRejectsMalformedPaths) +{ + jlogic::JsonDataSource src{json{{"q", {{"dims", {8, 16}}}}}}; + EXPECT_THROW(src.setData("q.dims[0", V(1)), std::invalid_argument); // missing ']' + EXPECT_THROW(src.setData("q.dims[x]", V(1)), std::invalid_argument); // non-numeric index + EXPECT_THROW(src.setData("q.dims.nope", V(1)), std::invalid_argument); // string key on array +} diff --git a/dnn-providers/hip-kernel-provider/src/tests/core/TestJsonLogic.cpp b/dnn-providers/hip-kernel-provider/src/tests/core/TestJsonLogic.cpp new file mode 100644 index 000000000000..3bcd748fce2f --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/tests/core/TestJsonLogic.cpp @@ -0,0 +1,399 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hip_kernel_provider_common/JsonDataSource.hpp" +#include "hip_kernel_provider_common/JsonLogic.hpp" + +namespace jlogic = hip_kernel_provider_common::jsonlogic; + +using json = nlohmann::json; +using V = jlogic::Value; + +namespace +{ +const jlogic::JsonDataSource D{json{{"x", 41}, + {"y", 8}, + {"name", "amd"}, + {"nested", {{"a", {{"b", 7}}}}}, + {"arr", {10, 20, 30}}, + {"grid", {{1, 2}, {3, 4}}}, + {"rows", {{{"name", "a0"}}, {{"name", "a1"}}}}, + {"flag", true}, + {"zero", 0}}}; + +// Compile a rule and evaluate it against the shared document D. +V eval(const json& rule) +{ + return jlogic::compile(rule)(D); +} +} // namespace + +TEST(TestJsonLogic, Literals) +{ + EXPECT_EQ(eval(42), V(42)); + EXPECT_EQ(eval("hello"), V("hello")); + EXPECT_EQ(eval(true), V(true)); + EXPECT_EQ(eval(json::array({1, 2, 3})), V(V::Array{V(1), V(2), V(3)})); +} + +TEST(TestJsonLogic, VarStockForm) +{ + EXPECT_EQ(eval(json({{"var", "x"}})), V(41)); + EXPECT_EQ(eval(json({{"var", "nested.a.b"}})), V(7)); + EXPECT_EQ(eval(json({{"var", "arr.1"}})), V(20)); + EXPECT_EQ(eval(json({{"var", "nope"}})), V()); + EXPECT_EQ(eval(json({{"var", json::array({"nope", 99})}})), V(99)); + EXPECT_EQ(eval(json({{"var", json::array({"x", 99})}})), V(41)); + EXPECT_EQ(eval(json({{"var", "arr"}})), V(V::Array{V(10), V(20), V(30)})); +} + +TEST(TestJsonLogic, InlineVariables) +{ + EXPECT_EQ(eval("$x"), V(41)); + EXPECT_EQ(eval("$nested.a.b"), V(7)); + EXPECT_EQ(eval("$arr.2"), V(30)); + EXPECT_EQ(eval("$nope"), V()); + EXPECT_EQ(eval("$$literal"), V("$literal")); + EXPECT_EQ(eval(json({{"+", json::array({"$x", "$y"})}})), V(49)); + EXPECT_EQ(eval(json({{"+", json::array({"$x", 1})}})), V(42)); + EXPECT_EQ(eval(json({{"==", json::array({"amd", "$name"})}})), V(true)); +} + +TEST(TestJsonLogic, SubscriptIndex) +{ + // [N] subscript, stock and inline forms + EXPECT_EQ(eval(json({{"var", "arr[0]"}})), V(10)); + EXPECT_EQ(eval("$arr[2]"), V(30)); + // subscript and dotted keys mixed + EXPECT_EQ(eval("$rows[1].name"), V("a1")); + // chained subscripts into a nested array + EXPECT_EQ(eval("$grid[0][1]"), V(2)); + EXPECT_EQ(eval("$grid[1][0]"), V(3)); + // dot-form array index still resolves + EXPECT_EQ(eval("$arr.1"), V(20)); + // out-of-range, non-numeric, and subscript-on-non-array resolve to null + EXPECT_EQ(eval("$arr[9]"), V()); + EXPECT_EQ(eval("$arr[x]"), V()); + EXPECT_EQ(eval("$nested[0]"), V()); +} + +TEST(TestJsonLogic, Arithmetic) +{ + EXPECT_EQ(eval(json({{"+", json::array({1, 2, 3})}})), V(6)); + EXPECT_EQ(eval(json({{"+", json::array({"2", "3"})}})), V(5)); + EXPECT_EQ(eval(json({{"-", json::array({10, 3})}})), V(7)); + EXPECT_EQ(eval(json({{"-", json::array({5})}})), V(-5)); + EXPECT_EQ(eval(json({{"*", json::array({2, 3, 4})}})), V(24)); + EXPECT_EQ(eval(json({{"/", json::array({12, 4})}})), V(3)); + EXPECT_EQ(eval(json({{"%", json::array({10, 3})}})), V(1)); + EXPECT_EQ(eval(json({{"min", json::array({3, 1, 2})}})), V(1)); + EXPECT_EQ(eval(json({{"max", json::array({"$x", "$y", 100})}})), V(100)); + EXPECT_EQ(eval(json({{"+", json::array({{{"var", "x"}}, 1})}})), V(42)); +} + +TEST(TestJsonLogic, ComparisonIsStrict) +{ + EXPECT_EQ(eval(json({{"==", json::array({1, 1})}})), V(true)); + EXPECT_EQ(eval(json({{"==", json::array({1, 1.0})}})), V(true)); + EXPECT_EQ(eval(json({{"==", json::array({1, "1"})}})), V(false)); // no coercion + EXPECT_EQ(eval(json({{"==", json::array({"amd", "amd"})}})), V(true)); + EXPECT_EQ(eval(json({{"!=", json::array({1, 2})}})), V(true)); + EXPECT_EQ(eval(json({{"!=", json::array({1, "1"})}})), V(true)); // no coercion + EXPECT_EQ(eval(json({{">", json::array({"$x", "$y"})}})), V(true)); + EXPECT_EQ(eval(json({{">=", json::array({5, 5})}})), V(true)); + EXPECT_EQ(eval(json({{"<", json::array({1, 2})}})), V(true)); + EXPECT_EQ(eval(json({{"<=", json::array({2, 2})}})), V(true)); + EXPECT_EQ(eval(json({{"<", json::array({1, 2, 3})}})), V(true)); + EXPECT_EQ(eval(json({{"<", json::array({1, 5, 3})}})), V(false)); + EXPECT_EQ(eval(json({{"<=", json::array({1, 1, 3})}})), V(true)); + EXPECT_EQ(eval(json({{"<", json::array({"abc", "abd"})}})), V(true)); +} + +TEST(TestJsonLogic, TruthinessAndLogic) +{ + EXPECT_EQ(eval(json({{"!", 0}})), V(true)); + EXPECT_EQ(eval(json({{"!", 1}})), V(false)); + EXPECT_EQ(eval(json({{"!", json::array({json::array()})}})), V(true)); + EXPECT_EQ(eval(json({{"!!", "hello"}})), V(true)); + EXPECT_EQ(eval(json({{"and", json::array({true, 1, "x"})}})), V("x")); + EXPECT_EQ(eval(json({{"and", json::array({true, 0, "x"})}})), V(0)); + EXPECT_EQ(eval(json({{"or", json::array({0, "", "hit"})}})), V("hit")); + EXPECT_EQ(eval(json({{"or", json::array({0, ""})}})), V("")); +} + +TEST(TestJsonLogic, IfTernary) +{ + EXPECT_EQ(eval(json({{"if", json::array({true, "yes", "no"})}})), V("yes")); + EXPECT_EQ(eval(json({{"if", json::array({false, "yes", "no"})}})), V("no")); + EXPECT_EQ(eval(json({{"if", json::array({false, "a", true, "b", "c"})}})), V("b")); + EXPECT_EQ(eval(json({{"if", json::array({false, "a"})}})), V()); + EXPECT_EQ(eval(json({{"if", json::array({{{">", json::array({"$x", "$y"})}}, "$x", "$y"})}})), + V(41)); +} + +TEST(TestJsonLogic, Composed) +{ + EXPECT_EQ( + eval(json({{"if", + json::array({{{"<", json::array({{{"+", json::array({"$x", "$y"})}}, 50})}}, + "small", + "big"})}})), + V("small")); +} + +TEST(TestJsonLogic, Membership) +{ + EXPECT_EQ(eval(json({{"in", json::array({20, "$arr"})}})), V(true)); + EXPECT_EQ(eval(json({{"in", json::array({99, "$arr"})}})), V(false)); + EXPECT_EQ(eval(json({{"in", json::array({"$x", json::array({40, 41, 42})})}})), V(true)); + // strict element equality: "41" != 41 + EXPECT_EQ(eval(json({{"in", json::array({"41", json::array({40, 41, 42})})}})), V(false)); + EXPECT_EQ(eval(json({{"in", json::array({"m", "$name"})}})), V(true)); + EXPECT_EQ(eval(json({{"in", json::array({"z", "$name"})}})), V(false)); +} + +TEST(TestJsonLogic, MathExtensions) +{ + EXPECT_EQ(eval(json({{"ceil_div", json::array({100, 16})}})), V(7)); + EXPECT_EQ(eval(json({{"ceil_div", json::array({32, 16})}})), V(2)); + EXPECT_EQ(eval(json({{"abs", -5}})), V(5)); + EXPECT_EQ(eval(json({{"abs", 5}})), V(5)); + EXPECT_EQ(eval(json({{"pow", json::array({2, 10})}})), V(1024)); + EXPECT_EQ(eval(json({{"log2", 8}})), V(3)); + EXPECT_EQ(eval(json({{"rsqrt", 4}})), V(0.5)); +} + +TEST(TestJsonLogic, ValueOrDefault) +{ + EXPECT_EQ(eval(json({{"value_or_default", json::array({"$x", 99})}})), V(41)); + EXPECT_EQ(eval(json({{"value_or_default", json::array({"$nope", 99})}})), V(99)); + // keys on existence, not truthiness: present 0 is returned + EXPECT_EQ(eval(json({{"value_or_default", json::array({"$zero", 99})}})), V(0)); + EXPECT_EQ(eval(json({{"value_or_default", json::array({"$nested.a.b", -1})}})), V(7)); + // default is itself an expression, evaluated lazily + EXPECT_EQ(eval(json({{"value_or_default", json::array({"$nope", "$x"})}})), V(41)); + EXPECT_EQ(eval(json({{"value_or_default", json::array({"$nope", "fallback"})}})), + V("fallback")); +} + +TEST(TestJsonLogic, PresenceOperators) +{ + // A resolving path is present; an unresolved one is not. + EXPECT_EQ(eval(json({{"present", json::array({"$x"})}})), V(true)); + EXPECT_EQ(eval(json({{"present", json::array({"$nope"})}})), V(false)); + EXPECT_EQ(eval(json({{"not_present", json::array({"$nope"})}})), V(true)); + EXPECT_EQ(eval(json({{"not_present", json::array({"$x"})}})), V(false)); + // Presence keys on existence, not truthiness: a present 0 / false is present. + EXPECT_EQ(eval(json({{"present", json::array({"$zero"})}})), V(true)); + EXPECT_EQ(eval(json({{"not_present", json::array({"$zero"})}})), V(false)); + // Unlike every other operator, an unresolved path yields a real boolean + // rather than propagating null, so the criterion decides instead of declining. + EXPECT_TRUE(eval(json({{"present", json::array({"$nope"})}})).isBool()); + EXPECT_TRUE(eval(json({{"not_present", json::array({"$nope"})}})).isBool()); + // n-ary: both fold with `and` over every argument. + EXPECT_EQ(eval(json({{"present", json::array({"$x", "$y", "$name"})}})), V(true)); + EXPECT_EQ(eval(json({{"present", json::array({"$x", "$nope"})}})), V(false)); + EXPECT_EQ(eval(json({{"not_present", json::array({"$nope", "$missing"})}})), V(true)); + EXPECT_EQ(eval(json({{"not_present", json::array({"$nope", "$x"})}})), V(false)); + // unary sugar (a bare argument, not an array) + EXPECT_EQ(eval(json({{"present", "$x"}})), V(true)); + // at least one argument is required + EXPECT_THROW(jlogic::compile(json({{"present", json::array()}})), + jlogic::JsonLogicCompileError); + EXPECT_THROW(jlogic::compile(json({{"not_present", json::array()}})), + jlogic::JsonLogicCompileError); +} + +TEST(TestJsonLogic, NullPropagatesThroughEveryOtherOperator) +{ + // An unresolved reference is "unknown", not a value. Every operator except + // the presence pair and value_or_default must yield null rather than + // coerce, because a coerced null reads as false / 0 / not-equal and would + // make a narrowing check silently PASS on data it never saw. + for(const json& rule : {json({{"!", "$nope"}}), + json({{"!!", "$nope"}}), + json({{"!=", json::array({"$nope", 1})}}), + json({{"==", json::array({"$nope", 1})}}), + json({{"<", json::array({"$nope", 5})}}), + json({{"<=", json::array({"$nope", 5})}}), + json({{">", json::array({"$nope", 5})}}), + json({{">=", json::array({"$nope", 0})}}), + json({{"+", json::array({"$nope", 1})}}), + json({{"*", json::array({"$nope", 1})}}), + json({{"-", json::array({"$nope", 1})}}), + json({{"in", json::array({"$nope", json::array({1, 2})})}}), + json({{"ceil_div", json::array({"$nope", 4})}}), + json({{"abs", "$nope"}}), + json({{"min", json::array({"$nope", 1})}}), + json({{"max", json::array({"$nope", 1})}}), + json({{"if", json::array({"$nope", 1, 2})}})}) + { + EXPECT_TRUE(eval(rule).isNull()) << rule.dump(); + } + // Two unresolved references are not "equal": the question is unanswerable. + EXPECT_TRUE(eval(json({{"==", json::array({"$nope", "$missing"})}})).isNull()); + EXPECT_TRUE(eval(json({{"!=", json::array({"$nope", "$missing"})}})).isNull()); +} + +TEST(TestJsonLogic, KleeneAndOrShortCircuitPastUnknown) +{ + // A definite false decides an `and` even beside an unknown, and a definite + // true decides an `or`. This is what lets "absent, or present and + // constrained" accept an absent operand whose field checks cannot run. + EXPECT_EQ(eval(json({{"and", json::array({false, "$nope"})}})), V(false)); + EXPECT_EQ(eval(json({{"or", json::array({true, "$nope"})}})), V(true)); + // Without a decisive argument the result stays unknown rather than + // collapsing to true/false. + EXPECT_TRUE(eval(json({{"and", json::array({true, "$nope"})}})).isNull()); + EXPECT_TRUE(eval(json({{"or", json::array({false, "$nope"})}})).isNull()); + // A fully-resolved expression is unaffected. + EXPECT_EQ(eval(json({{"and", json::array({true, true})}})), V(true)); + EXPECT_EQ(eval(json({{"or", json::array({false, false})}})), V(false)); +} + +TEST(TestJsonLogic, DivisionAndDomainErrorsFailClosed) +{ + // A zero divisor declines instead of yielding inf/NaN, giving uniform + // fail-closed zero-guarding (RFC 0018 A.7). + EXPECT_TRUE(eval(json({{"/", json::array({"$x", 0})}})).isNull()); + EXPECT_TRUE(eval(json({{"%", json::array({"$x", 0})}})).isNull()); + EXPECT_TRUE(eval(json({{"ceil_div", json::array({"$x", 0})}})).isNull()); + // log2/rsqrt decline on a non-positive argument rather than returning + // -inf/NaN. + EXPECT_TRUE(eval(json({{"log2", 0}})).isNull()); + EXPECT_TRUE(eval(json({{"rsqrt", 0}})).isNull()); + EXPECT_TRUE(eval(json({{"rsqrt", -4}})).isNull()); + // The well-behaved cases still compute. + EXPECT_EQ(eval(json({{"/", json::array({8, 2})}})), V(4)); + EXPECT_EQ(eval(json({{"log2", 8}})), V(3)); +} + +TEST(TestJsonLogic, Umd0018ConstraintShapes) +{ + EXPECT_EQ(eval(json({{"in", json::array({"$name", json::array({"amd", "xilinx"})})}})), + V(true)); + EXPECT_EQ(eval(json({{"==", json::array({"$arr", json::array({10, 20, 30})})}})), V(true)); + // 41 % 8 == 1 + EXPECT_EQ(eval(json({{"==", json::array({{{"%", json::array({"$x", "$y"})}}, 1})}})), V(true)); + // ceil(41/16) == 3 + EXPECT_EQ(eval(json({{"ceil_div", json::array({"$x", 16})}})), V(3)); +} + +TEST(TestJsonLogic, CompileOnceReuseAcrossData) +{ + const auto expr + = jlogic::compile(json({{"*", json::array({"$x", 2})}})); + const jlogic::JsonDataSource a{json{{"x", 3}}}; + const jlogic::JsonDataSource b{json{{"x", 10}}}; + EXPECT_EQ(expr(a), V(6)); + EXPECT_EQ(expr(b), V(20)); +} + +TEST(TestJsonLogic, MalformedRulesThrowAtCompileTime) +{ + EXPECT_THROW(jlogic::compile(json({{"nope", json::array({1, 2})}})), + jlogic::JsonLogicCompileError); + EXPECT_THROW(jlogic::compile(json({{"/", json::array({1, 2, 3})}})), + jlogic::JsonLogicCompileError); + EXPECT_THROW(jlogic::compile(json({{"abs", json::array({1, 2})}})), + jlogic::JsonLogicCompileError); +} + +TEST(TestJsonLogic, VariablesCollectsReferencedPaths) +{ + using S = std::set; + // Keep the Expression alive while draining the borrowed range into a set. + const auto vars = [](const json& rule) { + const auto expr = jlogic::compile(rule); + return S(expr.variables().begin(), expr.variables().end()); + }; + // inline vars across an op + EXPECT_EQ(vars(json({{"+", json::array({"$x", "$y"})}})), (S{"x", "y"})); + // stock var form keeps the dotted path verbatim + EXPECT_EQ(vars(json({{"var", "nested.a.b"}})), (S{"nested.a.b"})); + // a var default subtree contributes its own referenced vars + EXPECT_EQ(vars(json({{"var", json::array({"nope", "$y"})}})), (S{"nope", "y"})); + // nested composition reaches every leaf var + EXPECT_EQ( + vars(json({{"if", json::array({{{">", json::array({"$x", "$y"})}}, "$name", "$zero"})}})), + (S{"x", "y", "name", "zero"})); + // literal-only expressions reference nothing + EXPECT_EQ(vars(json({{"+", json::array({1, 2})}})), (S{})); + EXPECT_TRUE(vars(json(42)).empty()); +} + +TEST(TestJsonLogic, ReferencesVariableRootMatchesFirstToken) +{ + const auto refs = [](const json& rule, std::string_view root) { + const auto expr = jlogic::compile(rule); + return jlogic::referencesVariableRoot(expr, root); + }; + // matches on the first path token, before any '.' or '[' separator + EXPECT_TRUE(refs(json({{"<", json::array({"$kernel.tile_m", "$device.lds_size"})}}), "kernel")); + EXPECT_TRUE(refs(json({{"<", json::array({"$kernel.tile_m", "$device.lds_size"})}}), "device")); + EXPECT_TRUE(refs(json({{"==", json::array({"$kernel.vec[0]", 1})}}), "kernel")); + // a bare root (no field) still matches its own token + EXPECT_TRUE(refs(json("$kernel"), "kernel")); + // no variable has that root + EXPECT_FALSE(refs(json({{"==", json::array({"$q.head_size", 128})}}), "kernel")); + // a root is a whole-token match, not a prefix + EXPECT_FALSE(refs(json("$kernelish.x"), "kernel")); + // literal-only expression references nothing + EXPECT_FALSE(refs(json({{"+", json::array({1, 2})}}), "kernel")); +} + +TEST(TestJsonLogic, VariablesRangeIsLazyAndKeepsDuplicates) +{ + // range-for yields every occurrence, duplicates included + const auto expr + = jlogic::compile(json({{"+", json::array({"$x", "$x", "$y"})}})); + std::vector seen; + for(const std::string& v : expr.variables()) + { + seen.push_back(v); + } + EXPECT_EQ(seen.size(), 3u); + EXPECT_EQ(std::count(seen.begin(), seen.end(), std::string("x")), 2); + EXPECT_EQ(std::count(seen.begin(), seen.end(), std::string("y")), 1); + + // empty range: begin() == end(), the body never runs + const auto lit = jlogic::compile(json(42)); + EXPECT_TRUE(lit.variables().begin() == lit.variables().end()); + + // composes with STL algorithms over the borrowed references + const auto r = expr.variables(); + EXPECT_EQ(std::distance(r.begin(), r.end()), 3); + EXPECT_TRUE(std::any_of(r.begin(), r.end(), [](const std::string& s) { return s == "y"; })); +} + +TEST(TestJsonLogic, WholeDocumentAndDynamicKeysRejected) +{ + // whole-document references: inline sigil, empty var, empty array, null + EXPECT_THROW(jlogic::compile(json("$")), jlogic::JsonLogicCompileError); + EXPECT_THROW(jlogic::compile(json({{"var", ""}})), + jlogic::JsonLogicCompileError); + EXPECT_THROW(jlogic::compile(json({{"var", json::array()}})), + jlogic::JsonLogicCompileError); + EXPECT_THROW(jlogic::compile(json({{"var", nullptr}})), + jlogic::JsonLogicCompileError); + // computed / dynamic keys (object key, or computed first array element) + EXPECT_THROW( + jlogic::compile(json({{"var", {{"+", json::array({1, 2})}}}})), + jlogic::JsonLogicCompileError); + EXPECT_THROW(jlogic::compile( + json({{"var", json::array({{{"+", json::array({1, 2})}}, 0})}})), + jlogic::JsonLogicCompileError); +} diff --git a/dnn-providers/hip-kernel-provider/src/tests/core/TestUniversalGraphMatcher.cpp b/dnn-providers/hip-kernel-provider/src/tests/core/TestUniversalGraphMatcher.cpp new file mode 100644 index 000000000000..8a96bf9471ad --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/tests/core/TestUniversalGraphMatcher.cpp @@ -0,0 +1,1656 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +// +// RFC 0018 Phase 1: proves the bindings + UMD compiler + matcher end to end. +// - Compiler: A.10 validation (accept the SDPA descriptor; reject bad schema, +// mismatched `?`, unknown name, type errors, unknown operators). +// - Lowering: `shape` -> rank check with named dims; layout alias -> array. +// - Matcher: structural match + criteria evaluation over a live graph, with +// fail-closed rejection on the declarative gates. +// - Bindings: the queryable BindingContext resolves every `$`-namespace form. +// +// The full match-equivalence battery against SdpaFwdPlanBuilder::isApplicable +// and the enumerated binding-completeness proof are the Phase 2 deliverable. + +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "hip_kernel_provider_common/umd/BindingContext.hpp" +#include "hip_kernel_provider_common/umd/UmdCompiler.hpp" +#include "hip_kernel_provider_common/umd/UniversalGraphMatcher.hpp" + +namespace umd = hip_kernel_provider_common::umd; +namespace jlogic = hip_kernel_provider_common::jsonlogic; +namespace data = hipdnn_flatbuffers_sdk::data_objects; +namespace fbu = hipdnn_flatbuffers_sdk::flatbuffer_utilities; + +using json = nlohmann::json; + +namespace +{ + +// The RFC §18 SDPA-forward descriptor. The two gates that need real C++ +// (strides_fit_u32, sdpa_mask_consistent) are not criteria: the expression +// language has no custom-operation hatch, so they live in a native matcher +// beside this UMD (RFC 0018 §8). Name keys are the lowercase op-schema +// registry names. +json sdpaDescriptor() +{ + return json::parse(R"JSON({ + "schema": "hipdnn.umd/v1", + "id": "9c3f5b2a-7d41-4e88-b6a0-1f2e3d4c5b6a", + "name": "SDPA forward (d128, bf16/fp8) match", + "nodes": [ + {"kind": "op", "id": "sdpa_fwd", "op": "sdpa_fwd", + "operands": {"q": "$q", "k": "$k", "v": "$v", + "attn_mask": "$attn_mask?", "page_table_k": "$page_table_k?", + "page_table_v": "$page_table_v?"}, + "results": {"o": "$o"}} + ], + "criteria": {"and": [ + {"==": ["$graph.node_count", 1]}, + {"in": ["$q.dtype", ["BFLOAT16", "FP8_E4M3"]]}, + {"==": ["$k.dtype", "$q.dtype"]}, {"==": ["$v.dtype", "$q.dtype"]}, + {"shape": ["$q", ["batch", "num_heads", "seqlen_q", "head_size"]]}, + {"shape": ["$k", ["batch", "kv_heads", "seqlen_k", "head_size"]]}, + {"shape": ["$v", ["batch", "kv_heads", "seqlen_k", "head_size"]]}, + {"==": ["$o.rank", 4]}, + {"==": ["$q.head_size", 128]}, + {"==": ["$k.head_size", "$q.head_size"]}, + {"or": [{"!": "$sdpa_fwd.dropout_probability.present"}, {"==": ["$sdpa_fwd.dropout_probability", 0.0]}]}, + {"==": ["$sdpa_fwd.alibi_mask", false]}, + {"==": ["$sdpa_fwd.padding_mask", false]}, + {"or": [{"!": "$sdpa_fwd.generate_stats.present"}, {"==": ["$sdpa_fwd.generate_stats", false]}]}, + {"!": "$attn_mask.present"}, + {"!": "$page_table_k.present"}, {"!": "$page_table_v.present"} + ]} + })JSON"); +} + +std::vector contiguousStrides(const std::vector& dims) +{ + std::vector strides(dims.size(), 1); + for(std::size_t i = dims.size() - 1; i-- > 0;) + { + strides[i] = strides[i + 1] * dims[i + 1]; + } + return strides; +} + +// Build a single-node SDPA-forward graph with the given per-tensor dims and +// dtype (contiguous strides), returning the owning builder. `withScale` adds +// the optional `scale` operand as a pass-by-value tensor carrying a +// compile-time Float32 value of 1.0. +flatbuffers::FlatBufferBuilder buildSdpaGraph(const std::vector& dims, + data::DataType dtype, + bool withAttnMask = false, + bool overrideShape = false, + bool withScale = false) +{ + const std::vector strides = contiguousStrides(dims); + return hipdnn_test_sdk::utilities::createValidSdpaFwdGraph(dims, + strides, + dims, + strides, + dims, + strides, + dims, + strides, + dtype, + withAttnMask, + withScale, + /*withStats=*/false, + /*alibiMask=*/false, + /*paddingMask=*/false, + /*causalMask=*/false, + overrideShape); +} + +umd::UniversalGraphMatcher makeSdpaMatcher() +{ + return umd::UniversalGraphMatcher(sdpaDescriptor()); +} + +constexpr umd::DeviceProperties K_DEVICE{/*ldsSize=*/65536, /*warpSize=*/64}; + +// The sdpaDescriptor() `id`, in the form the compiled descriptor carries it. +const hipdnn_flatbuffers_sdk::utilities::UuidBytes K_SDPA_UMD_ID + = hipdnn_flatbuffers_sdk::utilities::parseUuid("9c3f5b2a-7d41-4e88-b6a0-1f2e3d4c5b6a"); + +} // namespace + +// ---- Compiler validation (A.10) ------------------------------------------ + +TEST(TestUmdCompiler, AcceptsSdpaDescriptor) +{ + EXPECT_NO_THROW(umd::UmdCompiler::compile(sdpaDescriptor())); +} + +TEST(TestUmdCompiler, PublishesReferencedBoundSymbols) +{ + const umd::CompiledUmd c = umd::UmdCompiler::compile(sdpaDescriptor()); + // The `shape` short-hands lower to `$q.rank` etc.; named dims are read + // directly. + EXPECT_TRUE(c.boundSymbols.count("q.head_size") == 1); + EXPECT_TRUE(c.boundSymbols.count("graph.node_count") == 1); + EXPECT_TRUE(c.boundSymbols.count("sdpa_fwd.dropout_probability") == 1); + EXPECT_TRUE(c.boundSymbols.count("attn_mask.present") == 1); +} + +TEST(TestUmdCompiler, RejectsWrongSchema) +{ + json d = sdpaDescriptor(); + d["schema"] = "hipdnn.umd/v2"; + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsMalformedUuid) +{ + json d = sdpaDescriptor(); + d["id"] = "not-a-uuid"; + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsWellSizedNonHexUuid) +{ + // Right shape, wrong alphabet: the id is parsed, not pattern-matched, so a + // non-hex digit must still surface as a compile error. + json d = sdpaDescriptor(); + d["id"] = "9c3f5b2a-7d41-4e88-b6a0-1f2e3d4c5bZZ"; + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsNonStringUuid) +{ + json d = sdpaDescriptor(); + d["id"] = 42; + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsUnknownTopLevelKey) +{ + json d = sdpaDescriptor(); + d["priority"] = 3; + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsUnknownOperandName) +{ + json d = sdpaDescriptor(); + d["nodes"][0]["operands"]["not_a_name"] = "$bogus"; + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsQuestionOnRequiredName) +{ + json d = sdpaDescriptor(); + d["nodes"][0]["operands"]["q"] = "$q?"; // q is required + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsMissingQuestionOnOptionalName) +{ + json d = sdpaDescriptor(); + d["nodes"][0]["operands"]["attn_mask"] = "$attn_mask"; // optional, needs ? + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsReservedRootAsNodeId) +{ + json d = sdpaDescriptor(); + d["nodes"][0]["id"] = "device"; + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsTypeMismatchIntVsDtype) +{ + json d = sdpaDescriptor(); + // $q.head_size is Int; comparing it to a Dtype string is a compile error. + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$q.head_size", "BFLOAT16"]})")); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsNonBooleanCriteria) +{ + json d = sdpaDescriptor(); + d["criteria"] = json::parse(R"({"+": ["$q.head_size", 1]})"); // Int, not Bool + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsUnknownOperator) +{ + // A.7 is exhaustive and there is no registry a descriptor can extend it + // through, so a namespaced key -- the form a custom operation would have + // taken -- is refused as an unrecognized operator like any other. + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"hipdnn.strides_fit_u32": ["$q"]})")); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsPresentOnRequiredOperand) +{ + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"!": "$q.present"})")); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsWrongArity) +{ + // Binary/unary operators with the wrong argument count must refuse at + // compile with a UmdCompileError, not escape as std::out_of_range. + for(const char* bad : {R"({"==": ["$q.rank"]})", + R"({"==": ["$q.rank", 4, 4]})", + R"({"<": ["$q.rank"]})", + R"({"in": ["$q.dtype"]})", + R"({"and": [true]})", + R"({"if": [true, 1]})"}) + { + json d = sdpaDescriptor(); + d["criteria"] = json::parse(bad); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError) << bad; + } +} + +TEST(TestUmdCompiler, RejectsMalformedSubscript) +{ + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$q.dims[x]", 2]})")); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsBareNodeIdAsValue) +{ + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$sdpa_fwd", 0]})")); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsLayoutAliasRankMismatch) +{ + // $q is pinned rank 4 by the shape short-hand; a rank-5 alias can never + // match, so it is refused at compile rather than always declining. + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$q.stride_order", "ncdhw"]})")); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, AcceptsWellFormedArithmeticAndIf) +{ + // Guard the arity checks against over-rejecting valid n-ary / binary / if + // forms. + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse( + R"({"==": [{"if": [{"<": ["$q.head_size", 64]}, {"-": ["$q.head_size", 1]}, 128]}, 128]})")); + EXPECT_NO_THROW(umd::UmdCompiler::compile(d)); +} + +// ---- Matcher: accept / reject -------------------------------------------- + +TEST(TestUniversalGraphMatcher, OwnsSingleDescriptor) +{ + // A matcher has a 1:1 relationship with its UMD: it exposes that one + // descriptor's id and compiled form. + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + EXPECT_EQ(m.umdId(), K_SDPA_UMD_ID); + EXPECT_EQ(m.descriptor().id, K_SDPA_UMD_ID); + EXPECT_EQ(m.descriptor().nodes.size(), 1u); +} + +TEST(TestUniversalGraphMatcher, MatchesValidSdpaForward) +{ + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + + const umd::MatchResult r = m.match(K_DEVICE, g); + ASSERT_TRUE(r.matched); + EXPECT_EQ(r.umdId, K_SDPA_UMD_ID); +} + +TEST(TestUniversalGraphMatcher, DeclinesWrongHeadSize) +{ + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + auto builder = buildSdpaGraph({2, 8, 16, 64}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, g).matched); +} + +TEST(TestUniversalGraphMatcher, DeclinesUnsupportedDtype) +{ + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::HALF); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, g).matched); +} + +TEST(TestUniversalGraphMatcher, DeclinesWhenUnsupportedOptionalPresent) +{ + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16, /*withAttnMask=*/true); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, g).matched); +} + +TEST(TestUniversalGraphMatcher, DeclinesOverrideShapeGraph) +{ + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + auto builder = buildSdpaGraph({2, 8, 16, 128}, + data::DataType::BFLOAT16, + /*withAttnMask=*/false, + /*overrideShape=*/true); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, g).matched); +} + +TEST(TestUniversalGraphMatcher, DeclinesGraphWithoutMatchingOpcode) +{ + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + auto builder = hipdnn_test_sdk::utilities::createValidBlockScaleQuantizeGraph(); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, g).matched); +} + +TEST(TestUniversalGraphMatcher, DeclinesNodelessGraphWithoutThrowing) +{ + // A valid but empty graph must fail closed, not throw (nodeWrappers() would + // throw on a null nodes vector). + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + auto builder = hipdnn_test_sdk::utilities::createEmptyValidGraph(); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_NO_THROW({ EXPECT_FALSE(m.match(K_DEVICE, g).matched); }); +} + +// ---- Bindings: queryable BindingContext (RFC 0018 §4/§15) ---------------- + +TEST(TestBindingContext, ResolvesEveryNamespaceForm) +{ + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + + const umd::MatchResult r = m.match(K_DEVICE, g); + ASSERT_TRUE(r.matched); + const umd::BindingContext& b = r.bindings; + + // Tensor namespace. + EXPECT_EQ(b.get("$q.uid").asInt(), 1); + EXPECT_EQ(b.get("$q.rank").asInt(), 4); + EXPECT_EQ(b.get("$q.dtype").asString(), "BFLOAT16"); + EXPECT_EQ(b.get("$q.dims[0]").asInt(), 2); // positional + EXPECT_EQ(b.get("$q.head_size").asInt(), 128); // named dim + EXPECT_EQ(b.get("$q.strides[3]").asInt(), 1); + EXPECT_TRUE(b.get("$q.packed").asBool()); + EXPECT_FALSE(b.get("$q.virtual").asBool()); + + // stride_order is published in the RFC 0017 §5 form: logical dim indices + // outermost-first, so a contiguous rank-4 tensor is [0,1,2,3]. That is the + // inverse of extractStrideOrder's stride-rank vector, which BindingContext + // converts once (RFC 0018 §7/A.8). + const jlogic::Value strideOrder = b.get("$q.stride_order"); + ASSERT_TRUE(strideOrder.isArray()); + EXPECT_EQ(strideOrder, + jlogic::Value(jlogic::Value::Array{jlogic::Value(std::int64_t{0}), + jlogic::Value(std::int64_t{1}), + jlogic::Value(std::int64_t{2}), + jlogic::Value(std::int64_t{3})})); + + // Attributes namespace. + EXPECT_FALSE(b.get("$sdpa_fwd.dropout_probability.present").asBool()); + EXPECT_FALSE(b.get("$sdpa_fwd.alibi_mask").asBool()); + + // Graph and device namespaces. + EXPECT_EQ(b.get("$graph.node_count").asInt(), 1); + EXPECT_EQ(b.get("$device.lds_size").asInt(), 65536); + EXPECT_EQ(b.get("$device.warp_size").asInt(), 64); +} + +TEST(TestBindingContext, FailsClosedOnBadPaths) +{ + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + + const umd::MatchResult r = m.match(K_DEVICE, g); + ASSERT_TRUE(r.matched); + const umd::BindingContext& b = r.bindings; + + EXPECT_TRUE(b.get("$q.dims[9]").isNull()); // out of range + EXPECT_TRUE(b.get("$q.unknown_dim").isNull()); // unknown dim-name + EXPECT_TRUE(b.get("$attn_mask.dtype").isNull()); // absent optional field read + EXPECT_TRUE(b.get("$device.unknown").isNull()); // unknown device field + EXPECT_TRUE(b.get("$nope.rank").isNull()); // unknown tensor var +} + +// ---- Lowering: layout alias (A.8) ---------------------------------------- + +TEST(TestUmdCompiler, LayoutAliasMatchesContiguousLayout) +{ + const umd::UniversalGraphMatcher m(json::parse(R"JSON({ + "schema": "hipdnn.umd/v1", + "id": "11111111-2222-3333-4444-555555555555", + "name": "layout alias smoke", + "nodes": [{"kind": "op", "id": "sdpa_fwd", "op": "sdpa_fwd", + "operands": {"q": "$q"}}], + "criteria": {"and": [ + {"==": ["$graph.node_count", 1]}, + {"shape": ["$q", ["b", "h", "s", "d"]]}, + {"==": ["$q.stride_order", "nchw"]} + ]} + })JSON")); + + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, g).matched); +} + +TEST(TestBindingContext, StrideOrderIsPublishedOutermostFirst) +{ + // The conversion is observable, not cosmetic: an NHWC tensor must publish + // [0,2,3,1] (N,H,W,C read left to right), NOT extractStrideOrder's + // [3,0,2,1] stride-rank vector. Reading the rank vector as if it were this + // form would name the dims H,C,W,N -- a layout that does not exist. + const std::vector dims{2, 8, 16, 128}; + // NHWC over an (n,c,h,w) logical dim order: c varies fastest. + const std::vector nhwcStrides + = hipdnn_data_sdk::utilities::generateStrides(dims, {3, 0, 2, 1}); + + auto builder = hipdnn_test_sdk::utilities::createValidSdpaFwdGraph(dims, + nhwcStrides, + dims, + nhwcStrides, + dims, + nhwcStrides, + dims, + nhwcStrides, + data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$q.stride_order", [0, 2, 3, 1]]})")); + const umd::UniversalGraphMatcher m(d); + + const umd::MatchResult r = m.match(K_DEVICE, g); + ASSERT_TRUE(r.matched); + EXPECT_EQ(r.bindings.get("$q.stride_order"), + jlogic::Value(jlogic::Value::Array{jlogic::Value(std::int64_t{0}), + jlogic::Value(std::int64_t{2}), + jlogic::Value(std::int64_t{3}), + jlogic::Value(std::int64_t{1})})); + + // The `nhwc` alias expands to the same array, so it matches the same graph. + json aliased = sdpaDescriptor(); + aliased["criteria"]["and"].push_back(json::parse(R"({"==": ["$q.stride_order", "nhwc"]})")); + EXPECT_TRUE(umd::UniversalGraphMatcher(aliased).match(K_DEVICE, g).matched); +} + +// ---- Multi-node patterns (RFC 0018 §3/A.3) ------------------------------- + +namespace +{ + +// Build a two-node SDPA graph. When `chained`, node1's output tensor (uid 4) is +// also node2's Q input, forming an edge; otherwise node2 reads an independent Q +// (uid 8), so no edge exists. `alibi2` sets node2's alibi_mask to exercise +// per-node attribute resolution. +flatbuffers::FlatBufferBuilder buildTwoNodeSdpaGraph(bool chained, bool alibi2 = false) +{ + flatbuffers::FlatBufferBuilder builder; + const std::vector dims{2, 8, 16, 128}; + const std::vector strides = contiguousStrides(dims); + const data::DataType dtype = data::DataType::BFLOAT16; + + const auto mkTensor = [&](std::int64_t uid, const char* name) { + return data::CreateTensorAttributesDirect(builder, uid, name, dtype, &strides, &dims); + }; + + std::vector<::flatbuffers::Offset> tensors; + tensors.push_back(mkTensor(1, "q1")); + tensors.push_back(mkTensor(2, "k1")); + tensors.push_back(mkTensor(3, "v1")); + tensors.push_back(mkTensor(4, "o1")); // node1 output + tensors.push_back(mkTensor(5, "k2")); + tensors.push_back(mkTensor(6, "v2")); + tensors.push_back(mkTensor(7, "o2")); // node2 output + const std::int64_t node2q = chained ? 4 : 8; + if(!chained) + { + tensors.push_back(mkTensor(8, "q2")); + } + + const auto mkSdpa + = [&](std::int64_t q, std::int64_t k, std::int64_t v, std::int64_t o, bool alibi) { + data::SdpaAttributesBuilder sb(builder); + sb.add_q_tensor_uid(q); + sb.add_k_tensor_uid(k); + sb.add_v_tensor_uid(v); + sb.add_o_tensor_uid(o); + sb.add_alibi_mask(alibi); + return sb.Finish(); + }; + const auto s1 = mkSdpa(1, 2, 3, 4, /*alibi=*/false); + const auto s2 = mkSdpa(node2q, 5, 6, 7, alibi2); + + std::vector<::flatbuffers::Offset> nodes; + nodes.push_back(data::CreateNodeDirect( + builder, "sdpa1", dtype, data::NodeAttributes::SdpaAttributes, s1.Union())); + nodes.push_back(data::CreateNodeDirect( + builder, "sdpa2", dtype, data::NodeAttributes::SdpaAttributes, s2.Union())); + + auto graph = data::CreateGraphDirect(builder, + "test", + data::DataType::FLOAT, + data::DataType::HALF, + data::DataType::BFLOAT16, + &tensors, + &nodes, + flatbuffers::nullopt, + /*is_override_shape_enabled=*/false); + builder.Finish(graph); + return builder; +} + +// A two-node fusion pattern: two SDPA nodes chained by the shared `$mid` edge, +// each bound to its own node id (`sdpa1`, `sdpa2`). +json twoNodeChainDescriptor() +{ + return json::parse(R"JSON({ + "schema": "hipdnn.umd/v1", + "id": "22222222-3333-4444-5555-666666666666", + "name": "two-node sdpa chain", + "nodes": [ + {"kind": "op", "id": "sdpa1", "op": "sdpa_fwd", + "operands": {"q": "$q", "k": "$k1", "v": "$v1"}, "results": {"o": "$mid"}}, + {"kind": "op", "id": "sdpa2", "op": "sdpa_fwd", + "operands": {"q": "$mid", "k": "$k2", "v": "$v2"}, "results": {"o": "$out"}} + ], + "criteria": {"and": [ + {"==": ["$graph.node_count", 2]}, + {"shape": ["$mid", ["b", "h", "s", "d"]]}, + {"==": ["$mid.d", 128]}, + {"==": ["$q.dtype", "$out.dtype"]}, + {"==": ["$sdpa1.alibi_mask", false]}, + {"==": ["$sdpa2.alibi_mask", false]} + ]} + })JSON"); +} + +} // namespace + +TEST(TestUmdCompiler, AcceptsMultiNodeDescriptor) +{ + const umd::CompiledUmd c = umd::UmdCompiler::compile(twoNodeChainDescriptor()); + EXPECT_EQ(c.nodes.size(), 2u); + ASSERT_NE(c.findNode("sdpa1"), nullptr); + ASSERT_NE(c.findNode("sdpa2"), nullptr); + // The `$mid` edge variable is shared by both nodes (result of one, operand + // of the other) but declared once. + ASSERT_NE(c.findTvar("mid"), nullptr); +} + +TEST(TestUmdCompiler, RejectsDuplicateNodeId) +{ + json d = twoNodeChainDescriptor(); + d["nodes"][1]["id"] = "sdpa1"; // collides with node 0 + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsVariableProducedTwice) +{ + json d = twoNodeChainDescriptor(); + d["nodes"][1]["results"]["o"] = "$mid"; // $mid produced by both nodes + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUniversalGraphMatcher, MatchesTwoNodeChainAndBindsEachNode) +{ + const umd::UniversalGraphMatcher m(twoNodeChainDescriptor()); + auto builder = buildTwoNodeSdpaGraph(/*chained=*/true); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + + const umd::MatchResult r = m.match(K_DEVICE, g); + ASSERT_TRUE(r.matched); + const umd::BindingContext& b = r.bindings; + // Each pattern variable resolved to its own graph tensor; the shared edge + // variable resolved to the tensor produced by node1 and consumed by node2. + EXPECT_EQ(b.get("$q.uid").asInt(), 1); + EXPECT_EQ(b.get("$mid.uid").asInt(), 4); + EXPECT_EQ(b.get("$out.uid").asInt(), 7); + EXPECT_EQ(b.get("$mid.d").asInt(), 128); // named dim on the edge tensor + // Each node id resolves its own attributes namespace. + EXPECT_FALSE(b.get("$sdpa1.alibi_mask").asBool()); + EXPECT_FALSE(b.get("$sdpa2.alibi_mask").asBool()); +} + +TEST(TestUniversalGraphMatcher, DeclinesWhenEdgeIsInconsistent) +{ + const umd::UniversalGraphMatcher m(twoNodeChainDescriptor()); + // node2 reads an independent Q, so no tensor satisfies the shared `$mid` + // edge (node1.o != node2.q) -> no consistent assignment -> decline. + auto builder = buildTwoNodeSdpaGraph(/*chained=*/false); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, g).matched); +} + +TEST(TestUniversalGraphMatcher, PerNodeAttributesAreDistinct) +{ + const umd::UniversalGraphMatcher m(twoNodeChainDescriptor()); + // Chained graph, but node2's alibi_mask is set; the criterion + // {"==": ["$sdpa2.alibi_mask", false]} must read node2's own attribute and + // decline, proving per-node attribute resolution. + auto builder = buildTwoNodeSdpaGraph(/*chained=*/true, /*alibi2=*/true); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, g).matched); +} + +// ---- Kernel metadata ($kernel) ------------------------------------------- + +namespace +{ + +// The SDPA descriptor with one `$kernel.` criterion added. Compiles only +// because the compiler resolves `$kernel.*` as a runtime-DYNAMIC scalar. +json sdpaKernelDescriptor() +{ + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$kernel.head_dim", "$q.head_size"]})")); + return d; +} + +} // namespace + +TEST(TestUmdCompiler, AcceptsKernelReference) +{ + // A criterion over a $kernel field compiles (DYNAMIC wildcard) and the + // field name it reads is recorded. + umd::CompiledUmd c; + ASSERT_NO_THROW(c = umd::UmdCompiler::compile(sdpaKernelDescriptor())); + EXPECT_EQ(c.kernelFields, (std::set{"head_dim"})); + EXPECT_TRUE(c.boundSymbols.count("kernel.head_dim") == 1); +} + +TEST(TestUmdCompiler, StockDescriptorDoesNotReferenceKernel) +{ + const umd::CompiledUmd c = umd::UmdCompiler::compile(sdpaDescriptor()); + EXPECT_TRUE(c.kernelFields.empty()); +} + +TEST(TestUmdCompiler, RecordsEveryDistinctKernelFieldRead) +{ + // The memoization key is the set of `$kernel.*` fields the matcher reads, + // without the `kernel.` prefix; a field read twice contributes once, and a + // subscripted path contributes its subscript form (RFC 0017 §5). + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$kernel.tile_m", 64]})")); + d["criteria"]["and"].push_back(json::parse(R"({"<": ["$kernel.tile_m", "$kernel.tile_n"]})")); + const umd::CompiledUmd c = umd::UmdCompiler::compile(d); + EXPECT_EQ(c.kernelFields, (std::set{"tile_m", "tile_n"})); +} + +TEST(TestUmdCompiler, KernelFieldUnifiesWithAnyScalarDomain) +{ + // A $kernel field is DYNAMIC: it compiles against an int, a dtype string, + // and a bool alike -- none is a static type error. + for(const char* crit : {R"({"==": ["$kernel.head_dim", 128]})", + R"({"==": ["$kernel.target_dtype", "BFLOAT16"]})", + R"({"<": ["$kernel.tile_m", "$q.head_size"]})", + R"({"!": "$kernel.causal"})"}) + { + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(crit)); + EXPECT_NO_THROW(umd::UmdCompiler::compile(d)) << crit; + } +} + +TEST(TestUmdCompiler, KernelReferenceNeedsAField) +{ + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$kernel", 1]})")); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, KernelFieldAsIfCondition) +{ + // A DYNAMIC $kernel field is accepted in an `if` condition, consistent with + // its acceptance in and/or/! boolean positions. + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"==": [{"if": ["$kernel.causal", 1, 0]}, 0]})")); + EXPECT_NO_THROW(umd::UmdCompiler::compile(d)); +} + +TEST(TestUmdCompiler, KernelFieldDoesNotUnifyWithArray) +{ + // DYNAMIC is a scalar wildcard: comparing a $kernel field to an array + // literal is a static type error, not a runtime-deferred check. + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$kernel.tiles", [1, 2, 3]]})")); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, KernelIsReservedAsPatternVariable) +{ + json d = sdpaDescriptor(); + d["nodes"][0]["operands"]["q"] = "$kernel"; // bind a tensor var to the reserved root + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, KernelIsReservedAsNodeId) +{ + json d = sdpaDescriptor(); + d["nodes"][0]["id"] = "kernel"; + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUniversalGraphMatcher, ReportsKernelMetadataReference) +{ + const umd::UniversalGraphMatcher mk(sdpaKernelDescriptor()); + EXPECT_TRUE(mk.referencesKernelMetadata()); + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + EXPECT_FALSE(m.referencesKernelMetadata()); +} + +TEST(TestUniversalGraphMatcher, PublishesKernelFieldsItReads) +{ + // The set is the loader's KMD existence check and the per-kernel + // memoization key (RFC 0017 §5): exactly the names read, prefix stripped. + const umd::UniversalGraphMatcher mk(sdpaKernelDescriptor()); + EXPECT_EQ(mk.kernelFields(), (std::set{"head_dim"})); + // A graph-only matcher reads none, which is what makes the two-argument + // match() overload legal for it. + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + EXPECT_TRUE(m.kernelFields().empty()); +} + +TEST(TestUniversalGraphMatcher, PublishesBoundSymbols) +{ + // A pack's dispatch descriptor is checked against the fields its matchers + // bind, so the set must be reachable from the matcher (RFC 0017 §6). + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + EXPECT_EQ(m.boundSymbols(), m.descriptor().boundSymbols); + EXPECT_TRUE(m.boundSymbols().count("q.head_size") == 1); +} + +TEST(TestUniversalGraphMatcher, TwoArgMatchThrowsWhenDescriptorNeedsKernel) +{ + const umd::UniversalGraphMatcher m(sdpaKernelDescriptor()); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + // Wrong overload for a kernel-referencing UMD -> programming error. + EXPECT_THROW(m.match(K_DEVICE, g), std::logic_error); +} + +TEST(TestUniversalGraphMatcher, MatchesWithSuppliedKernelMetadata) +{ + const umd::UniversalGraphMatcher m(sdpaKernelDescriptor()); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + + // head_dim matches $q.head_size (128) -> match; the value is queryable. + const umd::MatchResult r = m.match(K_DEVICE, g, json{{"head_dim", 128}}); + ASSERT_TRUE(r.matched); + EXPECT_EQ(r.bindings.get("$kernel.head_dim").asInt(), 128); +} + +TEST(TestUniversalGraphMatcher, DeclinesWhenKernelMetadataMismatches) +{ + const umd::UniversalGraphMatcher m(sdpaKernelDescriptor()); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + // head_dim != $q.head_size (128) -> criterion false -> decline. + EXPECT_FALSE(m.match(K_DEVICE, g, json{{"head_dim", 64}}).matched); +} + +TEST(TestUniversalGraphMatcher, EmptyKernelMetadataDeclinesComparison) +{ + // Defensive edge case: production metadata is fully resolved so every + // referenced field is present, but an unbound/empty document must not + // throw -- $kernel.head_dim reads null -> comparison false -> decline. + const umd::UniversalGraphMatcher m(sdpaKernelDescriptor()); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_NO_THROW({ EXPECT_FALSE(m.match(K_DEVICE, g, json::object()).matched); }); +} + +// ---- Descriptor versions (RFC 0017 §4 / A.10 §1) ------------------------- + +TEST(TestUmdCompiler, AcceptsAndPublishesWellFormedVersions) +{ + json d = sdpaDescriptor(); + d["version"] = "1.0"; + d["sdk_version"] = "1.2"; + umd::CompiledUmd c; + ASSERT_NO_THROW(c = umd::UmdCompiler::compile(d)); + EXPECT_EQ(c.version, "1.0"); + EXPECT_EQ(c.sdkVersion, "1.2"); +} + +TEST(TestUmdCompiler, VersionKeysAreOptionalAndDefaultToOneZero) +{ + // A descriptor omitting both still compiles, and both read "1.0": the + // version a descriptor authored before either key existed implies + // (RFC 0018 A.1). The published value is never empty, so the per-graph SDK + // floor always has a concrete version to compare against. + const umd::CompiledUmd c = umd::UmdCompiler::compile(sdpaDescriptor()); + EXPECT_EQ(c.version, "1.0"); + EXPECT_EQ(c.sdkVersion, "1.0"); +} + +TEST(TestUmdCompiler, RejectsMalformedVersion) +{ + for(const char* bad : {"1", "1.0.0", "1.", ".0", "+1.0", "-1.0", "1.x", "one.zero", ""}) + { + json d = sdpaDescriptor(); + d["version"] = bad; + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError) << bad; + json e = sdpaDescriptor(); + e["sdk_version"] = bad; + EXPECT_THROW(umd::UmdCompiler::compile(e), umd::UmdCompileError) << bad; + } +} + +TEST(TestUmdCompiler, RejectsNonStringVersion) +{ + json d = sdpaDescriptor(); + d["version"] = 1.0; + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, RejectsVersionNewerThanTheRuntime) +{ + // RFC 0017 §4: a minor newer than the runtime's carries features this + // runtime cannot understand, so the descriptor is refused, never silently + // reinterpreted. + json newerMinor = sdpaDescriptor(); + newerMinor["version"] = "1.9"; + EXPECT_THROW(umd::UmdCompiler::compile(newerMinor), umd::UmdCompileError); + + // A differing major is refused in either direction. + for(const char* major : {"0.9", "2.0"}) + { + json d = sdpaDescriptor(); + d["version"] = major; + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError) << major; + } + + // Same for the SDK version's ceiling half. + json newerSdk = sdpaDescriptor(); + newerSdk["sdk_version"] = "1.99"; + EXPECT_THROW(umd::UmdCompiler::compile(newerSdk), umd::UmdCompileError); +} + +TEST(TestUmdCompiler, AcceptsAnOlderMinorWithinTheSameMajor) +{ + // RFC 0017 §4: "A file stamped 1.0 loads on a 1.1 runtime." An older minor + // is never refused, so a descriptor stays loadable on the oldest runtime + // that can serve it. This is the case the previous floor-only gate broke. + EXPECT_TRUE(umd::UmdCompiler::versionLoadableOn("1.0", "1.1")); + EXPECT_TRUE(umd::UmdCompiler::versionLoadableOn("1.1", "1.1")); + EXPECT_FALSE(umd::UmdCompiler::versionLoadableOn("1.2", "1.1")); + EXPECT_FALSE(umd::UmdCompiler::versionLoadableOn("2.0", "1.1")); + EXPECT_FALSE(umd::UmdCompiler::versionLoadableOn("0.9", "1.1")); + // an unparseable version loads nowhere (fail closed) + EXPECT_FALSE(umd::UmdCompiler::versionLoadableOn("1", "1.1")); + + // The SDK version's runtime is 1.2, so a matcher authored against the + // older 1.0 and 1.1 schemas still compiles. + for(const char* older : {"1.0", "1.1", "1.2"}) + { + json d = sdpaDescriptor(); + d["sdk_version"] = older; + EXPECT_NO_THROW(umd::UmdCompiler::compile(d)) << older; + } +} + +TEST(TestUmdCompiler, VersionComparisonIsNumericNotLexicographic) +{ + // "1.10" is above "1.9" numerically but below it as text; a lexicographic + // comparison would order these backwards. + EXPECT_TRUE(umd::UmdCompiler::versionAtLeast("1.10", "1.9")); + EXPECT_FALSE(umd::UmdCompiler::versionAtLeast("1.9", "1.10")); + EXPECT_FALSE(umd::UmdCompiler::versionAtLeast("2.0", "10.0")); + EXPECT_TRUE(umd::UmdCompiler::versionAtLeast("10.0", "2.0")); + // equal is at the floor, not below it + EXPECT_TRUE(umd::UmdCompiler::versionAtLeast("1.0", "1.0")); + // an unparseable version is below every floor (fail closed) + EXPECT_FALSE(umd::UmdCompiler::versionAtLeast("1", "1.0")); + // the ceiling comparison is numeric too: 1.10 is newer than a 1.9 runtime + EXPECT_FALSE(umd::UmdCompiler::versionLoadableOn("1.10", "1.9")); + EXPECT_TRUE(umd::UmdCompiler::versionLoadableOn("1.9", "1.10")); +} + +// ---- Per-graph sdk_version floor (RFC 0017 §4) --------------------------- + +namespace +{ + +// Rebuild an SDPA graph with `min_required_engine_api_version` stamped, the +// field hipDNN computes from the optional features a graph uses (RFC 0008 +// override shapes at 1.1, RFC 0016 runtime pass-by-value at 1.2). The test-SDK +// fixture leaves it unset, so the floor is set explicitly here. +flatbuffers::FlatBufferBuilder buildSdpaGraphRequiringSchema(std::uint32_t major, + std::uint32_t minor) +{ + flatbuffers::FlatBufferBuilder builder; + const std::vector dims{2, 8, 16, 128}; + const std::vector strides = contiguousStrides(dims); + + std::vector<::flatbuffers::Offset> tensors; + for(const auto& [uid, name] : + {std::pair{1, "q"}, {2, "k"}, {3, "v"}, {4, "o"}}) + { + tensors.push_back(data::CreateTensorAttributesDirect( + builder, uid, name, data::DataType::BFLOAT16, &strides, &dims)); + } + + data::SdpaAttributesBuilder sb(builder); + sb.add_q_tensor_uid(1); + sb.add_k_tensor_uid(2); + sb.add_v_tensor_uid(3); + sb.add_o_tensor_uid(4); + const auto sdpa = sb.Finish(); + + std::vector<::flatbuffers::Offset> nodes; + nodes.push_back(data::CreateNodeDirect(builder, + "sdpa_fwd", + data::DataType::BFLOAT16, + data::NodeAttributes::SdpaAttributes, + sdpa.Union())); + + const data::EngineApiVersion required(major, minor, 0); + auto graph = data::CreateGraphDirect(builder, + "test", + data::DataType::FLOAT, + data::DataType::HALF, + data::DataType::BFLOAT16, + &tensors, + &nodes, + flatbuffers::nullopt, + /*is_override_shape_enabled=*/false, + &required); + builder.Finish(graph); + return builder; +} + +} // namespace + +TEST(TestUniversalGraphMatcher, DeclinesAMatcherBelowTheGraphsRequiredSchema) +{ + // RFC 0017 §4: a graph reports the schema version its own contents require, + // and a matcher declaring less is skipped instead of asked -- it would + // otherwise match on the fields it knows and silently ignore a field that + // changes what the graph means. The descriptor is otherwise a full match. + json d = sdpaDescriptor(); + d["sdk_version"] = "1.0"; + const umd::UniversalGraphMatcher m(d); + + auto baseline = buildSdpaGraphRequiringSchema(1, 0); + fbu::GraphWrapper const gBaseline(baseline.GetBufferPointer(), baseline.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, gBaseline).matched); + + // The graph sets an optional feature this matcher never accounted for. + auto newer = buildSdpaGraphRequiringSchema(1, 2); + fbu::GraphWrapper const gNewer(newer.GetBufferPointer(), newer.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, gNewer).matched); +} + +TEST(TestUniversalGraphMatcher, MatcherAtOrAboveTheGraphsRequiredSchemaRuns) +{ + // The same graph, matched by a descriptor that adopted the newer schema. + json d = sdpaDescriptor(); + d["sdk_version"] = "1.2"; + const umd::UniversalGraphMatcher m(d); + + auto newer = buildSdpaGraphRequiringSchema(1, 2); + fbu::GraphWrapper const gNewer(newer.GetBufferPointer(), newer.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, gNewer).matched); + + // A matcher above the floor still serves a graph that requires less. + auto baseline = buildSdpaGraphRequiringSchema(1, 0); + fbu::GraphWrapper const gBaseline(baseline.GetBufferPointer(), baseline.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, gBaseline).matched); +} + +TEST(TestUniversalGraphMatcher, AnUnstampedGraphReadsAsTheBaselineFloor) +{ + // A hand-built fixture or a graph written before the field existed carries + // no stamp; it reads as the 1.0 baseline rather than declining every + // matcher, mirroring the plugin SDK's null-tolerant accessor. + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, g).matched); +} + +// ---- present / not_present (RFC 0017 §5) --------------------------------- + +namespace +{ + +// The SDPA descriptor with its `.present` field gates on the three optional +// operands replaced by the `not_present` operator over the same list. +json sdpaNotPresentDescriptor() +{ + json d = sdpaDescriptor(); + json& crit = d["criteria"]["and"]; + // Drop the three trailing `{"!": "$x.present"}` field-form gates. + crit.erase(crit.size() - 1); + crit.erase(crit.size() - 1); + crit.erase(crit.size() - 1); + crit.push_back( + json::parse(R"({"not_present": ["$attn_mask", "$page_table_k", "$page_table_v"]})")); + return d; +} + +} // namespace + +TEST(TestUmdCompiler, AcceptsPresenceOperators) +{ + EXPECT_NO_THROW(umd::UmdCompiler::compile(sdpaNotPresentDescriptor())); + + // `present`/`not_present` are the general form: unlike the `.present` + // FIELD, they apply to a required operand too. + for(const char* crit : {R"({"present": ["$q"]})", + R"({"not_present": ["$attn_mask"]})", + R"({"present": ["$q", "$k", "$v"]})", + R"({"or": [{"not_present": ["$attn_mask"]}, {"present": ["$q"]}]})"}) + { + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(crit)); + EXPECT_NO_THROW(umd::UmdCompiler::compile(d)) << crit; + } +} + +TEST(TestUmdCompiler, RejectsPresenceOperatorOnNonReference) +{ + for(const char* bad : {R"({"present": [4]})", + R"({"present": ["BFLOAT16"]})", + R"({"not_present": [{"+": ["$q.rank", 1]}]})", + R"({"present": ["$q", 4]})", + R"({"present": []})"}) + { + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(bad)); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError) << bad; + } +} + +TEST(TestUmdCompiler, RejectsPresenceOperatorOnUnresolvedReference) +{ + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"present": ["$not_a_tensor"]})")); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestUniversalGraphMatcher, NotPresentOperatorMatchesWhenOptionalsAbsent) +{ + const umd::UniversalGraphMatcher m(sdpaNotPresentDescriptor()); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, g).matched); +} + +TEST(TestUniversalGraphMatcher, NotPresentOperatorDeclinesWhenAnyOptionalBound) +{ + // The n-ary form is an `and`-fold: one supplied operand out of the list + // decides the whole call. + const umd::UniversalGraphMatcher m(sdpaNotPresentDescriptor()); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16, /*withAttnMask=*/true); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, g).matched); +} + +TEST(TestUniversalGraphMatcher, PresenceOperatorsEvaluateRatherThanDecline) +{ + // The key distinction from a bare field read: a field read on an absent + // optional resolves null and declines, but `present`/`not_present` always + // produce a boolean, so a criterion built only from them still matches. + json d = sdpaDescriptor(); + json& crit = d["criteria"]["and"]; + crit.erase(crit.size() - 1); + crit.erase(crit.size() - 1); + crit.erase(crit.size() - 1); + // `$attn_mask` is absent; `present` on it is false, so `!` of it is true. + crit.push_back(json::parse(R"({"!": {"present": ["$attn_mask"]}})")); + // `$q` is a required operand and always bound. + crit.push_back(json::parse(R"({"present": ["$q", "$k", "$v", "$o"]})")); + + const umd::UniversalGraphMatcher m(d); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, g).matched); +} + +// ---- Absent optional operands neither pass nor fail (RFC 0017 §5) -------- + +namespace +{ + +// The SDPA descriptor with its three trailing `.present` field gates dropped +// and `extra` appended, so a single criterion can be exercised against a graph +// that supplies no optional operands. +json sdpaWithCriterion(const char* extra) +{ + json d = sdpaDescriptor(); + json& crit = d["criteria"]["and"]; + crit.erase(crit.size() - 1); + crit.erase(crit.size() - 1); + crit.erase(crit.size() - 1); + crit.push_back(json::parse(extra)); + return d; +} + +} // namespace + +TEST(TestUniversalGraphMatcher, FieldReadOnAnAbsentOptionalNeverAccepts) +{ + // RFC 0017 §5: "A dtype or layout check on an absent $bias neither passes + // nor fails, it simply does not run." The hazard is the negative and + // ordering forms: if an unresolved read coerced to false/0 instead of + // propagating, each of these would evaluate TRUE and the pack would accept + // a graph carrying an operand its kernel cannot serve. + for(const char* crit : {R"({"!": "$attn_mask.packed"})", + R"({"!=": ["$attn_mask.dtype", "BFLOAT16"]})", + R"({"<": ["$attn_mask.rank", 5]})", + R"({">=": ["$attn_mask.rank", 0]})", + R"({"==": ["$attn_mask.dtype", "$page_table_k.dtype"]})", + R"({"==": [{"+": ["$attn_mask.rank", 1]}, 1]})"}) + { + const umd::UniversalGraphMatcher m(sdpaWithCriterion(crit)); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, g).matched) << crit; + } +} + +TEST(TestUniversalGraphMatcher, AbsentOrPresentAndConstrainedAcceptsBothShapes) +{ + // RFC 0017 §5: the pair a pack writes when it serves an optional operand + // only in a particular form. The `or` must see past the unresolved second + // arm when the operand is absent, and must still enforce it when present. + const char* crit = R"({"or": [ + {"not_present": ["$attn_mask"]}, + {"and": [{"present": ["$attn_mask"]}, + {"==": ["$attn_mask.dtype", "BFLOAT16"]}]} + ]})"; + const umd::UniversalGraphMatcher m(sdpaWithCriterion(crit)); + + auto absent = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const gAbsent(absent.GetBufferPointer(), absent.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, gAbsent).matched); + + auto present = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16, /*withAttnMask=*/true); + fbu::GraphWrapper const gPresent(present.GetBufferPointer(), present.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, gPresent).matched); +} + +TEST(TestUniversalGraphMatcher, DefiniteFalseStillDeclinesBesideAnUnresolvedCheck) +{ + // An unresolved sibling must not rescue a criterion that definitely fails: + // `and` short-circuits on the definite false rather than going unresolved. + const char* crit = R"({"and": [{"==": ["$q.rank", 9]}, + {"==": ["$attn_mask.dtype", "BFLOAT16"]}]})"; + const umd::UniversalGraphMatcher m(sdpaWithCriterion(crit)); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, g).matched); +} + +TEST(TestUniversalGraphMatcher, PresentOperatorSeesABoundOptional) +{ + // Same descriptor, but the graph supplies `attn_mask`: `present` now reads + // true and the criterion declines, proving it tracks binding rather than + // returning a constant. + json d = sdpaDescriptor(); + json& crit = d["criteria"]["and"]; + crit.erase(crit.size() - 1); + crit.erase(crit.size() - 1); + crit.erase(crit.size() - 1); + crit.push_back(json::parse(R"({"!": {"present": ["$attn_mask"]}})")); + + const umd::UniversalGraphMatcher m(d); + auto bound = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16, /*withAttnMask=*/true); + fbu::GraphWrapper const gb(bound.GetBufferPointer(), bound.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, gb).matched); + + auto absent = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const ga(absent.GetBufferPointer(), absent.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, ga).matched); +} + +// ---- value_or_default with an expression fallback (RFC 0017 §5) ---------- + +TEST(TestUmdCompiler, ValueOrDefaultAcceptsFieldReferenceFallback) +{ + // "this field, else that one": both arms are Int, so the pair unifies. + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back( + json::parse(R"({"==": [{"value_or_default": ["$q.head_size", "$k.head_size"]}, 128]})")); + EXPECT_NO_THROW(umd::UmdCompiler::compile(d)); +} + +TEST(TestUmdCompiler, ValueOrDefaultAcceptsDynamicArm) +{ + // A `$kernel.*` arm is DYNAMIC, so the other arm supplies the result kind + // and the comparison against an Int still type-checks. + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back( + json::parse(R"({"==": [{"value_or_default": ["$kernel.tile_m", 64]}, 128]})")); + EXPECT_NO_THROW(umd::UmdCompiler::compile(d)); + json e = sdpaDescriptor(); + e["criteria"]["and"].push_back( + json::parse(R"({"==": [{"value_or_default": ["$q.head_size", "$kernel.tile_m"]}, 128]})")); + EXPECT_NO_THROW(umd::UmdCompiler::compile(e)); +} + +TEST(TestUmdCompiler, RejectsTypeIncompatibleValueOrDefaultArms) +{ + for(const char* bad : + {R"({"==": [{"value_or_default": ["$q.head_size", "BFLOAT16"]}, 128]})", + R"({"==": [{"value_or_default": ["$q.dtype", 4]}, "BFLOAT16"]})", + R"({"==": [{"value_or_default": ["$q.head_size", "$q.stride_order"]}, 128]})"}) + { + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(bad)); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError) << bad; + } +} + +TEST(TestUniversalGraphMatcher, ValueOrDefaultFallsBackToTheSecondField) +{ + // `$sdpa_fwd.dropout_probability` is an unset optional attribute, so the + // fallback expression is evaluated and its value is what the comparison + // sees. + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse( + R"({"==": [{"value_or_default": ["$sdpa_fwd.dropout_probability", "$q.head_size"]}, 128]})")); + const umd::UniversalGraphMatcher m(d); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, g).matched); +} + +// ---- $graph.is_override_shape_enabled (RFC 0017 §5) ---------------------- + +TEST(TestUmdCompiler, AcceptsGraphOverrideShapeFlag) +{ + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"!": "$graph.is_override_shape_enabled"})")); + umd::CompiledUmd c; + ASSERT_NO_THROW(c = umd::UmdCompiler::compile(d)); + EXPECT_TRUE(c.boundSymbols.count("graph.is_override_shape_enabled") == 1); +} + +TEST(TestUmdCompiler, RejectsGraphOverrideShapeFlagAgainstNonBool) +{ + // The flag is Bool; comparing it to an int is a static type error. + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back( + json::parse(R"({"==": ["$graph.is_override_shape_enabled", 1]})")); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); +} + +TEST(TestBindingContext, ResolvesGraphOverrideShapeFlag) +{ + // The descriptor key `allow_override_shape` is the MATCHER's opt-in; the + // `$graph.is_override_shape_enabled` token is the GRAPH's state. A matcher + // that opts in still sees the graph's flag. + json d = sdpaDescriptor(); + d["allow_override_shape"] = true; + const umd::UniversalGraphMatcher m(d); + + auto plain = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const gp(plain.GetBufferPointer(), plain.GetSize()); + const umd::MatchResult rp = m.match(K_DEVICE, gp); + ASSERT_TRUE(rp.matched); + EXPECT_TRUE(rp.bindings.get("$graph.is_override_shape_enabled").isBool()); + EXPECT_FALSE(rp.bindings.get("$graph.is_override_shape_enabled").asBool()); + + auto overriden = buildSdpaGraph({2, 8, 16, 128}, + data::DataType::BFLOAT16, + /*withAttnMask=*/false, + /*overrideShape=*/true); + fbu::GraphWrapper const go(overriden.GetBufferPointer(), overriden.GetSize()); + const umd::MatchResult ro = m.match(K_DEVICE, go); + ASSERT_TRUE(ro.matched); + EXPECT_TRUE(ro.bindings.get("$graph.is_override_shape_enabled").asBool()); +} + +TEST(TestUniversalGraphMatcher, DeclinesOnGraphOverrideShapeFlagCriterion) +{ + // An opted-in matcher can still refuse an override-shape graph by reading + // the graph's own flag. + json d = sdpaDescriptor(); + d["allow_override_shape"] = true; + d["criteria"]["and"].push_back(json::parse(R"({"!": "$graph.is_override_shape_enabled"})")); + const umd::UniversalGraphMatcher m(d); + + auto overriden = buildSdpaGraph({2, 8, 16, 128}, + data::DataType::BFLOAT16, + /*withAttnMask=*/false, + /*overrideShape=*/true); + fbu::GraphWrapper const go(overriden.GetBufferPointer(), overriden.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, go).matched); + + auto plain = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const gp(plain.GetBufferPointer(), plain.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, gp).matched); +} + +// ---- $q.is_runtime_pass_by_value / $q.value_f32 (RFC 0017 §5) ------------ + +namespace +{ + +// The SDPA descriptor with the optional `scale` operand bound, so the tensor +// carrying a compile-time value is reachable as `$scale`. +json sdpaScaleDescriptor() +{ + json d = sdpaDescriptor(); + d["nodes"][0]["operands"]["scale"] = "$scale?"; + return d; +} + +} // namespace + +TEST(TestUmdCompiler, AcceptsTensorValueFields) +{ + json d = sdpaScaleDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"!": "$scale.is_runtime_pass_by_value"})")); + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$scale.value_f32", 1.0]})")); + EXPECT_NO_THROW(umd::UmdCompiler::compile(d)); +} + +TEST(TestUmdCompiler, TensorValueFieldsAreTyped) +{ + // is_runtime_pass_by_value is Bool and value_f32 is Float; using either in + // the wrong domain is a static type error. + json d = sdpaScaleDescriptor(); + d["criteria"]["and"].push_back( + json::parse(R"({"==": ["$scale.is_runtime_pass_by_value", 1]})")); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError); + + json e = sdpaScaleDescriptor(); + e["criteria"]["and"].push_back(json::parse(R"({"==": ["$scale.value_f32", "BFLOAT16"]})")); + EXPECT_THROW(umd::UmdCompiler::compile(e), umd::UmdCompileError); + + json f = sdpaScaleDescriptor(); + f["criteria"]["and"].push_back(json::parse(R"({"<": ["$scale.value_f32", 2.0]})")); + EXPECT_NO_THROW(umd::UmdCompiler::compile(f)); +} + +TEST(TestUmdCompiler, RejectsShapeDimNameCollidingWithTensorValueFields) +{ + // A.10 §7: a `shape` short-hand may not name a dim after a reserved field. + for(const char* name : {"is_runtime_pass_by_value", "value_f32"}) + { + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back( + json{{"shape", json::array({"$o", json::array({"b", "h", "s", name})})}}); + EXPECT_THROW(umd::UmdCompiler::compile(d), umd::UmdCompileError) << name; + } +} + +TEST(TestBindingContext, ResolvesTensorValueFields) +{ + const umd::UniversalGraphMatcher m(sdpaScaleDescriptor()); + auto builder = buildSdpaGraph({2, 8, 16, 128}, + data::DataType::BFLOAT16, + /*withAttnMask=*/false, + /*overrideShape=*/false, + /*withScale=*/true); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + + const umd::MatchResult r = m.match(K_DEVICE, g); + ASSERT_TRUE(r.matched); + const umd::BindingContext& b = r.bindings; + + // The scale tensor carries a compile-time Float32 value of 1.0, baked into + // the graph rather than supplied per execution. + EXPECT_FALSE(b.get("$scale.is_runtime_pass_by_value").asBool()); + EXPECT_DOUBLE_EQ(b.get("$scale.value_f32").toNumber(), 1.0); + + // A regular data tensor carries neither. + EXPECT_FALSE(b.get("$q.is_runtime_pass_by_value").asBool()); + EXPECT_TRUE(b.get("$q.value_f32").isNull()); +} + +TEST(TestBindingContext, ValueF32DeclinesWithoutACompileTimeValue) +{ + // "present only when the tensor carries a compile-time value at all": the + // union is NONE, so the token reads null and a criterion over it declines + // rather than seeing a zero that would satisfy a comparison. + const umd::UniversalGraphMatcher m = makeSdpaMatcher(); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + + const umd::MatchResult r = m.match(K_DEVICE, g); + ASSERT_TRUE(r.matched); + EXPECT_TRUE(r.bindings.get("$q.value_f32").isNull()); + EXPECT_FALSE(r.bindings.get("$q.value_f32").isNumber()); +} + +TEST(TestUniversalGraphMatcher, DeclinesOnValueF32OfAValuelessTensor) +{ + // The fail-closed consequence: a criterion reading value_f32 on a tensor + // with no compile-time value declines the whole match. + json d = sdpaDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$q.value_f32", 0.0]})")); + const umd::UniversalGraphMatcher m(d); + auto builder = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, g).matched); +} + +TEST(TestUniversalGraphMatcher, MatchesOnACompileTimeScaleValue) +{ + json d = sdpaScaleDescriptor(); + d["criteria"]["and"].push_back(json::parse(R"({"==": ["$scale.value_f32", 1.0]})")); + const umd::UniversalGraphMatcher m(d); + + auto withScale = buildSdpaGraph({2, 8, 16, 128}, + data::DataType::BFLOAT16, + /*withAttnMask=*/false, + /*overrideShape=*/false, + /*withScale=*/true); + fbu::GraphWrapper const gs(withScale.GetBufferPointer(), withScale.GetSize()); + EXPECT_TRUE(m.match(K_DEVICE, gs).matched); + + // No scale operand at all -> the field read on an absent optional declines. + auto noScale = buildSdpaGraph({2, 8, 16, 128}, data::DataType::BFLOAT16); + fbu::GraphWrapper const gn(noScale.GetBufferPointer(), noScale.GetSize()); + EXPECT_FALSE(m.match(K_DEVICE, gn).matched); +} + +// ---- value_f32 union coercion (RFC 0017 §5) ------------------------------ + +namespace +{ + +// A single-node SDPA graph whose optional `scale` operand carries `value`, the +// tagged union arm under test, with the tensor's own data_type. The test-SDK +// fixture only ever builds the Float32 arm, so the other seven (and the +// data_type-dispatched Float8 decode) are unreachable through it. +template +flatbuffers::FlatBufferBuilder buildSdpaGraphWithScaleValue(data::DataType scaleType, + AddValue&& addValue) +{ + flatbuffers::FlatBufferBuilder builder; + const std::vector dims{2, 8, 16, 128}; + const std::vector strides = contiguousStrides(dims); + const std::vector scalarDims{1}; + const std::vector scalarStrides{1}; + + std::vector<::flatbuffers::Offset> tensors; + for(const auto& [uid, name] : + {std::pair{1, "q"}, {2, "k"}, {3, "v"}, {4, "o"}}) + { + tensors.push_back(data::CreateTensorAttributesDirect( + builder, uid, name, data::DataType::BFLOAT16, &strides, &dims)); + } + + // addValue writes the union arm and returns its (type, offset) pair. + const auto [valueType, valueOffset] = addValue(builder); + const auto scaleName = builder.CreateString("scale"); + const auto scaleDims = builder.CreateVector(scalarDims); + const auto scaleStrides = builder.CreateVector(scalarStrides); + data::TensorAttributesBuilder tb(builder); + tb.add_uid(5); + tb.add_name(scaleName); + tb.add_data_type(scaleType); + tb.add_dims(scaleDims); + tb.add_strides(scaleStrides); + tb.add_value_type(valueType); + tb.add_value(valueOffset); + tensors.push_back(tb.Finish()); + + data::SdpaAttributesBuilder sb(builder); + sb.add_q_tensor_uid(1); + sb.add_k_tensor_uid(2); + sb.add_v_tensor_uid(3); + sb.add_o_tensor_uid(4); + sb.add_scale_tensor_uid(5); + const auto sdpa = sb.Finish(); + + std::vector<::flatbuffers::Offset> nodes; + nodes.push_back(data::CreateNodeDirect(builder, + "sdpa_fwd", + data::DataType::BFLOAT16, + data::NodeAttributes::SdpaAttributes, + sdpa.Union())); + + auto graph = data::CreateGraphDirect(builder, + "test", + data::DataType::FLOAT, + data::DataType::HALF, + data::DataType::BFLOAT16, + &tensors, + &nodes, + flatbuffers::nullopt, + /*is_override_shape_enabled=*/false); + builder.Finish(graph); + return builder; +} + +// Resolve `$scale.value_f32` for a graph carrying one union arm. +jlogic::Value scaleValueF32(flatbuffers::FlatBufferBuilder& builder) +{ + const umd::UniversalGraphMatcher m(sdpaScaleDescriptor()); + fbu::GraphWrapper const g(builder.GetBufferPointer(), builder.GetSize()); + const umd::MatchResult r = m.match(K_DEVICE, g); + EXPECT_TRUE(r.matched); + return r.bindings.get("$scale.value_f32"); +} + +} // namespace + +TEST(TestBindingContext, ValueF32CoercesEveryUnionArm) +{ + // RFC 0017 §5: the schema layer "coerces whichever arm is set to f32" and + // publishes one typed token. Every arm must produce a number, not null. + const auto f32 = scaleValueF32(*std::make_unique( + buildSdpaGraphWithScaleValue(data::DataType::FLOAT, [](auto& b) { + return std::pair{ + data::TensorValue::Float32Value, + ::flatbuffers::Offset(b.CreateStruct(data::Float32Value(2.5F)).o)}; + }))); + EXPECT_DOUBLE_EQ(f32.toNumber(), 2.5); + + const auto f64 = scaleValueF32(*std::make_unique( + buildSdpaGraphWithScaleValue(data::DataType::DOUBLE, [](auto& b) { + return std::pair{ + data::TensorValue::Float64Value, + ::flatbuffers::Offset(b.CreateStruct(data::Float64Value(0.25)).o)}; + }))); + EXPECT_DOUBLE_EQ(f64.toNumber(), 0.25); + + const auto i32 = scaleValueF32(*std::make_unique( + buildSdpaGraphWithScaleValue(data::DataType::INT32, [](auto& b) { + return std::pair{data::TensorValue::Int32Value, + ::flatbuffers::Offset(b.CreateStruct(data::Int32Value(-7)).o)}; + }))); + EXPECT_DOUBLE_EQ(i32.toNumber(), -7.0); + + const auto i64 = scaleValueF32(*std::make_unique( + buildSdpaGraphWithScaleValue(data::DataType::INT64, [](auto& b) { + return std::pair{data::TensorValue::Int64Value, + ::flatbuffers::Offset(b.CreateStruct(data::Int64Value(42)).o)}; + }))); + EXPECT_DOUBLE_EQ(i64.toNumber(), 42.0); + + const auto boolean = scaleValueF32(*std::make_unique( + buildSdpaGraphWithScaleValue(data::DataType::BOOLEAN, [](auto& b) { + return std::pair{data::TensorValue::BoolValue, + ::flatbuffers::Offset(b.CreateStruct(data::BoolValue(true)).o)}; + }))); + EXPECT_DOUBLE_EQ(boolean.toNumber(), 1.0); +} + +TEST(TestBindingContext, ValueF32DecodesFloat8ThroughTheTensorDataType) +{ + // Float8Value stores raw bits; the tensor's data_type says which 8-bit + // format they encode. The bits 0x38 are 1.0 in E4M3 (exponent bias 7) but + // 0.5 in E5M2 (bias 15), so decoding through the wrong format -- or not + // decoding at all -- is directly observable. + const auto e4m3 = scaleValueF32(*std::make_unique( + buildSdpaGraphWithScaleValue(data::DataType::FP8_E4M3, [](auto& b) { + return std::pair{ + data::TensorValue::Float8Value, + ::flatbuffers::Offset(b.CreateStruct(data::Float8Value(0x38)).o)}; + }))); + EXPECT_DOUBLE_EQ(e4m3.toNumber(), 1.0); + + const auto e5m2 = scaleValueF32(*std::make_unique( + buildSdpaGraphWithScaleValue(data::DataType::FP8_E5M2, [](auto& b) { + return std::pair{ + data::TensorValue::Float8Value, + ::flatbuffers::Offset(b.CreateStruct(data::Float8Value(0x38)).o)}; + }))); + EXPECT_DOUBLE_EQ(e5m2.toNumber(), 0.5); + + // E8M0 is an 8-bit format too: bits 127 encode the 2^0 scale. It must + // decode rather than fall through to the fail-closed null. + const auto e8m0 = scaleValueF32(*std::make_unique( + buildSdpaGraphWithScaleValue(data::DataType::FP8_E8M0, [](auto& b) { + return std::pair{data::TensorValue::Float8Value, + ::flatbuffers::Offset(b.CreateStruct(data::Float8Value(127)).o)}; + }))); + ASSERT_FALSE(e8m0.isNull()); + EXPECT_DOUBLE_EQ(e8m0.toNumber(), 1.0); + + // INT8 raw bits are signed: 0xFF is -1, not 255. + const auto i8 = scaleValueF32(*std::make_unique( + buildSdpaGraphWithScaleValue(data::DataType::INT8, [](auto& b) { + return std::pair{ + data::TensorValue::Float8Value, + ::flatbuffers::Offset(b.CreateStruct(data::Float8Value(0xFF)).o)}; + }))); + EXPECT_DOUBLE_EQ(i8.toNumber(), -1.0); + + // UINT8 keeps the unsigned reading of the same bits. + const auto u8 = scaleValueF32(*std::make_unique( + buildSdpaGraphWithScaleValue(data::DataType::UINT8, [](auto& b) { + return std::pair{ + data::TensorValue::Float8Value, + ::flatbuffers::Offset(b.CreateStruct(data::Float8Value(0xFF)).o)}; + }))); + EXPECT_DOUBLE_EQ(u8.toNumber(), 255.0); + + // A pairing that names no 8-bit format resolves to null rather than + // reinterpreting the bits. + const auto mismatched = scaleValueF32(*std::make_unique( + buildSdpaGraphWithScaleValue(data::DataType::FLOAT, [](auto& b) { + return std::pair{ + data::TensorValue::Float8Value, + ::flatbuffers::Offset(b.CreateStruct(data::Float8Value(0x38)).o)}; + }))); + EXPECT_TRUE(mismatched.isNull()); +} diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/asm_sdpa_engine/AsmSdpaConfigHelpers.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/asm_sdpa_engine/AsmSdpaConfigHelpers.cpp index 5cc90ac82957..9d8395d48275 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/asm_sdpa_engine/AsmSdpaConfigHelpers.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/asm_sdpa_engine/AsmSdpaConfigHelpers.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: MIT #include "AsmSdpaConfigHelpers.hpp" -#include "../../engines/asm_sdpa_engine/plans/SdpaPlanUtils.hpp" +#include "engines/asm_sdpa_engine/plans/SdpaPlanUtils.hpp" #include "hip_kernel_provider_common/SdpaConfigConstants.hpp" #include "hip_kernel_provider_common/SdpaConfigEnumerations.hpp" diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/CMakeLists.txt b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/CMakeLists.txt new file mode 100644 index 000000000000..f96a93370092 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +target_sources(hip_kernel_provider_tests PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/TestHandleDeviceResolver.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/TestKernelIngestorEngine.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/packs/TestPointwiseAddDispatchHandler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/packs/TestPointwiseAddMatchers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/packs/TestPointwiseAddPack.cpp +) diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/TestHandleDeviceResolver.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/TestHandleDeviceResolver.cpp new file mode 100644 index 000000000000..46bf5568c7e7 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/TestHandleDeviceResolver.cpp @@ -0,0 +1,207 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include + +#include + +#include +#include + +#include "core/Handle.hpp" +#include "engines/kernel_ingestor_engine/HandleDeviceResolver.hpp" + +/** + * @file TestHandleDeviceResolver.cpp + * @brief HandleDeviceResolver's device-id resolution and its device-properties cache. + * + * The resolver is a process-lifetime static (see KernelIngestorEngine.cpp's + * deviceResolver()), so a stale or invalidated cache entry is a process-lifetime bug, + * not one scoped to a single engine or container. + * + * deviceId() has four paths; the fourth (no usable HIP context) has no test hook here + * and is documented but not exercised. + */ +namespace +{ + +using hip_kernel_provider::kernel_ingestor_engine::HandleDeviceResolver; + +/// Answers every property query successfully, so a test can grow the cache past a +/// rehash without needing many physical devices. +class FakeQueryResolver : public HandleDeviceResolver +{ +public: + /// warpSize is set from the id so a caller can tell one faked entry from another. + hipError_t queryDeviceProperties(hipDeviceProp_t* properties, + hipdnn_plugin_sdk::ingestor::DeviceId deviceId) const override + { + *properties = hipDeviceProp_t{}; + properties->warpSize = static_cast(deviceId); + return hipSuccess; + } +}; + +/// Fails every property query, to drive the refusal path. +class FailingQueryResolver : public HandleDeviceResolver +{ +public: + hipError_t + queryDeviceProperties(hipDeviceProp_t* /*properties*/, + hipdnn_plugin_sdk::ingestor::DeviceId /*deviceId*/) const override + { + return hipErrorInvalidDevice; + } +}; + +// --------------------------------------------------------------------------- +// deviceId() +// --------------------------------------------------------------------------- + +TEST(TestHandleDeviceResolver, ResolvesTheCurrentDeviceForANullStream) +{ + SKIP_IF_NO_DEVICES(); + + // A null stream means the default stream, which belongs to the current device. + const HandleDeviceResolver resolver; + Handle handle; + handle.setStream(nullptr); + + int currentDevice = -1; + ASSERT_EQ(hipGetDevice(¤tDevice), hipSuccess); + + EXPECT_EQ(resolver.deviceId(handle), currentDevice); +} + +TEST(TestHandleDeviceResolver, ResolvesTheStreamsOwnDeviceWhenItDiffersFromCurrent) +{ + SKIP_IF_NO_DEVICES(); + + // Resolves via hipStreamGetDevice, not whichever device is current on this thread. + hipStream_t stream = nullptr; + ASSERT_EQ(hipStreamCreate(&stream), hipSuccess); + + int streamDevice = -1; + ASSERT_EQ(hipStreamGetDevice(stream, &streamDevice), hipSuccess); + + const HandleDeviceResolver resolver; + Handle handle; + handle.setStream(stream); + + EXPECT_EQ(resolver.deviceId(handle), streamDevice); + + static_cast(hipStreamDestroy(stream)); +} + +TEST(TestHandleDeviceResolver, FallsThroughToTheCurrentDeviceWhenTheStreamCannotBeResolved) +{ + SKIP_IF_NO_DEVICES(); + + // A stream hipStreamGetDevice cannot resolve (already destroyed) falls through like + // a null stream. Destroying it leaves a pending HIP error, cleared below so it does + // not fail an unrelated test. + hipStream_t stream = nullptr; + ASSERT_EQ(hipStreamCreate(&stream), hipSuccess); + ASSERT_EQ(hipStreamDestroy(stream), hipSuccess); + + int currentDevice = -1; + ASSERT_EQ(hipGetDevice(¤tDevice), hipSuccess); + + const HandleDeviceResolver resolver; + Handle handle; + handle.setStream(stream); + + EXPECT_EQ(resolver.deviceId(handle), currentDevice); + + static_cast(hipGetLastError()); + static_cast(hipExtGetLastError()); +} + +// --------------------------------------------------------------------------- +// deviceProperties(): cache hit vs miss, and the growth-safety invariant +// --------------------------------------------------------------------------- + +TEST(TestHandleDeviceResolver, CachesDevicePropertiesAcrossCalls) +{ + SKIP_IF_NO_DEVICES(); + + const HandleDeviceResolver resolver; + + // A hit for the same device returns the same address: deviceProperties() promises a + // reference stable for the resolver's lifetime. + const auto& first = resolver.deviceProperties(0); + const auto& second = resolver.deviceProperties(0); + + EXPECT_EQ(&first, &second); + EXPECT_EQ(first.warpSize, second.warpSize); +} + +TEST(TestHandleDeviceResolver, ReferencesStayValidAcrossCacheGrowth) +{ + // Verifies HandleDeviceResolver's stated invariant: references stay valid across + // cache growth (std::unordered_map keeps node handles stable across rehash). + // + // Uses FakeQueryResolver rather than invalid device ids, since deviceProperties() + // does not cache failed queries. CPU-only. + const FakeQueryResolver resolver; + + const auto& firstInserted = resolver.deviceProperties(1000); + + // Enough insertions to force a rehash of the default bucket count. + for(int deviceId = 1001; deviceId < 1064; ++deviceId) + { + static_cast(resolver.deviceProperties(deviceId)); + } + + const auto& sameEntryAfterGrowth = resolver.deviceProperties(1000); + EXPECT_EQ(&firstInserted, &sameEntryAfterGrowth); + // Value must match too -- a surviving address alone is not sufficient proof. + EXPECT_EQ(sameEntryAfterGrowth.warpSize, 1000); +} + +TEST(TestHandleDeviceResolver, RefusesAndDoesNotCacheAFailedPropertyQuery) +{ + // A zeroed hipDeviceProp_t is not cached: this cache is never invalidated, so a + // false answer would persist for the process's life. + const FailingQueryResolver resolver; + + EXPECT_THROW(static_cast(resolver.deviceProperties(7)), + hipdnn_plugin_sdk::HipdnnPluginException); + // Still refused on the second ask -- the failure was not remembered as an answer. + EXPECT_THROW(static_cast(resolver.deviceProperties(7)), + hipdnn_plugin_sdk::HipdnnPluginException); +} + +TEST(TestHandleDeviceResolver, ConcurrentDevicePropertyLookupsAreSafe) +{ + // The cache is mutex-guarded (see class doc); many threads querying the same small + // id set maximize a real race's chance to corrupt the map. + const FakeQueryResolver resolver; + + std::vector threads; + threads.reserve(8); + for(int t = 0; t < 8; ++t) + { + threads.emplace_back([&resolver, t]() { + for(int i = 0; i < 200; ++i) + { + static_cast(resolver.deviceProperties((t % 4) + 2000)); + } + }); + } + for(auto& thread : threads) + { + thread.join(); + } + + static_cast(hipGetLastError()); + static_cast(hipExtGetLastError()); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/TestKernelIngestorEngine.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/TestKernelIngestorEngine.cpp new file mode 100644 index 000000000000..1a3a1b3ca8b4 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/TestKernelIngestorEngine.cpp @@ -0,0 +1,236 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include + +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include "core/Container.hpp" +#include "core/Context.hpp" +#include "core/Handle.hpp" +#include "engines/kernel_ingestor_engine/KernelIngestorEngine.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddPack.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddSymbols.hpp" +#include "tests/engines/kernel_ingestor_engine/packs/PointwiseAddTestGraphs.hpp" + +/** + * @file TestKernelIngestorEngine.cpp + * @brief Tests registerNativeIngestorSymbols() and makePointwiseAddEngine(), which + * replace the deleted PointwiseAddEngine forwarding wrapper. GenericEngine + * itself is the SDK's type and covered by the SDK's suite; this file only + * verifies the factory wires it up correctly, reached through Container and + * EngineManager since makePointwiseAddEngine() takes no injectable seams. + */ +namespace +{ + +using namespace hip_kernel_provider; +using namespace hip_kernel_provider::kernel_ingestor_engine; +using namespace hip_kernel_provider::kernel_ingestor_engine::testing; +using hip_kernel_provider::core::Container; +using hipdnn_flatbuffers_sdk::flatbuffer_utilities::GraphWrapper; +using hipdnn_test_sdk::utilities::MockEngineConfig; + +/// Wraps a built graph buffer as the frontend hands one to an engine. +GraphWrapper wrap(const flatbuffers::FlatBufferBuilder& builder) +{ + return GraphWrapper(builder.GetBufferPointer(), builder.GetSize()); +} + +/// Stubs a config naming this pack's engine, keyed the way getMaxWorkspaceSize() and +/// initializeExecutionContext() both look it up: EngineManager::getEngine(engineId()). +void stubAsThisEnginesConfig(MockEngineConfig& config) +{ + EXPECT_CALL(config, isValid()).WillRepeatedly(::testing::Return(false)); + EXPECT_CALL(config, engineId()).WillRepeatedly(::testing::Return(pointwiseAddEngineId())); +} + +// --------------------------------------------------------------------------- +// registerNativeIngestorSymbols(): idempotent across repeated calls +// --------------------------------------------------------------------------- + +TEST(TestKernelIngestorEngine, RegisterNativeIngestorSymbolsIsIdempotentAcrossRepeatedCalls) +{ + // once_flag-guarded; Container's constructor calls this on every Container built + // (may be many, via SharedContainerManager's weak_ptr), so repeats must be a no-op. + EXPECT_NO_THROW(registerNativeIngestorSymbols()); + EXPECT_NO_THROW(registerNativeIngestorSymbols()); + EXPECT_NO_THROW(registerNativeIngestorSymbols()); +} + +TEST(TestKernelIngestorEngine, RegistrationFailureReportsTheSameCauseOnRetryAfterRollback) +{ + using hipdnn_plugin_sdk::ingestor::DispatchRegistry; + + // Unregisters to drive controlled attempts; every other test relies on these + // symbols for matching and dispatch, so always restore. + unregisterPointwiseAddMatchers(); + DispatchRegistry::unregisterSymbol(std::string(DISPATCH_SYMBOL)); + struct RestoreGuard + { + ~RestoreGuard() + { + registerNativeIngestorSymbolsOnce(); + } + } const restoreGuard; + + // Occupies the dispatch symbol so registration succeeds on every matcher and fails + // only on dispatch -- the partial-failure shape rollback exists for. + DispatchRegistry::registerSymbol(std::string(DISPATCH_SYMBOL), nullptr); + + const auto attempt = [] { + try + { + registerNativeIngestorSymbolsOnce(); + } + catch(const std::runtime_error& e) + { + return std::string(e.what()); + } + ADD_FAILURE() << "expected a duplicate-symbol throw"; + return std::string(); + }; + + const auto firstMessage = attempt(); + const auto secondMessage = attempt(); + + // Without rollback, the first attempt's matcher symbols stay registered and the + // second attempt fails on a different, misleading cause. + EXPECT_EQ(firstMessage, secondMessage); + + DispatchRegistry::unregisterSymbol(std::string(DISPATCH_SYMBOL)); +} + +// --------------------------------------------------------------------------- +// makePointwiseAddEngine(): a working GenericEngine, reached through Container +// --------------------------------------------------------------------------- + +TEST(TestKernelIngestorEngine, MakePointwiseAddEngineIsReachableWithTheDescriptorEngineId) +{ + Container container; + auto& engineManager = container.getEngineManager(); + + // pointwiseAddEngineId() registers the engine's name on first call; + // getAllEngineIds() proves the factory installed it, not just compiled. + const auto allEngineIds = engineManager.getAllEngineIds(); + EXPECT_NE(std::find(allEngineIds.begin(), allEngineIds.end(), pointwiseAddEngineId()), + allEngineIds.end()); +} + +TEST(TestKernelIngestorEngine, IsApplicableAcceptsAGraphThisPacksMatchersAccept) +{ + // Matchers decline outright with no device resolved, so an accept is only + // meaningful where there is one. + SKIP_IF_NO_DEVICES(); + + Container container; + auto& engineManager = container.getEngineManager(); + Handle handle; + + const auto graph = buildPointwiseGraph(); + const auto applicable = engineManager.getApplicableEngineIds(handle, wrap(graph)); + + EXPECT_NE(std::find(applicable.begin(), applicable.end(), pointwiseAddEngineId()), + applicable.end()); +} + +TEST(TestKernelIngestorEngine, GetEngineDetailsReportsTheBlockSizeKnob) +{ + // The reported knob set comes from the matched catalog, empty without a device. + SKIP_IF_NO_DEVICES(); + + Container container; + auto& engineManager = container.getEngineManager(); + Handle handle; + + const auto graph = buildPointwiseGraph(); + hipdnnPluginConstData_t details{}; + engineManager.getEngineDetails(handle, wrap(graph), pointwiseAddEngineId(), details); + + const hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineDetailsWrapper wrapper(details.ptr, + details.size); + ASSERT_EQ(wrapper.knobCount(), 1U); + EXPECT_EQ(wrapper.getKnobByName("block_size").knobId(), "block_size"); +} + +TEST(TestKernelIngestorEngine, GetMaxWorkspaceSizeReportsTheLargerBlocksRequirement) +{ + // Sizes a workspace from the kernels the graph matched -- none, without a device. + SKIP_IF_NO_DEVICES(); + + Container container; + auto& engineManager = container.getEngineManager(); + const Handle handle; + + const auto graph = buildPointwiseGraph(); + MockEngineConfig engineConfig; + stubAsThisEnginesConfig(engineConfig); + + // The catalog's two surviving FLOAT kernels report 0 and 1024 bytes; the answer is + // a max across admitted kernels, proving the query reaches the real catalog. + const auto workspaceSize = engineManager.getMaxWorkspaceSize(handle, wrap(graph), engineConfig); + EXPECT_EQ(workspaceSize, 1024U); +} + +TEST(TestKernelIngestorEngine, InitializeExecutionContextBuildsAPlanForTheTopRankedKernel) +{ + // Builds a plan by compiling the selected kernel through hiprtc, so unlike the + // tests above this needs a device. + SKIP_IF_NO_DEVICES(); + + Container container; + auto& engineManager = container.getEngineManager(); + const Handle handle; + + const auto graph = buildPointwiseGraph(); + MockEngineConfig engineConfig; + stubAsThisEnginesConfig(engineConfig); + + Context context; + ASSERT_NO_THROW( + engineManager.initializeExecutionContext(handle, wrap(graph), engineConfig, context)); + ASSERT_TRUE(context.hasValidPlan()); + + // pointwiseAddScore ranks on block size and both FLOAT kernels are admitted, so 256 + // is the defined winner -- hasValidPlan() alone would not catch a wrong selection. + const auto& plan + = dynamic_cast&>(context.plan()); + EXPECT_EQ(plan.kernel().getIntMetadata(std::string(BLOCK_SIZE_FIELD)), 256); +} + +// --------------------------------------------------------------------------- +// Unhappy path: a graph none of this pack's kernels serve +// --------------------------------------------------------------------------- + +TEST(TestKernelIngestorEngine, DeclinesAGraphThisPacksMatchersRefuse) +{ + Container container; + auto& engineManager = container.getEngineManager(); + Handle handle; + + // A multiplication is refused at the graph-scoped matcher (also covered directly by + // TestPointwiseAddGraphMatcherRefusal), exercised here through the full engine. + const auto graph + = buildPointwiseGraph(hipdnn_flatbuffers_sdk::data_objects::PointwiseMode::MUL); + const auto applicable = engineManager.getApplicableEngineIds(handle, wrap(graph)); + + EXPECT_EQ(std::find(applicable.begin(), applicable.end(), pointwiseAddEngineId()), + applicable.end()); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/PointwiseAddTestGraphs.hpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/PointwiseAddTestGraphs.hpp new file mode 100644 index 000000000000..5015afd6de62 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/PointwiseAddTestGraphs.hpp @@ -0,0 +1,271 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "engines/kernel_ingestor_engine/packs/PointwiseAddSymbols.hpp" + +namespace hip_kernel_provider::kernel_ingestor_engine::testing +{ + +/// Tensor uids the builders below use, in argument order. +constexpr int64_t INPUT_A_UID = 1; +constexpr int64_t INPUT_B_UID = 2; +constexpr int64_t OUTPUT_UID = 3; +/// A real third operand, added when `includeThirdOperand` is set. +constexpr int64_t INPUT_C_UID = 4; +/// Uid named by `in_1_tensor_uid` when `danglingInputBUid` is not overridden; guaranteed +/// absent from every graph this file builds. +constexpr int64_t DEFAULT_DANGLING_UID = 999; + +/// A fixed, warp-64 device, for CPU-only matcher tests that never compile or launch. +inline hipDeviceProp_t testDeviceProperties() +{ + hipDeviceProp_t properties{}; + properties.warpSize = 64; + return properties; +} + +/// The real current device's properties, queried once; zeroed if no device is current. +inline hipDeviceProp_t currentDeviceProperties() +{ + hipDeviceProp_t properties{}; + int deviceId = 0; + if(hipGetDevice(&deviceId) == hipSuccess) + { + static_cast(hipGetDeviceProperties(&properties, deviceId)); + } + return properties; +} + +/** + * @brief Builds a single-node binary-pointwise-add graph, parameterized on everything + * this pack's matchers gate: operation, dtype (uniform and per-operand), shape, + * arity, and operand validity. + * + * @param inputBDataType Overrides input B's dtype away from @p dataType. + * @param includeThirdOperand Adds a real third tensor (`INPUT_C_UID`) and sets + * `in_2_tensor_uid`, producing a ternary op. + * @param danglingInputBUid When set, `in_1_tensor_uid` names this value instead of + * `INPUT_B_UID`, and no tensor is inserted for it. + * @param inputAVirtual Marks input A virtual. + * @param inputAIsRuntimePassByValue Marks input A runtime-pass-by-value. + */ +inline flatbuffers::FlatBufferBuilder buildPointwiseGraph( + hipdnn_flatbuffers_sdk::data_objects::PointwiseMode operation + = hipdnn_flatbuffers_sdk::data_objects::PointwiseMode::ADD, + hipdnn_flatbuffers_sdk::data_objects::DataType dataType + = hipdnn_flatbuffers_sdk::data_objects::DataType::FLOAT, + const std::vector& dims = {1, 1, 1, 1}, + std::optional graphId = std::nullopt, + bool binary = true, + const std::optional>& explicitStrides = std::nullopt, + std::optional inputBDataType = std::nullopt, + bool includeThirdOperand = false, + std::optional danglingInputBUid = std::nullopt, + bool inputAVirtual = false, + bool inputAIsRuntimePassByValue = false) +{ + namespace data_objects = hipdnn_flatbuffers_sdk::data_objects; + + flatbuffers::FlatBufferBuilder builder; + // Packed strides by default; an explicit set describes a view into a larger buffer. + const std::vector strides + = explicitStrides.has_value() ? *explicitStrides : std::vector(dims.size(), 1); + const auto resolvedInputBDataType = inputBDataType.value_or(dataType); + + std::vector> tensors; + tensors.push_back(data_objects::CreateTensorAttributesDirect(builder, + INPUT_A_UID, + nullptr, + dataType, + &strides, + &dims, + inputAVirtual, + data_objects::TensorValue::NONE, + 0, + inputAIsRuntimePassByValue)); + tensors.push_back(data_objects::CreateTensorAttributesDirect( + builder, INPUT_B_UID, nullptr, resolvedInputBDataType, &strides, &dims, false)); + tensors.push_back(data_objects::CreateTensorAttributesDirect( + builder, OUTPUT_UID, nullptr, dataType, &strides, &dims, false)); + if(includeThirdOperand) + { + tensors.push_back(data_objects::CreateTensorAttributesDirect( + builder, INPUT_C_UID, nullptr, dataType, &strides, &dims, false)); + } + + data_objects::PointwiseAttributesBuilder attributesBuilder(builder); + attributesBuilder.add_operation(operation); + attributesBuilder.add_in_0_tensor_uid(INPUT_A_UID); + if(binary) + { + attributesBuilder.add_in_1_tensor_uid(danglingInputBUid.value_or(INPUT_B_UID)); + } + if(includeThirdOperand) + { + attributesBuilder.add_in_2_tensor_uid(INPUT_C_UID); + } + attributesBuilder.add_out_0_tensor_uid(OUTPUT_UID); + auto attributes = attributesBuilder.Finish(); + + std::vector> nodes; + nodes.push_back( + data_objects::CreateNodeDirect(builder, + "pointwise", + dataType, + data_objects::NodeAttributes::PointwiseAttributes, + attributes.Union())); + + auto name = builder.CreateString("pointwise_add_test"); + auto tensorsVector = builder.CreateVector(tensors); + auto nodesVector = builder.CreateVector(nodes); + + data_objects::GraphBuilder graphBuilder(builder); + graphBuilder.add_name(name); + graphBuilder.add_tensors(tensorsVector); + graphBuilder.add_nodes(nodesVector); + + // Held for the duration of the GraphBuilder: add_id stores a pointer to it. + data_objects::Uuid uuid{}; + if(graphId.has_value()) + { + uuid = hipdnn_flatbuffers_sdk::utilities::toFlatbufferUuid(*graphId); + graphBuilder.add_id(&uuid); + } + builder.Finish(graphBuilder.Finish()); + + return builder; +} + +/// @brief A graph with two pointwise nodes, which no prebuilt single-op kernel serves. +inline flatbuffers::FlatBufferBuilder buildTwoNodePointwiseGraph() +{ + namespace data_objects = hipdnn_flatbuffers_sdk::data_objects; + + flatbuffers::FlatBufferBuilder builder; + const std::vector dims = {1, 1, 1, 1}; + const std::vector strides = {1, 1, 1, 1}; + constexpr int64_t INTERMEDIATE_UID = 4; + + std::vector> tensors; + for(const auto uid : {INPUT_A_UID, INPUT_B_UID, OUTPUT_UID, INTERMEDIATE_UID}) + { + tensors.push_back(data_objects::CreateTensorAttributesDirect(builder, + uid, + nullptr, + data_objects::DataType::FLOAT, + &strides, + &dims, + uid == INTERMEDIATE_UID)); + } + + std::vector> nodes; + for(const auto& [in0, in1, out] : {std::tuple{INPUT_A_UID, INPUT_B_UID, INTERMEDIATE_UID}, + std::tuple{INTERMEDIATE_UID, INPUT_B_UID, OUTPUT_UID}}) + { + data_objects::PointwiseAttributesBuilder attributesBuilder(builder); + attributesBuilder.add_operation(data_objects::PointwiseMode::ADD); + attributesBuilder.add_in_0_tensor_uid(in0); + attributesBuilder.add_in_1_tensor_uid(in1); + attributesBuilder.add_out_0_tensor_uid(out); + auto attributes = attributesBuilder.Finish(); + + nodes.push_back( + data_objects::CreateNodeDirect(builder, + "pointwise", + data_objects::DataType::FLOAT, + data_objects::NodeAttributes::PointwiseAttributes, + attributes.Union())); + } + + builder.Finish(data_objects::CreateGraphDirect(builder, + "two_node_pointwise", + data_objects::DataType::FLOAT, + data_objects::DataType::FLOAT, + data_objects::DataType::FLOAT, + &tensors, + &nodes)); + + return builder; +} + +/// @brief A distinct graph identity, so cache-keyed tests do not collide. +inline hipdnn_flatbuffers_sdk::utilities::UuidBytes makeGraphId(uint8_t seed) +{ + hipdnn_flatbuffers_sdk::utilities::UuidBytes id{}; + id.fill(seed); + return id; +} + +/** + * @brief Wraps a built graph buffer so a test reads it the way an engine does. + * + * Parameterized on the device-properties source: the matcher tests are CPU-only and use + * a fixed testDeviceProperties(), while the dispatch-handler tests compile through + * hiprtc and need currentDeviceProperties() for the device they launch on. + */ +class GraphFixture +{ +public: + explicit GraphFixture(flatbuffers::FlatBufferBuilder builder, + hipDeviceProp_t properties = testDeviceProperties()) + : _builder(std::move(builder)) + , _graph(_builder.GetBufferPointer(), _builder.GetSize()) + , _properties(properties) + { + } + + hipdnn_plugin_sdk::ingestor::MatchContext context() const + { + return hipdnn_plugin_sdk::ingestor::MatchContext{_graph, 0, _properties}; + } + + const hipDeviceProp_t& deviceProperties() const + { + return _properties; + } + +private: + flatbuffers::FlatBufferBuilder _builder; + hipdnn_flatbuffers_sdk::flatbuffer_utilities::GraphWrapper _graph; + hipDeviceProp_t _properties; +}; + +/// @brief A KernelDefinition for this pack's kernel, for tests that never need a real +/// descriptor set. +inline hipdnn_plugin_sdk::ingestor::KernelDefinition makeKernel(int64_t blockSize, + const std::string& dtype) +{ + hipdnn_plugin_sdk::ingestor::KernelDefinition kernel; + kernel.kernelId + = hipdnn_flatbuffers_sdk::utilities::parseUuid("00000000-0000-4000-8000-000000000001"); + kernel.packId + = hipdnn_flatbuffers_sdk::utilities::parseUuid("00000000-0000-4000-8000-000000000002"); + kernel.dispatchId + = hipdnn_flatbuffers_sdk::utilities::parseUuid("00000000-0000-4000-8000-000000000003"); + kernel.source.sourceFile = "PointwiseAdd.cpp"; + kernel.source.entryPoint = "PointwiseAdd"; + kernel.metadata + = {{std::string(BLOCK_SIZE_FIELD), blockSize}, {std::string(DTYPE_FIELD), dtype}}; + return kernel; +} + +} // namespace hip_kernel_provider::kernel_ingestor_engine::testing + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestPointwiseAddDispatchHandler.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestPointwiseAddDispatchHandler.cpp new file mode 100644 index 000000000000..87e94775d752 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestPointwiseAddDispatchHandler.cpp @@ -0,0 +1,375 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/Container.hpp" +#include "core/Handle.hpp" +#include "engines/hip_mlops_engine/HipMlopsKernelCompiler.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddDispatchHandler.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddSymbols.hpp" +#include "mocks/MockCompiledProgram.hpp" +#include "mocks/MockKernelCompiler.hpp" +#include "mocks/MockRunnableKernel.hpp" +#include "tests/engines/kernel_ingestor_engine/packs/PointwiseAddTestGraphs.hpp" + +/** + * @file TestPointwiseAddDispatchHandler.cpp + * @brief The pack's dispatch: workspace sizing, prepare's compile options, and a real + * compile-and-launch. + * + * The launch tests run on device to exercise the runtime compile and uid-to-pointer + * resolution. Everything that fails before those is asserted CPU-only. + */ +namespace +{ + +using namespace hip_kernel_provider; +using namespace hip_kernel_provider::kernel_ingestor_engine; +using namespace hip_kernel_provider::kernel_ingestor_engine::testing; +using hipdnn_plugin_sdk::ingestor::BoundTokens; +using hipdnn_plugin_sdk::ingestor::MatchContext; + +/// The bindings a real plan build would hand the handler, produced by running the graph +/// matcher rather than built by hand. +BoundTokens bindingsFor(const MatchContext& context) +{ + BoundTokens bound; + if(!pointwiseAddGraphMatches(context, bound)) + { + throw std::logic_error("test graph does not match the pack it is dispatched against"); + } + return bound; +} + +/// Device buffers for one 1-element add, freed on scope exit. Templated on the element +/// type to serve every dtype this pack's dispatch handler compiles for. +template +class AddBuffers +{ +public: + AddBuffers(T a, T b) + { + EXPECT_EQ(hipSuccess, hipMalloc(&_a, sizeof(T))); + EXPECT_EQ(hipSuccess, hipMalloc(&_b, sizeof(T))); + EXPECT_EQ(hipSuccess, hipMalloc(&_c, sizeof(T))); + EXPECT_EQ(hipSuccess, hipMemcpy(_a, &a, sizeof(T), hipMemcpyHostToDevice)); + EXPECT_EQ(hipSuccess, hipMemcpy(_b, &b, sizeof(T), hipMemcpyHostToDevice)); + } + + ~AddBuffers() + { + static_cast(hipFree(_a)); + static_cast(hipFree(_b)); + static_cast(hipFree(_c)); + } + + AddBuffers(const AddBuffers&) = delete; + AddBuffers& operator=(const AddBuffers&) = delete; + + std::array descriptors() const + { + return {hipdnnPluginDeviceBuffer_t{INPUT_A_UID, _a}, + hipdnnPluginDeviceBuffer_t{INPUT_B_UID, _b}, + hipdnnPluginDeviceBuffer_t{OUTPUT_UID, _c}}; + } + + T readResult() const + { + T result{}; + EXPECT_EQ(hipSuccess, hipMemcpy(&result, _c, sizeof(T), hipMemcpyDeviceToHost)); + return result; + } + +private: + void* _a = nullptr; + void* _b = nullptr; + void* _c = nullptr; +}; + +// --------------------------------------------------------------------------- +// Workspace +// --------------------------------------------------------------------------- + +struct WorkspaceCase +{ + int64_t blockSize; + size_t expectedBytes; +}; + +class TestPointwiseAddDispatchWorkspace : public ::testing::TestWithParam +{ +}; + +TEST_P(TestPointwiseAddDispatchWorkspace, ReportsWorkspaceFromKernelMetadata) +{ + const GraphFixture fixture(buildPointwiseGraph(), currentDeviceProperties()); + const HipMlopsKernelCompiler compiler; + const PointwiseAddDispatchHandler handler(compiler); + + EXPECT_EQ(handler.workspaceBytes(fixture.context(), + bindingsFor(fixture.context()), + makeKernel(GetParam().blockSize, "FLOAT")), + GetParam().expectedBytes); +} + +INSTANTIATE_TEST_SUITE_P(, + TestPointwiseAddDispatchWorkspace, + ::testing::Values(WorkspaceCase{64, 0U}, WorkspaceCase{256, 1024U}), + [](const ::testing::TestParamInfo& info) { + return "BlockSize" + std::to_string(info.param.blockSize); + }); + +TEST(TestPointwiseAddDispatch, ReportsWorkspaceWithoutSeeingTheRestOfTheCatalog) +{ + const GraphFixture fixture(buildPointwiseGraph(), currentDeviceProperties()); + const HipMlopsKernelCompiler compiler; + const PointwiseAddDispatchHandler handler(compiler); + + // The query is answered per kernel, before selection and before any plan exists, so + // the answer must not depend on which other kernels are in the catalog. + const auto first = handler.workspaceBytes( + fixture.context(), bindingsFor(fixture.context()), makeKernel(256, "FLOAT")); + static_cast(handler.workspaceBytes( + fixture.context(), bindingsFor(fixture.context()), makeKernel(64, "FLOAT"))); + const auto second = handler.workspaceBytes( + fixture.context(), bindingsFor(fixture.context()), makeKernel(256, "FLOAT")); + + EXPECT_EQ(first, second); +} + +// --------------------------------------------------------------------------- +// Prepare: compile options and unhappy paths -- all CPU-only, since nothing here +// reaches a real compiler. +// --------------------------------------------------------------------------- + +TEST(TestPointwiseAddDispatch, PreparePassesTheKernelsTypeAndBlockSizeToTheCompiler) +{ + // A mock compiler proves exactly what prepare() sends it, without hiprtc or a device. + const GraphFixture fixture(buildPointwiseGraph()); + const MockKernelCompiler compiler; + std::vector capturedOptions; + + EXPECT_CALL(compiler, compile("PointwiseAdd.cpp", ::testing::_)) + .WillOnce([&](const std::string&, const std::vector& options) { + capturedOptions = options; + + auto kernel = std::make_unique(); + EXPECT_CALL(*kernel, setBlockSize(::testing::_, ::testing::_, ::testing::_)).Times(1); + EXPECT_CALL(*kernel, setGridSize(::testing::_, ::testing::_, ::testing::_)).Times(1); + + auto program = std::make_unique(); + EXPECT_CALL(*program, getKernel("PointwiseAdd")) + .WillOnce(::testing::Return(::testing::ByMove(std::move(kernel)))); + return program; + }); + + const PointwiseAddDispatchHandler handler(compiler); + const auto prepared = handler.prepare( + fixture.context(), bindingsFor(fixture.context()), makeKernel(256, "FLOAT")); + ASSERT_NE(prepared, nullptr); + + const auto hasOption = [&capturedOptions](const std::string& option) { + return std::find(capturedOptions.begin(), capturedOptions.end(), option) + != capturedOptions.end(); + }; + EXPECT_TRUE(hasOption("-DHIP_PLUGIN_POINTWISE_ADD_TYPE=float")); + EXPECT_TRUE(hasOption("-DHIP_PLUGIN_POINTWISE_ADD_BLOCK_SIZE=256")); +} + +TEST(TestPointwiseAddDispatch, PrepareRejectsAKernelDeclaringAnUnsupportedDtype) +{ + // elementTypeFor's dtype switch is unreachable via matching, which admits only the + // dtypes this pack declares (FLOAT, HALF). Never reaches the compiler -- + // elementTypeFor throws before prepare() calls it. + const GraphFixture fixture(buildPointwiseGraph()); + const HipMlopsKernelCompiler compiler; + const PointwiseAddDispatchHandler handler(compiler); + + EXPECT_THROW(handler.prepare( + fixture.context(), bindingsFor(fixture.context()), makeKernel(64, "BFLOAT16")), + hipdnn_plugin_sdk::HipdnnPluginException); +} + +TEST(TestPointwiseAddDispatch, RefusesToPrepareWithoutTheMatcherSBindings) +{ + // pointwiseAddBinding() throws on a missing token before prepare() touches HIP. + const GraphFixture fixture(buildPointwiseGraph()); + const HipMlopsKernelCompiler compiler; + const PointwiseAddDispatchHandler handler(compiler); + + // Preparation reads the operand uids the matcher bound rather than re-deriving them; + // bindings not produced by this pack's matcher are a wiring error. + EXPECT_THROW(handler.prepare(fixture.context(), BoundTokens{}, makeKernel(64, "FLOAT")), + hipdnn_plugin_sdk::HipdnnPluginException); +} + +// --------------------------------------------------------------------------- +// Prepare and launch on device +// --------------------------------------------------------------------------- + +// Runs over every dtype the pack ships a kernel for. +struct RealLaunchCase +{ + std::string name; + hipdnn_flatbuffers_sdk::data_objects::DataType dataType; + std::string kernelDtype; +}; + +class TestPointwiseAddDispatchRealLaunch : public ::testing::TestWithParam +{ +}; + +TEST_P(TestPointwiseAddDispatchRealLaunch, LaunchesARealAddOnDevice) +{ + SKIP_IF_NO_DEVICES(); + + const auto& param = GetParam(); + const GraphFixture fixture( + buildPointwiseGraph(hipdnn_flatbuffers_sdk::data_objects::PointwiseMode::ADD, + param.dataType), + currentDeviceProperties()); + const HipMlopsKernelCompiler compiler; + const PointwiseAddDispatchHandler handler(compiler); + + const auto prepared = handler.prepare( + fixture.context(), bindingsFor(fixture.context()), makeKernel(64, param.kernelDtype)); + ASSERT_NE(prepared, nullptr); + + const Handle handle; + if(param.dataType == hipdnn_flatbuffers_sdk::data_objects::DataType::HALF) + { + const AddBuffers buffers(hipdnn_data_sdk::types::half(3.0f), + hipdnn_data_sdk::types::half(4.0f)); + const auto descriptors = buffers.descriptors(); + + handler.launch(handle, *prepared, descriptors.data(), descriptors.size(), nullptr); + ASSERT_EQ(hipSuccess, hipDeviceSynchronize()); + EXPECT_NEAR(static_cast(buffers.readResult()), 7.0f, 1e-2f); + } + else + { + const AddBuffers buffers(3.0f, 4.0f); + const auto descriptors = buffers.descriptors(); + + handler.launch(handle, *prepared, descriptors.data(), descriptors.size(), nullptr); + ASSERT_EQ(hipSuccess, hipDeviceSynchronize()); + EXPECT_FLOAT_EQ(buffers.readResult(), 7.0f); + } +} + +INSTANTIATE_TEST_SUITE_P( + , + TestPointwiseAddDispatchRealLaunch, + ::testing::Values( + RealLaunchCase{"Float", hipdnn_flatbuffers_sdk::data_objects::DataType::FLOAT, "FLOAT"}, + RealLaunchCase{"Half", hipdnn_flatbuffers_sdk::data_objects::DataType::HALF, "HALF"}), + [](const ::testing::TestParamInfo& info) { return info.param.name; }); + +TEST(TestPointwiseAddDispatch, LaunchesTheSameResultForEitherBlockSize) +{ + SKIP_IF_NO_DEVICES(); + + const GraphFixture fixture(buildPointwiseGraph(), currentDeviceProperties()); + const HipMlopsKernelCompiler compiler; + const PointwiseAddDispatchHandler handler(compiler); + + // block_size reaches the compiler and the launch geometry; a one-element add must + // still agree across both kernels. + for(const int64_t blockSize : {64, 256}) + { + const auto prepared = handler.prepare( + fixture.context(), bindingsFor(fixture.context()), makeKernel(blockSize, "FLOAT")); + const AddBuffers buffers(1.5f, 2.25f); + const auto descriptors = buffers.descriptors(); + const Handle handle; + + handler.launch(handle, *prepared, descriptors.data(), descriptors.size(), nullptr); + ASSERT_EQ(hipSuccess, hipDeviceSynchronize()); + + EXPECT_FLOAT_EQ(buffers.readResult(), 3.75f) << "block size " << blockSize; + } +} + +TEST(TestPointwiseAddDispatch, PreparedLaunchIsReusableAcrossExecutions) +{ + SKIP_IF_NO_DEVICES(); + + const GraphFixture fixture(buildPointwiseGraph(), currentDeviceProperties()); + const HipMlopsKernelCompiler compiler; + const PointwiseAddDispatchHandler handler(compiler); + + // A plan is built once and executes many times, so preparation must hold nothing + // tied to one execution. + const auto prepared = handler.prepare( + fixture.context(), bindingsFor(fixture.context()), makeKernel(64, "FLOAT")); + const Handle handle; + + for(const auto& [a, b, expected] : std::vector>{ + {1.0f, 2.0f, 3.0f}, {10.0f, -4.0f, 6.0f}, {0.5f, 0.25f, 0.75f}}) + { + const AddBuffers buffers(a, b); + const auto descriptors = buffers.descriptors(); + + handler.launch(handle, *prepared, descriptors.data(), descriptors.size(), nullptr); + ASSERT_EQ(hipSuccess, hipDeviceSynchronize()); + + EXPECT_FLOAT_EQ(buffers.readResult(), expected); + } +} + +TEST(TestPointwiseAddDispatch, DispatchStaysResolvableAcrossContainerLifetimes) +{ + SKIP_IF_NO_DEVICES(); + + // Registered dispatch implementations live in a process-wide registry and must + // outlive every container. This asserts a rebuilt container still resolves and runs + // the registration; a use-after-free is instead caught by the integration binary's + // exit status at process teardown. + { + const core::Container first; + } + + const core::Container second; + + const auto* handler = hipdnn_plugin_sdk::ingestor::DispatchRegistry::resolve( + std::string(DISPATCH_SYMBOL)); + ASSERT_NE(handler, nullptr); + + const GraphFixture fixture(buildPointwiseGraph(), currentDeviceProperties()); + const auto prepared = handler->prepare( + fixture.context(), bindingsFor(fixture.context()), makeKernel(64, "FLOAT")); + ASSERT_NE(prepared, nullptr); + + const AddBuffers buffers(2.0f, 5.0f); + const auto descriptors = buffers.descriptors(); + const Handle handle; + + handler->launch(handle, *prepared, descriptors.data(), descriptors.size(), nullptr); + ASSERT_EQ(hipSuccess, hipDeviceSynchronize()); + + EXPECT_FLOAT_EQ(buffers.readResult(), 7.0f); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestPointwiseAddMatchers.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestPointwiseAddMatchers.cpp new file mode 100644 index 000000000000..7d04ac66c8d2 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestPointwiseAddMatchers.cpp @@ -0,0 +1,282 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include "engines/kernel_ingestor_engine/packs/PointwiseAddMatchers.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddSymbols.hpp" +#include "tests/engines/kernel_ingestor_engine/packs/PointwiseAddTestGraphs.hpp" + +/** + * @file TestPointwiseAddMatchers.cpp + * @brief The pack's two matcher shapes: what each accepts, and what each refuses. + * + * An under-specified refusal accepts a graph the kernel cannot serve, which is silently + * wrong rather than merely a missed optimization. + */ +namespace +{ + +using namespace hip_kernel_provider::kernel_ingestor_engine; +using namespace hip_kernel_provider::kernel_ingestor_engine::testing; +using hipdnn_plugin_sdk::ingestor::BoundTokens; +using hipdnn_plugin_sdk::ingestor::MatchContext; +namespace data_objects = hipdnn_flatbuffers_sdk::data_objects; + +/// Runs the graph matcher, discarding what it binds; binding is asserted separately below. +bool matches(const MatchContext& context) +{ + BoundTokens bound; + return pointwiseAddGraphMatches(context, bound); +} + +// --------------------------------------------------------------------------- +// Graph-scoped matcher: acceptances +// --------------------------------------------------------------------------- + +TEST(TestPointwiseAddGraphMatcher, AcceptsASingleElementFloatAdd) +{ + const GraphFixture fixture(buildPointwiseGraph()); + + EXPECT_TRUE(matches(fixture.context())); +} + +TEST(TestPointwiseAddGraphMatcher, AcceptsAHalfPrecisionAdd) +{ + // The graph-level gate is dtype-agnostic; pinning the kernel's baked dtype is the + // kernel-scoped matcher's job. + const GraphFixture fixture( + buildPointwiseGraph(data_objects::PointwiseMode::ADD, data_objects::DataType::HALF)); + + EXPECT_TRUE(matches(fixture.context())); +} + +TEST(TestPointwiseAddGraphMatcher, AcceptsTheUpperSupportedRank) +{ + const GraphFixture fixture(buildPointwiseGraph( + data_objects::PointwiseMode::ADD, data_objects::DataType::FLOAT, {1, 1, 1, 1, 1})); + + EXPECT_TRUE(matches(fixture.context())); +} + +// --------------------------------------------------------------------------- +// Graph-scoped matcher: refusals +// --------------------------------------------------------------------------- + +/// One graph the matcher must refuse, plus a readable name for a failing run. The builder +/// is a plain function pointer, not std::function, since FlatBufferBuilder is move-only. +struct GraphMatcherRefusalCase +{ + std::string name; + flatbuffers::FlatBufferBuilder (*buildGraph)(); +}; + +class TestPointwiseAddGraphMatcherRefusal : public ::testing::TestWithParam +{ +}; + +TEST_P(TestPointwiseAddGraphMatcherRefusal, Refuses) +{ + const GraphFixture fixture(GetParam().buildGraph()); + + EXPECT_FALSE(matches(fixture.context())); +} + +INSTANTIATE_TEST_SUITE_P( + , + TestPointwiseAddGraphMatcherRefusal, + ::testing::ValuesIn(std::vector{ + {"AnotherPointwiseOperation", + []() { return buildPointwiseGraph(data_objects::PointwiseMode::MUL); }}, + {"MultiElementTensors", + // The kernel writes only element 0; a larger tensor leaves the rest unwritten. + []() { + return buildPointwiseGraph( + data_objects::PointwiseMode::ADD, data_objects::DataType::FLOAT, {1, 1, 2, 2}); + }}, + {"ARankTheDispatchPathCannotServe", + // The provider derives layout from tensor rank and rejects anything below rank 4. + []() { + return buildPointwiseGraph( + data_objects::PointwiseMode::ADD, data_objects::DataType::FLOAT, {1}); + }}, + {"AStrideOrderTheDispatchPathCannotClassify", + // The provider derives layout from tensor strides and only classifies NCHW or + // NHWC; other orders reach plan build and fail there (RFC 0017 §8.6). + []() { + return buildPointwiseGraph(data_objects::PointwiseMode::ADD, + data_objects::DataType::FLOAT, + {1, 1, 1, 1}, + std::nullopt, + /*binary=*/true, + /*explicitStrides=*/std::vector{8, 2, 4, 1}); + }}, + {"AUnaryPointwise", + []() { + return buildPointwiseGraph(data_objects::PointwiseMode::ADD, + data_objects::DataType::FLOAT, + {1, 1, 1, 1}, + std::nullopt, + /*binary=*/false); + }}, + {"AMultiNodeGraph", + // A prebuilt kernel serves one complete graph, not a graph containing it. + []() { return buildTwoNodePointwiseGraph(); }}, + {"CrossOperandDtypeMismatch", + // This pack's kernel reads one dtype for every operand; mismatched inputs are + // unreadable, not merely unoptimized. + []() { + return buildPointwiseGraph(data_objects::PointwiseMode::ADD, + data_objects::DataType::FLOAT, + {1, 1, 1, 1}, + std::nullopt, + /*binary=*/true, + /*explicitStrides=*/std::nullopt, + /*inputBDataType=*/data_objects::DataType::HALF); + }}, + {"AThirdOperand", + // A third operand is a different operation from this pack's binary add, not a + // larger instance of the same one. + []() { + return buildPointwiseGraph(data_objects::PointwiseMode::ADD, + data_objects::DataType::FLOAT, + {1, 1, 1, 1}, + std::nullopt, + /*binary=*/true, + /*explicitStrides=*/std::nullopt, + /*inputBDataType=*/std::nullopt, + /*includeThirdOperand=*/true); + }}, + {"ADanglingTensorUid", + // The uid names a tensor absent from the graph's tensor map, distinct from one + // the matcher reads and declines. + []() { + return buildPointwiseGraph(data_objects::PointwiseMode::ADD, + data_objects::DataType::FLOAT, + {1, 1, 1, 1}, + std::nullopt, + /*binary=*/true, + /*explicitStrides=*/std::nullopt, + /*inputBDataType=*/std::nullopt, + /*includeThirdOperand=*/false, + /*danglingInputBUid=*/DEFAULT_DANGLING_UID); + }}, + {"AVirtualOperand", + // A virtual tensor never appears in the launch's variant pack. + []() { + return buildPointwiseGraph(data_objects::PointwiseMode::ADD, + data_objects::DataType::FLOAT, + {1, 1, 1, 1}, + std::nullopt, + /*binary=*/true, + /*explicitStrides=*/std::nullopt, + /*inputBDataType=*/std::nullopt, + /*includeThirdOperand=*/false, + /*danglingInputBUid=*/std::nullopt, + /*inputAVirtual=*/true); + }}, + {"ARuntimePassByValueOperand", + // This pack's supported shape is indistinguishable from a pass-by-value + // scalar, whose variant-pack slot is a host pointer, not a device one. + []() { + return buildPointwiseGraph(data_objects::PointwiseMode::ADD, + data_objects::DataType::FLOAT, + {1, 1, 1, 1}, + std::nullopt, + /*binary=*/true, + /*explicitStrides=*/std::nullopt, + /*inputBDataType=*/std::nullopt, + /*includeThirdOperand=*/false, + /*danglingInputBUid=*/std::nullopt, + /*inputAVirtual=*/false, + /*inputAIsRuntimePassByValue=*/true); + }}, + }), + [](const ::testing::TestParamInfo& info) { return info.param.name; }); + +// --------------------------------------------------------------------------- +// Kernel-scoped matcher +// --------------------------------------------------------------------------- + +TEST(TestPointwiseAddKernelMatcher, AcceptsAKernelWhoseDtypeMatchesTheGraph) +{ + const GraphFixture fixture(buildPointwiseGraph()); + + EXPECT_TRUE(pointwiseAddKernelMatches(fixture.context(), makeKernel(64, "FLOAT"))); +} + +TEST(TestPointwiseAddKernelMatcher, RefusesAKernelBakedForAnotherDtype) +{ + // An f16 kernel handed f32 operands does not fail; it returns wrong numbers. + const GraphFixture fixture(buildPointwiseGraph()); + + EXPECT_FALSE(pointwiseAddKernelMatches(fixture.context(), makeKernel(64, "HALF"))); +} + +TEST(TestPointwiseAddKernelMatcher, AcceptsAHalfKernelForAHalfGraph) +{ + const GraphFixture fixture( + buildPointwiseGraph(data_objects::PointwiseMode::ADD, data_objects::DataType::HALF)); + + EXPECT_TRUE(pointwiseAddKernelMatches(fixture.context(), makeKernel(64, "HALF"))); +} + +TEST(TestPointwiseAddKernelMatcher, IgnoresBlockSizeWhichTheGraphDoesNotConstrain) +{ + // block_size ranks kernels but never gates applicability. + const GraphFixture fixture(buildPointwiseGraph()); + + EXPECT_TRUE(pointwiseAddKernelMatches(fixture.context(), makeKernel(64, "FLOAT"))); + EXPECT_TRUE(pointwiseAddKernelMatches(fixture.context(), makeKernel(256, "FLOAT"))); +} + +// --------------------------------------------------------------------------- +// Score and binding +// --------------------------------------------------------------------------- + +TEST(TestPointwiseAddScore, PrefersTheLargerBlockSize) +{ + const GraphFixture fixture(buildPointwiseGraph()); + + EXPECT_GT(pointwiseAddScore(makeKernel(256, "FLOAT"), fixture.context()), + pointwiseAddScore(makeKernel(64, "FLOAT"), fixture.context())); +} + +TEST(TestPointwiseAddBinding, TheMatcherBindsTheOperandUidsItResolved) +{ + const GraphFixture fixture(buildPointwiseGraph()); + + BoundTokens bound; + ASSERT_TRUE(pointwiseAddGraphMatches(fixture.context(), bound)); + + const auto binding = pointwiseAddBinding(bound); + EXPECT_EQ(binding.inputA, INPUT_A_UID); + EXPECT_EQ(binding.inputB, INPUT_B_UID); + EXPECT_EQ(binding.output, OUTPUT_UID); +} + +TEST(TestPointwiseAddBinding, ARejectedGraphBindsNothingToDispatchFrom) +{ + const GraphFixture fixture(buildTwoNodePointwiseGraph()); + + BoundTokens bound; + ASSERT_FALSE(pointwiseAddGraphMatches(fixture.context(), bound)); + + EXPECT_TRUE(bound.empty()); + EXPECT_THROW(pointwiseAddBinding(bound), hipdnn_plugin_sdk::HipdnnPluginException); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestPointwiseAddPack.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestPointwiseAddPack.cpp new file mode 100644 index 000000000000..8cc3d2085839 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestPointwiseAddPack.cpp @@ -0,0 +1,154 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include + +#include + +#include + +#include "engines/kernel_ingestor_engine/packs/PointwiseAddPack.hpp" +#include "engines/kernel_ingestor_engine/packs/PointwiseAddSymbols.hpp" + +/** + * @file TestPointwiseAddPack.cpp + * @brief The pack's descriptor data: its shape, its cross-references, and its engine id. + */ +namespace +{ + +using namespace hip_kernel_provider::kernel_ingestor_engine; + +TEST(TestPointwiseAddPack, ShipsThreeKernelsCoveringTwoBlockSizesAndTwoDataTypes) +{ + const auto set = buildPointwiseAddDescriptorSet(); + + ASSERT_EQ(set.packs.size(), 1U); + const auto& kernels = set.packs.front().kernels; + ASSERT_EQ(kernels.size(), 3U); + + // The two FLOAT kernels differ only in block size; HALF is pruned on a FLOAT graph. + const auto describes = [&kernels](int64_t blockSize, const std::string& dtype) { + return std::any_of(kernels.begin(), kernels.end(), [&](const auto& kernel) { + return std::get(kernel.metadata.at(std::string(BLOCK_SIZE_FIELD))) == blockSize + && std::get(kernel.metadata.at(std::string(DTYPE_FIELD))) == dtype; + }); + }; + + EXPECT_TRUE(describes(64, "FLOAT")); + EXPECT_TRUE(describes(256, "FLOAT")); + EXPECT_TRUE(describes(64, "HALF")); +} + +TEST(TestPointwiseAddPack, EveryKernelNamesTheEmbeddedSource) +{ + const auto set = buildPointwiseAddDescriptorSet(); + + for(const auto& kernel : set.packs.front().kernels) + { + EXPECT_EQ(kernel.source.kind, + hipdnn_plugin_sdk::ingestor::KernelSourceKind::EMBEDDED_SOURCE); + EXPECT_EQ(kernel.source.sourceFile, "PointwiseAdd.cpp"); + EXPECT_EQ(kernel.source.entryPoint, "PointwiseAdd"); + } +} + +TEST(TestPointwiseAddPack, PackCrossReferencesResolveWithinTheDescriptorSet) +{ + const auto set = buildPointwiseAddDescriptorSet(); + const auto& pack = set.packs.front(); + + // A pack names its engine, matchers, and dispatch by id; every one must resolve. + EXPECT_EQ(pack.engineId, set.engine.id); + + ASSERT_EQ(pack.matcherIds.size(), 2U); + for(const auto& matcherId : pack.matcherIds) + { + EXPECT_TRUE(std::any_of(set.matchers.begin(), set.matchers.end(), [&](const auto& matcher) { + return matcher.id == matcherId; + })); + } + + EXPECT_TRUE(std::any_of(set.dispatches.begin(), + set.dispatches.end(), + [&](const auto& dispatch) { return dispatch.id == pack.dispatchId; })); +} + +TEST(TestPointwiseAddPack, EngineNamesItsHeuristicAndMetadataSchema) +{ + const auto set = buildPointwiseAddDescriptorSet(); + + // An engine carries exactly one heuristic and one metadata schema. + EXPECT_EQ(set.engine.heuristicId, set.heuristic.id); + EXPECT_EQ(set.engine.metadataSchemaId, set.schema.id); +} + +TEST(TestPointwiseAddPack, ExposesBlockSizeAsAKnobAndDtypeAsInternal) +{ + const auto set = buildPointwiseAddDescriptorSet(); + + // dtype is pinned by the graph rather than chosen. + ASSERT_EQ(set.engine.knobs.size(), 1U); + EXPECT_EQ(set.engine.knobs.front(), std::string(BLOCK_SIZE_FIELD)); +} + +TEST(TestPointwiseAddPack, EveryKnobNamesADeclaredMetadataField) +{ + const auto set = buildPointwiseAddDescriptorSet(); + + // A knob naming a field the schema does not declare could never be filtered or defaulted. + for(const auto& knob : set.engine.knobs) + { + EXPECT_TRUE(std::any_of(set.schema.fields.begin(), + set.schema.fields.end(), + [&](const auto& field) { return field.name == knob; })); + } +} + +TEST(TestPointwiseAddPack, MatchersCoverBothScopes) +{ + const auto set = buildPointwiseAddDescriptorSet(); + + // Graph-scoped prunes the whole pack once per graph; kernel-scoped prunes per candidate. + EXPECT_EQ(std::count_if(set.matchers.begin(), + set.matchers.end(), + [](const auto& matcher) { + return matcher.scope + == hipdnn_plugin_sdk::ingestor::MatchScope::GRAPH; + }), + 1); + EXPECT_EQ(std::count_if(set.matchers.begin(), + set.matchers.end(), + [](const auto& matcher) { + return matcher.scope + == hipdnn_plugin_sdk::ingestor::MatchScope::KERNEL; + }), + 1); +} + +TEST(TestPointwiseAddPack, EngineIdIsTheHashOfItsScopedName) +{ + // A descriptor-backed engine's id comes from its UED name. + EXPECT_EQ(pointwiseAddEngineId(), hipdnn_data_sdk::utilities::engineNameToId(ENGINE_NAME)); +} + +TEST(TestPointwiseAddPack, EngineIdIsStableAcrossCalls) +{ + // The id keys the engine in hipDNN's registry and this provider's engine table. + EXPECT_EQ(pointwiseAddEngineId(), pointwiseAddEngineId()); +} + +TEST(TestPointwiseAddPack, RegistersTheEngineNameForDiagnostics) +{ + // Registered at run time so a log line or collision report can name the engine. + const auto id = pointwiseAddEngineId(); + + EXPECT_EQ(hipdnn_data_sdk::utilities::getEngineNameFromId(id), ENGINE_NAME); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/.clang-tidy b/projects/hipdnn/.clang-tidy index 1208549203a4..e2269a310b80 100644 --- a/projects/hipdnn/.clang-tidy +++ b/projects/hipdnn/.clang-tidy @@ -259,7 +259,13 @@ HeaderFileExtensions: ['h','hpp'] # This regex is a inclusion filter for the files to be checked. Its a Posix regex and # does not seem to support negative lookahead. HeaderFilterRegex: '.' -ExcludeHeaderFilterRegex: '^(.*(data_objects|python3\.[0-9]+|nanobind|_deps).*)$' +# Skip machine-generated headers: the FlatBuffers reflection headers live under +# data_objects, and the umd registry emitter (umd_registry_gen) writes +# umd/*_generated.hpp (e.g. umd/op_schema_registry_generated.hpp). Their C-style +# arrays, non-conforming identifiers, and brace-less statements are artifacts of the +# emitter, not fixable in-tree. The umd match is scoped to that directory so a +# hand-written file elsewhere that happens to end in _generated is still checked. +ExcludeHeaderFilterRegex: '^(.*(data_objects|python3\.[0-9]+|nanobind|_deps).*|.*/umd/.*_generated\.(h|hpp))$' CheckOptions: # Disable type analysis for misc-use-internal-linkage to maintain consistent diff --git a/projects/hipdnn/CMakeLists.txt b/projects/hipdnn/CMakeLists.txt index 4f11bc09a361..7c737a877420 100644 --- a/projects/hipdnn/CMakeLists.txt +++ b/projects/hipdnn/CMakeLists.txt @@ -60,6 +60,16 @@ else() message(STATUS "hipDNN: SDPA support disabled") endif() +# Gates the generic kernel ingestor (RFC 0017: Universal Kernel Descriptors), set by +# TheRock's THEROCK_FLAG_HIPDNN_ENABLE_KERNEL_INGESTOR. +option(HIPDNN_ENABLE_KERNEL_INGESTOR "Enable the generic kernel ingestor (RFC 0017)" OFF) +if(HIPDNN_ENABLE_KERNEL_INGESTOR) + message(STATUS "hipDNN: kernel ingestor enabled") + add_compile_definitions(HIPDNN_ENABLE_KERNEL_INGESTOR) +else() + message(STATUS "hipDNN: kernel ingestor disabled") +endif() + # Gates install of the cuDNN-compatibility shim headers and build/run of its # tests. The shim sources always exist in the tree; this option controls only # whether they are installed and tested. diff --git a/projects/hipdnn/backend/src/PlatformUtils.hpp b/projects/hipdnn/backend/src/PlatformUtils.hpp index 77ea47e4044a..fac853066236 100644 --- a/projects/hipdnn/backend/src/PlatformUtils.hpp +++ b/projects/hipdnn/backend/src/PlatformUtils.hpp @@ -3,6 +3,8 @@ #pragma once +#include +#include #include #include #include @@ -24,5 +26,6 @@ PluginLibHandle openLibrary(const std::filesystem::path& libraryPath); void closeLibrary(PluginLibHandle handle); void* getSymbol(PluginLibHandle handle, const char* symbolName); std::string getSystemInfo(); +std::array generateUuidV4(); } diff --git a/projects/hipdnn/backend/src/PlatformUtils.linux.cpp b/projects/hipdnn/backend/src/PlatformUtils.linux.cpp index d2ed2df605dc..21869e401a09 100644 --- a/projects/hipdnn/backend/src/PlatformUtils.linux.cpp +++ b/projects/hipdnn/backend/src/PlatformUtils.linux.cpp @@ -8,6 +8,7 @@ #include "HipdnnException.hpp" #include #include +#include #include namespace hipdnn_backend::platform_utilities @@ -79,6 +80,21 @@ std::string getSystemInfo() buffer.machine); } +std::array generateUuidV4() +{ + // getentropy() fills the whole buffer or fails, for any request up to 256 bytes. + std::array bytes{}; + if(getentropy(bytes.data(), bytes.size()) != 0) + { + throw HipdnnException(HIPDNN_STATUS_INTERNAL_ERROR, + "Failed to generate graph UUID using getentropy."); + } + + bytes[6] = static_cast((bytes[6] & 0x0fU) | 0x40U); + bytes[8] = static_cast((bytes[8] & 0x3fU) | 0x80U); + return bytes; +} + } #endif // defined(__linux__) diff --git a/projects/hipdnn/backend/src/PlatformUtils.windows.cpp b/projects/hipdnn/backend/src/PlatformUtils.windows.cpp index 7773d38de5d6..d6d88b41adbc 100644 --- a/projects/hipdnn/backend/src/PlatformUtils.windows.cpp +++ b/projects/hipdnn/backend/src/PlatformUtils.windows.cpp @@ -1,12 +1,20 @@ // Copyright © Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT +#ifdef _WIN32 +// Must precede every include: only declares rand_s when this is already defined, and +// any header that reaches it first silently leaves rand_s undeclared. +#define _CRT_RAND_S +#endif + #include "PlatformUtils.hpp" #ifdef _WIN32 #include "HipdnnException.hpp" #include +#include +#include #include #include @@ -142,6 +150,25 @@ std::string getSystemInfo() architecture); } +std::array generateUuidV4() +{ + std::array bytes{}; + for(size_t offset = 0; offset < bytes.size(); offset += sizeof(unsigned int)) + { + unsigned int randomValue; + if(rand_s(&randomValue) != 0) + { + throw HipdnnException(HIPDNN_STATUS_INTERNAL_ERROR, + "Failed to generate graph UUID using rand_s."); + } + std::memcpy(bytes.data() + offset, &randomValue, sizeof(randomValue)); + } + + bytes[6] = static_cast((bytes[6] & 0x0fU) | 0x40U); + bytes[8] = static_cast((bytes[8] & 0x3fU) | 0x80U); + return bytes; +} + } #endif // _WIN32 diff --git a/projects/hipdnn/backend/src/descriptors/GraphDescriptor.cpp b/projects/hipdnn/backend/src/descriptors/GraphDescriptor.cpp index 28f90ae8c572..035e2b34a7f2 100644 --- a/projects/hipdnn/backend/src/descriptors/GraphDescriptor.cpp +++ b/projects/hipdnn/backend/src/descriptors/GraphDescriptor.cpp @@ -9,11 +9,14 @@ #include "HipdnnBackendDescriptorType.h" #include "HipdnnException.hpp" #include "NodeFactory.hpp" +#include "PlatformUtils.hpp" #include +#include #include #include #include +#include #include #include @@ -33,11 +36,34 @@ void GraphDescriptor::finalize() HIPDNN_STATUS_BAD_PARAM, "GraphDescriptor::finalize: no operations set"); - invalidateCache(); - buildSerializedGraph(); + const bool needsId = !_graphId.has_value(); + if(needsId) + { + _graphId = platform_utilities::generateUuidV4(); + } + + try + { + invalidateCache(); + buildSerializedGraph(); + } + catch(...) + { + // Uncovered by tests: reaching here needs buildSerializedGraph() to fail on operations + // that already validated, which no input can force through the public API. + if(needsId) + { + _graphId.reset(); + } + throw; + } HipdnnBackendDescriptorImpl::finalize(); + HIPDNN_BACKEND_LOG_INFO("Finalized graph \"{}\" with id {}", + _name, + hipdnn_flatbuffers_sdk::utilities::formatUuid(*_graphId)); + if(logging::GraphLogger::isEnabled()) { auto serialized = getSerializedGraph(); @@ -58,6 +84,11 @@ std::unique_ptr graph->preferred_engine_id = _preferredEngineId; graph->is_override_shape_enabled = _isOverrideShapeEnabled; graph->name = _name; + if(_graphId.has_value()) + { + graph->id = std::make_unique( + hipdnn_flatbuffers_sdk::utilities::toFlatbufferUuid(*_graphId)); + } std::unordered_map> seenTensors; @@ -378,21 +409,30 @@ void GraphDescriptor::setAttribute(hipdnnBackendAttributeName_t attributeName, // Non-mutating attributes (e.g., HANDLE) set invalidate = false. if(invalidate) { + if(_graphId.has_value()) + { + HIPDNN_BACKEND_LOG_DEBUG( + "Discarding id {} for graph \"{}\": its contents changed, so finalizing again " + "generates a new id", + hipdnn_flatbuffers_sdk::utilities::formatUuid(*_graphId), + _name); + } invalidateCache(); + _graphId.reset(); } } void GraphDescriptor::deserializeGraph(const uint8_t* serializedGraph, size_t graphByteSize) { + THROW_IF_TRUE(isFinalized(), + HIPDNN_STATUS_NOT_INITIALIZED, + "GraphDescriptor::deserializeGraph() failed: Already finalized."); THROW_IF_NULL(serializedGraph, HIPDNN_STATUS_BAD_PARAM_NULL_POINTER, "GraphDescriptor::deserializeGraph: serializedGraph is null"); THROW_IF_TRUE(graphByteSize == 0, HIPDNN_STATUS_BAD_PARAM, "GraphDescriptor::deserializeGraph: graphByteSize is 0"); - - invalidateCache(); - // Parse FlatBuffer and eagerly unpack into _operations std::unique_ptr graph; flatbuffer_utilities::convertSerializedGraphToGraph(serializedGraph, graphByteSize, graph); @@ -405,7 +445,22 @@ void GraphDescriptor::deserializeGraph(const uint8_t* serializedGraph, size_t gr HIPDNN_STATUS_NOT_SUPPORTED, "Serialized graph requires a newer engine plugin API version than this build supports."); - // Extract graph-level attributes + std::optional> graphId; + if(graph->id) + { + graphId = hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*graph->id); + } + + auto tensorMap = NodeFactory::buildTensorMap(graph->tensors); + std::vector> unpacked; + unpacked.reserve(graph->nodes.size()); + for(const auto& nodeT : graph->nodes) + { + unpacked.push_back(NodeFactory::createOperationFromNode(*nodeT, tensorMap)); + } + + // Commit only after every validation and conversion succeeds. + invalidateCache(); _computeDataType = graph->compute_data_type; _intermediateDataType = graph->intermediate_data_type; _ioDataType = graph->io_data_type; @@ -416,14 +471,12 @@ void GraphDescriptor::deserializeGraph(const uint8_t* serializedGraph, size_t gr _isRuntimePassByValueEnabled = hipdnn_flatbuffers_sdk::utilities::anyTensorIsRuntimePassByValue( graph->tensors, [](const auto& tensor) { return tensor.get(); }); _name = graph->name; - - // Populate _operations from the deserialized graph nodes - auto tensorMap = NodeFactory::buildTensorMap(graph->tensors); - std::vector> unpacked; - unpacked.reserve(graph->nodes.size()); - for(const auto& nodeT : graph->nodes) + _graphId = graphId; + if(_graphId.has_value()) { - unpacked.push_back(NodeFactory::createOperationFromNode(*nodeT, tensorMap)); + HIPDNN_BACKEND_LOG_DEBUG("Deserialized graph \"{}\" with inherited id {}", + _name, + hipdnn_flatbuffers_sdk::utilities::formatUuid(*_graphId)); } _operations = std::move(unpacked); } @@ -578,6 +631,9 @@ std::string GraphDescriptor::toString() const std::string str = "GraphDescriptor: {handle="; str += _handle != nullptr ? fmt::format("{:p}", static_cast(_handle)) : "null"; str += ", name=" + (_name.empty() ? std::string("(empty)") : _name); + str += ", id=" + + (_graphId.has_value() ? hipdnn_flatbuffers_sdk::utilities::formatUuid(*_graphId) + : std::string("(none)")); str += ", serializedGraphSize=" + std::to_string(_graphSerializedBuffer.size()); str += '}'; return str; diff --git a/projects/hipdnn/backend/src/descriptors/GraphDescriptor.hpp b/projects/hipdnn/backend/src/descriptors/GraphDescriptor.hpp index 139555246b57..07f5f2343838 100644 --- a/projects/hipdnn/backend/src/descriptors/GraphDescriptor.hpp +++ b/projects/hipdnn/backend/src/descriptors/GraphDescriptor.hpp @@ -5,6 +5,7 @@ #include "BackendDescriptor.hpp" #include "IGraphOperation.hpp" +#include #include #include #include @@ -37,6 +38,11 @@ class GraphDescriptor : public HipdnnBackendDescriptorImpl // Cleared by invalidateCache() when operations are mutated. flatbuffers::DetachedBuffer _graphSerializedBuffer; + // Stable identity of a logical finalized graph across serialization round trips. + // Deserialized copies preserve it, so it is not a unique live-object token; use it only to + // key state derived from immutable graph content. Newly generated values use UUID v4. + std::optional> _graphId; + // Source of truth for the graph's operations. Populated via setOperations() (C-API flow) // or eagerly from deserializeGraph() (FlatBuffer flow). // Stored as IBackendDescriptor so getOperations() can pack them without cross-casting. diff --git a/projects/hipdnn/backend/src/logging/GraphLogger.cpp b/projects/hipdnn/backend/src/logging/GraphLogger.cpp index 9fd4640809ae..208091ca7bad 100644 --- a/projects/hipdnn/backend/src/logging/GraphLogger.cpp +++ b/projects/hipdnn/backend/src/logging/GraphLogger.cpp @@ -10,11 +10,10 @@ #include #include #include +#include #include -#include #include #include -#include namespace hipdnn_backend::logging { @@ -90,38 +89,63 @@ std::filesystem::path GraphLogger::getOutputDirectory() void GraphLogger::logGraph(const uint8_t* serializedGraph, size_t size) { - auto hash = hipdnn_data_sdk::utilities::fnv1aHash(serializedGraph, size); + flatbuffers::Verifier verifier(serializedGraph, size); + if(!hipdnn_flatbuffers_sdk::data_objects::VerifyGraphBuffer(verifier)) + { + HIPDNN_BACKEND_LOG_WARN("Not logging a graph whose buffer failed verification"); + return; + } + + const auto* graph + = flatbuffers::GetRoot(serializedGraph); - std::ostringstream oss; - oss << "graph_" << std::hex << std::setfill('0') << std::setw(16) << hash << ".json"; - auto fullPath = getOutputDirectory() / oss.str(); + // Dumps are named by the graph's ID, so re-finalizing a descriptor or replaying a serialized + // graph reuses one file, while a graph rebuilt from scratch is a distinct graph object and + // gets its own. + if(graph->id() == nullptr) + { + HIPDNN_BACKEND_LOG_WARN("Not logging a graph without an identity; only a finalized graph " + "has one"); + return; + } - if(fullPath.empty()) + const auto graphId = hipdnn_flatbuffers_sdk::utilities::formatUuid( + hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*graph->id())); + const auto graphName = graph->name() != nullptr ? graph->name()->str() : std::string(); + + const auto outputDirectory = getOutputDirectory(); + if(outputDirectory.empty()) { HIPDNN_BACKEND_LOG_WARN("Graph logging is enabled but no valid output directory is set"); return; } + const auto fullPath = outputDirectory / ("graph_" + graphId + ".json"); if(std::filesystem::exists(fullPath)) { - HIPDNN_BACKEND_LOG_INFO("Skipping duplicate graph logged to {}", fullPath.string()); + HIPDNN_BACKEND_LOG_INFO("Skipping graph \"{}\" with id {}, already logged to {}", + graphName, + graphId, + fullPath.string()); return; } - auto* graph - = flatbuffers::GetRoot(serializedGraph); - const nlohmann::json j = *graph; + const nlohmann::json graphJson = *graph; std::ofstream file(fullPath); if(file.is_open()) { - file << j.dump(2); + file << graphJson.dump(2); file.close(); - HIPDNN_BACKEND_LOG_INFO("Graph logged to {}", fullPath.string()); + HIPDNN_BACKEND_LOG_INFO( + "Writing graph \"{}\" with id {} to {}", graphName, graphId, fullPath.string()); } else { - HIPDNN_BACKEND_LOG_WARN("Failed to open graph log file: {}", fullPath.string()); + HIPDNN_BACKEND_LOG_WARN("Failed to open graph log file {} for graph \"{}\" with id {}", + fullPath.string(), + graphName, + graphId); } } diff --git a/projects/hipdnn/backend/src/logging/GraphLogger.hpp b/projects/hipdnn/backend/src/logging/GraphLogger.hpp index 1a125b2a8d50..b8a1908f2dd9 100644 --- a/projects/hipdnn/backend/src/logging/GraphLogger.hpp +++ b/projects/hipdnn/backend/src/logging/GraphLogger.hpp @@ -13,8 +13,10 @@ namespace hipdnn_backend::logging class GraphLogger { public: - /// Logs the graph as JSON. Deduplicates by content hash — if the same - /// flatbuffer binary has already been logged, this is a no-op. + /// Logs the graph as JSON, named by the graph's ID. Only a finalized graph carries an ID; + /// an unidentified or malformed buffer is not logged. Logging a graph whose ID already has + /// a file in the output directory is a no-op, so re-finalized and replayed graphs are + /// written once; a graph rebuilt from scratch has a new ID and is written separately. /// @param serializedGraph pointer to flatbuffer binary /// @param size size of the flatbuffer binary static void logGraph(const uint8_t* serializedGraph, size_t size); diff --git a/projects/hipdnn/backend/tests/TestGraphAndPlanSerializationApi.cpp b/projects/hipdnn/backend/tests/TestGraphAndPlanSerializationApi.cpp index d0e92ae46c22..7f23448e4803 100644 --- a/projects/hipdnn/backend/tests/TestGraphAndPlanSerializationApi.cpp +++ b/projects/hipdnn/backend/tests/TestGraphAndPlanSerializationApi.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -134,6 +135,32 @@ class TestGraphAndPlanSerializationApi : public ::testing::Test return createValidGraph().Release(); } + // Same shape as makeBareGraphBlob(), but with an explicit graph id - the + // container format (HDGP) is a distinct wire format from the bare graph + // blob, so id survival has to be proven through it separately. + static flatbuffers::DetachedBuffer + makeBareGraphBlobWithId(const hipdnn_flatbuffers_sdk::utilities::UuidBytes& idBytes) + { + using namespace hipdnn_flatbuffers_sdk::data_objects; + flatbuffers::FlatBufferBuilder builder; + const std::vector<::flatbuffers::Offset> tensors; + const std::vector<::flatbuffers::Offset> nodes; + const auto uuid = hipdnn_flatbuffers_sdk::utilities::toFlatbufferUuid(idBytes); + auto graphOffset = CreateGraphDirect(builder, + "test", + DataType::FLOAT, + DataType::HALF, + DataType::BFLOAT16, + &tensors, + &nodes, + flatbuffers::nullopt, + false, + nullptr, + &uuid); + builder.Finish(graphOffset); + return builder.Release(); + } + // A bare SerializedExecutionPlan FlatBuffer, i.e. NOT wrapped in an HDGP // container. flatbuffers::DetachedBuffer makeBarePlanBlob(uint32_t version = 1, @@ -331,6 +358,36 @@ TEST_F(TestGraphAndPlanSerializationApi, ComboSerializeProducesContainerWithGrap } } +TEST_F(TestGraphAndPlanSerializationApi, ComboSerializePreservesGraphId) +{ + const auto id + = hipdnn_flatbuffers_sdk::utilities::parseUuid("01234567-89ab-4def-8123-456789abcdef"); + auto graphBlob = makeBareGraphBlobWithId(id); + hipdnnBackendDescriptor_t rawGraph = nullptr; + ASSERT_EQ( + hipdnnBackendCreateAndDeserializeGraph_ext(&rawGraph, graphBlob.data(), graphBlob.size()), + HIPDNN_STATUS_SUCCESS); + const ScopedBackendDescriptor graphDesc(rawGraph); + + makeExecutionPlanFinalized(); + expectPlanSerialization(); + + size_t size = 0; + ASSERT_EQ(hipdnnBackendGetSerializedBinaryGraphAndPlan_ext( + graphDesc.get(), _planWrapper.get(), 0, &size, nullptr), + HIPDNN_STATUS_SUCCESS); + std::vector blob(size); + ASSERT_EQ(hipdnnBackendGetSerializedBinaryGraphAndPlan_ext( + graphDesc.get(), _planWrapper.get(), size, &size, blob.data()), + HIPDNN_STATUS_SUCCESS); + + const auto graphView + = hipdnn_flatbuffers_sdk::flatbuffer_utilities::extractGraphBlob(blob.data(), blob.size()); + const auto* embeddedGraph = fbs::GetGraph(static_cast(graphView.data)); + ASSERT_NE(embeddedGraph->id(), nullptr); + EXPECT_EQ(hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*embeddedGraph->id()), id); +} + TEST_F(TestGraphAndPlanSerializationApi, ComboSerializeGraphOnlyWhenPlanDescIsNull) { auto graphBlob = makeBareGraphBlob(); @@ -567,6 +624,29 @@ TEST_F(TestGraphAndPlanSerializationApi, DeserializeGraphFromContainerExtractsGr verifyDeserializedGraphMatchesSource(desc.get()); } +TEST_F(TestGraphAndPlanSerializationApi, DeserializeGraphFromContainerPreservesId) +{ + const auto id + = hipdnn_flatbuffers_sdk::utilities::parseUuid("01234567-89ab-4def-8123-456789abcdef"); + auto graphBlob = makeBareGraphBlobWithId(id); + auto planBlob = makeBarePlanBlob(); + auto container = makeContainer({graphBlob.data(), graphBlob.data() + graphBlob.size()}, + {planBlob.data(), planBlob.data() + planBlob.size()}); + + hipdnnBackendDescriptor_t rawDesc = nullptr; + ASSERT_EQ( + hipdnnBackendCreateAndDeserializeGraph_ext(&rawDesc, container.data(), container.size()), + HIPDNN_STATUS_SUCCESS); + const ScopedBackendDescriptor desc(rawDesc); + + auto graphDesc = desc->asDescriptor(); + graphDesc->buildSerializedGraph(); + auto roundTripped = graphDesc->getSerializedGraph(); + const auto* reconstructed = fbs::GetGraph(static_cast(roundTripped.ptr)); + ASSERT_NE(reconstructed->id(), nullptr); + EXPECT_EQ(hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*reconstructed->id()), id); +} + TEST_F(TestGraphAndPlanSerializationApi, DeserializeGraphFromLegacyBareBlobUnchanged) { auto graphBlob = makeBareGraphBlob(); diff --git a/projects/hipdnn/backend/tests/TestGraphLogger.cpp b/projects/hipdnn/backend/tests/TestGraphLogger.cpp index 9046cd111ba8..598e67767d01 100644 --- a/projects/hipdnn/backend/tests/TestGraphLogger.cpp +++ b/projects/hipdnn/backend/tests/TestGraphLogger.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -118,11 +119,6 @@ TEST_F(TestGraphLogger, GraphLoggedWhenEnabled) auto jsonFiles = getJsonFilesInDir(_tempDir); ASSERT_EQ(jsonFiles.size(), 1u); - // Verify filename format: graph_<16 hex chars>.json - auto filename = jsonFiles[0].filename().string(); - EXPECT_EQ(filename.rfind("graph_", 0), 0u); - EXPECT_EQ(filename.rfind(".json"), filename.size() - 5); - // Verify the file contains valid JSON with expected graph fields std::ifstream file(jsonFiles[0]); ASSERT_TRUE(file.is_open()); @@ -131,19 +127,61 @@ TEST_F(TestGraphLogger, GraphLoggedWhenEnabled) EXPECT_TRUE(j.contains("nodes")); EXPECT_TRUE(j.contains("tensors")); EXPECT_TRUE(j.contains("name")); + ASSERT_TRUE(j.contains("id")); + + // The dump is named after the graph it holds. + EXPECT_EQ(jsonFiles[0].filename().string(), "graph_" + j["id"].get() + ".json"); } -TEST_F(TestGraphLogger, DuplicateGraphNotLoggedTwice) +TEST_F(TestGraphLogger, EquivalentGraphsAreLoggedSeparately) { hipdnn_data_sdk::utilities::setEnv("HIPDNN_LOG_GRAPH_DIR", _tempDirStr.c_str()); hipdnn_backend::logging::loggerShutdown(); - // Finalize the same graph twice + // Two independently built graphs are two graph objects with two IDs, so each is dumped. auto descriptor1 = createAndFinalizeGraph(); auto descriptor2 = createAndFinalizeGraph(); auto jsonFiles = getJsonFilesInDir(_tempDir); - EXPECT_EQ(jsonFiles.size(), 1u); + EXPECT_EQ(jsonFiles.size(), 2u); +} + +TEST_F(TestGraphLogger, ReLoggingSameFinalizedGraphIsDeduplicated) +{ + hipdnn_data_sdk::utilities::setEnv("HIPDNN_LOG_GRAPH_DIR", _tempDirStr.c_str()); + hipdnn_backend::logging::loggerShutdown(); + + auto descriptor = createAndFinalizeGraph(); + const auto serialized = descriptor.getSerializedGraph(); + logging::GraphLogger::logGraph(static_cast(serialized.ptr), serialized.size); + + EXPECT_EQ(getJsonFilesInDir(_tempDir).size(), 1u); +} + +TEST_F(TestGraphLogger, GraphWithoutIdIsNotLogged) +{ + hipdnn_data_sdk::utilities::setEnv("HIPDNN_LOG_GRAPH_DIR", _tempDirStr.c_str()); + hipdnn_backend::logging::loggerShutdown(); + + // The guard is the missing id, not the route that produced it: a legacy graph carrying no id + // reaches the logger the same way a graph serialized before finalizing does. + auto builder = hipdnn_test_sdk::utilities::createEmptyValidGraph(); + ASSERT_EQ(hipdnn_flatbuffers_sdk::data_objects::GetGraph(builder.GetBufferPointer())->id(), + nullptr); + logging::GraphLogger::logGraph(builder.GetBufferPointer(), builder.GetSize()); + + EXPECT_TRUE(getJsonFilesInDir(_tempDir).empty()); +} + +TEST_F(TestGraphLogger, MalformedGraphIsNotLogged) +{ + hipdnn_data_sdk::utilities::setEnv("HIPDNN_LOG_GRAPH_DIR", _tempDirStr.c_str()); + hipdnn_backend::logging::loggerShutdown(); + + const std::array garbage{}; + logging::GraphLogger::logGraph(garbage.data(), garbage.size()); + + EXPECT_TRUE(getJsonFilesInDir(_tempDir).empty()); } TEST_F(TestGraphLogger, DifferentGraphsLoggedSeparately) diff --git a/projects/hipdnn/backend/tests/TestPlatformUtils.cpp b/projects/hipdnn/backend/tests/TestPlatformUtils.cpp index 786068373b2d..9d3debd944d0 100644 --- a/projects/hipdnn/backend/tests/TestPlatformUtils.cpp +++ b/projects/hipdnn/backend/tests/TestPlatformUtils.cpp @@ -4,10 +4,12 @@ #include #include #include +#include #include "HipdnnException.hpp" #include "PlatformUtils.hpp" #include "TestPluginConstants.hpp" +#include TEST(TestPlatformUtils, GetSystemInfoReturnsNonEmpty) { @@ -107,3 +109,31 @@ TEST(TestPlatformUtils, OpenLibraryThrowsHipdnnExceptionForMissingLibrary) hipdnn_data_sdk::utilities::getLibraryName("hipdnn_missing_test_library")), hipdnn_backend::HipdnnException); } + +TEST(TestPlatformUtils, GenerateUuidV4ProducesValidVersion4Uuid) +{ + const auto uuid = hipdnn_backend::platform_utilities::generateUuidV4(); + EXPECT_TRUE(hipdnn_flatbuffers_sdk::utilities::isUuidV4(uuid)); +} + +TEST(TestPlatformUtils, GenerateUuidV4ProducesDistinctValues) +{ + const auto first = hipdnn_backend::platform_utilities::generateUuidV4(); + const auto second = hipdnn_backend::platform_utilities::generateUuidV4(); + EXPECT_NE(first, second); +} + +TEST(TestPlatformUtils, GenerateUuidV4ProducesManyDistinctValues) +{ + // A broken RNG (fixed seed, zeroed buffer, stale entropy pool) would collapse + // this set well below 1000; a healthy one should never collide at this scale. + constexpr int UUID_COUNT = 1000; + std::unordered_set seen; + for(int i = 0; i < UUID_COUNT; ++i) + { + const auto uuid = hipdnn_backend::platform_utilities::generateUuidV4(); + EXPECT_TRUE(hipdnn_flatbuffers_sdk::utilities::isUuidV4(uuid)); + seen.insert(hipdnn_flatbuffers_sdk::utilities::formatUuid(uuid)); + } + EXPECT_EQ(seen.size(), static_cast(UUID_COUNT)); +} diff --git a/projects/hipdnn/backend/tests/descriptors/TestGraphDescriptor.cpp b/projects/hipdnn/backend/tests/descriptors/TestGraphDescriptor.cpp index f4bd077575a9..ea8a0516acd8 100644 --- a/projects/hipdnn/backend/tests/descriptors/TestGraphDescriptor.cpp +++ b/projects/hipdnn/backend/tests/descriptors/TestGraphDescriptor.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -28,6 +29,23 @@ class TestGraphDescriptor : public ::testing::Test return test_utilities::createValidGraph(); } + static std::unique_ptr + unpack(const GraphDescriptor& descriptor) + { + const auto serialized = descriptor.getSerializedGraph(); + return hipdnn_flatbuffers_sdk::data_objects::UnPackGraph( + static_cast(serialized.ptr)); + } + + static void setHandle(GraphDescriptor& descriptor) + { + auto handle = reinterpret_cast(0x12345678); + descriptor.setAttribute(HIPDNN_ATTR_OPERATIONGRAPH_HANDLE, + HIPDNN_TYPE_HANDLE, + 1, + static_cast(&handle)); + } + static void verifyGraph(const hipdnn_flatbuffers_sdk::data_objects::GraphT& graph) { EXPECT_EQ(graph.name, "test"); @@ -650,6 +668,27 @@ TEST_F(TestGraphDescriptor, DeserializeInvalidatesSerializedBuffer) EXPECT_EQ(std::string(nameBuffer.data()), "graphB"); } +TEST_F(TestGraphDescriptor, DeserializeFailsAfterFinalizeAndPreservesIdentity) +{ + auto builder = createValidGraph(); + auto serialized = builder.Release(); + + GraphDescriptor descriptor; + descriptor.deserializeGraph(serialized.data(), serialized.size()); + setHandle(descriptor); + descriptor.finalize(); + const auto original = unpack(descriptor); + ASSERT_NE(original->id, nullptr); + const auto originalId = hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*original->id); + + ASSERT_THROW_HIPDNN_STATUS(descriptor.deserializeGraph(serialized.data(), serialized.size()), + HIPDNN_STATUS_NOT_INITIALIZED); + EXPECT_TRUE(descriptor.isFinalized()); + const auto preserved = unpack(descriptor); + ASSERT_NE(preserved->id, nullptr); + EXPECT_EQ(hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*preserved->id), originalId); +} + // ============================================================================ // Binary/JSON serialization error-path tests // ============================================================================ @@ -856,6 +895,163 @@ TEST_F(TestGraphDescriptor, BinaryRoundTripViaApi) EXPECT_EQ(*graph1, *graph2); } +TEST_F(TestGraphDescriptor, FinalizingLegacyGraphGeneratesValidStableUuid) +{ + auto builder = createValidGraph(); + auto serializedGraph = builder.Release(); + GraphDescriptor descriptor; + descriptor.deserializeGraph(serializedGraph.data(), serializedGraph.size()); + descriptor.buildSerializedGraph(); + ASSERT_EQ(unpack(descriptor)->id, nullptr); + + setHandle(descriptor); + descriptor.finalize(); + const auto first = unpack(descriptor); + ASSERT_NE(first->id, nullptr); + const auto firstId = hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*first->id); + EXPECT_TRUE(hipdnn_flatbuffers_sdk::utilities::isUuidV4(firstId)); + + const auto second = unpack(descriptor); + ASSERT_NE(second->id, nullptr); + EXPECT_EQ(firstId, hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*second->id)); +} + +TEST_F(TestGraphDescriptor, EquivalentGraphsReceiveDifferentUuids) +{ + auto firstBuilder = createValidGraph(); + auto firstSerialized = firstBuilder.Release(); + GraphDescriptor first; + first.deserializeGraph(firstSerialized.data(), firstSerialized.size()); + setHandle(first); + first.finalize(); + + auto secondBuilder = createValidGraph(); + auto secondSerialized = secondBuilder.Release(); + GraphDescriptor second; + second.deserializeGraph(secondSerialized.data(), secondSerialized.size()); + setHandle(second); + second.finalize(); + + const auto firstGraph = unpack(first); + const auto secondGraph = unpack(second); + ASSERT_NE(firstGraph->id, nullptr); + ASSERT_NE(secondGraph->id, nullptr); + EXPECT_NE(*firstGraph->id, *secondGraph->id); +} + +TEST_F(TestGraphDescriptor, ExistingUuidSurvivesDeserializeHandleAndFinalize) +{ + auto legacyBuilder = createValidGraph(); + auto legacySerialized = legacyBuilder.Release(); + GraphDescriptor original; + original.deserializeGraph(legacySerialized.data(), legacySerialized.size()); + setHandle(original); + original.finalize(); + const auto originalGraph = unpack(original); + ASSERT_NE(originalGraph->id, nullptr); + const auto expectedId = hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*originalGraph->id); + + const auto originalData = original.getSerializedGraph(); + GraphDescriptor revived; + revived.deserializeGraph(static_cast(originalData.ptr), originalData.size); + setHandle(revived); + revived.finalize(); + const auto revivedGraph = unpack(revived); + ASSERT_NE(revivedGraph->id, nullptr); + EXPECT_EQ(hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*revivedGraph->id), expectedId); +} + +TEST_F(TestGraphDescriptor, ContentMutationReplacesInheritedUuid) +{ + auto legacyBuilder = createValidGraph(); + auto legacySerialized = legacyBuilder.Release(); + GraphDescriptor original; + original.deserializeGraph(legacySerialized.data(), legacySerialized.size()); + setHandle(original); + original.finalize(); + const auto originalGraph = unpack(original); + ASSERT_NE(originalGraph->id, nullptr); + const auto originalId = hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*originalGraph->id); + + const auto originalData = original.getSerializedGraph(); + GraphDescriptor mutated; + mutated.deserializeGraph(static_cast(originalData.ptr), originalData.size); + const std::array name{'m', 'u', 't', 'a', 't', 'e', 'd', '\0'}; + mutated.setAttribute(HIPDNN_ATTR_OPERATIONGRAPH_NAME_EXT, + HIPDNN_TYPE_CHAR, + static_cast(name.size()), + name.data()); + setHandle(mutated); + mutated.finalize(); + const auto mutatedGraph = unpack(mutated); + ASSERT_NE(mutatedGraph->id, nullptr); + EXPECT_NE(hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*mutatedGraph->id), originalId); +} + +TEST_F(TestGraphDescriptor, PreservesOpaqueSerializedGraphId) +{ + auto builder = createValidGraph(); + auto serialized = builder.Release(); + auto graph = hipdnn_flatbuffers_sdk::data_objects::UnPackGraph(serialized.data()); + std::array opaqueId{}; + opaqueId[6] = 0x30; + opaqueId[8] = 0x40; + graph->id = std::make_unique( + flatbuffers::span(opaqueId)); + + flatbuffers::FlatBufferBuilder identifiedBuilder; + identifiedBuilder.Finish( + hipdnn_flatbuffers_sdk::data_objects::Graph::Pack(identifiedBuilder, graph.get())); + auto identified = identifiedBuilder.Release(); + + GraphDescriptor descriptor; + descriptor.deserializeGraph(identified.data(), identified.size()); + descriptor.buildSerializedGraph(); + const auto roundTripped = unpack(descriptor); + ASSERT_NE(roundTripped->id, nullptr); + EXPECT_EQ(hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*roundTripped->id), opaqueId); +} + +TEST_F(TestGraphDescriptor, JsonRoundTripPreservesUuid) +{ + auto legacyBuilder = createValidGraph(); + auto legacySerialized = legacyBuilder.Release(); + GraphDescriptor original; + original.deserializeGraph(legacySerialized.data(), legacySerialized.size()); + setHandle(original); + original.finalize(); + const auto originalGraph = unpack(original); + ASSERT_NE(originalGraph->id, nullptr); + const auto expectedId = hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*originalGraph->id); + + const auto json = original.getSerializedJsonGraph(); + EXPECT_EQ(nlohmann::json::parse(json).at("id"), + hipdnn_flatbuffers_sdk::utilities::formatUuid(expectedId)); + + GraphDescriptor revived; + GraphDescriptor::createFromJsonGraph(revived, json.c_str(), json.size()); + revived.buildSerializedGraph(); + const auto revivedGraph = unpack(revived); + ASSERT_NE(revivedGraph->id, nullptr); + EXPECT_EQ(hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*revivedGraph->id), expectedId); +} + +TEST_F(TestGraphDescriptor, ToStringReportsGraphId) +{ + auto builder = createValidGraph(); + auto serialized = builder.Release(); + + GraphDescriptor descriptor; + descriptor.deserializeGraph(serialized.data(), serialized.size()); + EXPECT_NE(descriptor.toString().find("id=(none)"), std::string::npos); + + setHandle(descriptor); + descriptor.finalize(); + const auto id = hipdnn_flatbuffers_sdk::utilities::formatUuid( + hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*unpack(descriptor)->id)); + EXPECT_NE(descriptor.toString().find("id=" + id), std::string::npos); +} + // ============================================================================ // HIPDNN_ATTR_OPERATIONGRAPH_IS_OVERRIDE_SHAPE_ENABLED_EXT (RFC 0008) // ============================================================================ diff --git a/projects/hipdnn/cmake/FlatBuffersGenerate.cmake b/projects/hipdnn/cmake/FlatBuffersGenerate.cmake index dc6f40d2f482..304341ee31b8 100644 --- a/projects/hipdnn/cmake/FlatBuffersGenerate.cmake +++ b/projects/hipdnn/cmake/FlatBuffersGenerate.cmake @@ -148,3 +148,58 @@ function(hipdnn_generate_flatbuffer_headers) add_dependencies(${ARG_TARGET} ${_gen_target_name}) endfunction() + +# Generate a FlatBuffers binary reflection schema (.bfbs) from one .fbs schema. +# Unlike the C++ header path (flatc --cpp), the reflection schema retains custom +# field attributes (umd_input_tensor / umd_output_tensor / umd_name), which the +# umd_registry_gen tool reads via the FlatBuffers reflection API to emit the UMD +# op-schema registry (RFC 0018 Appendix B). flatc writes .bfbs into +# the -o directory; OUTPUT MUST name that file. +# Usage: +# hipdnn_generate_bfbs( +# SCHEMA schemas/graph.fbs +# SCHEMAS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/schemas +# OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/umd/graph.bfbs +# ) +function(hipdnn_generate_bfbs) + set(_one_value_args SCHEMA SCHEMAS_DIR OUTPUT) + cmake_parse_arguments(ARG "" "${_one_value_args}" "" ${ARGN}) + + foreach(_required SCHEMA SCHEMAS_DIR OUTPUT) + if(NOT ARG_${_required}) + message(FATAL_ERROR "hipdnn_generate_bfbs: missing required argument ${_required}") + endif() + endforeach() + + if(IS_ABSOLUTE "${ARG_SCHEMA}") + set(_schema_path "${ARG_SCHEMA}") + else() + set(_schema_path "${CMAKE_CURRENT_SOURCE_DIR}/${ARG_SCHEMA}") + endif() + + get_filename_component(_out_dir "${ARG_OUTPUT}" DIRECTORY) + + _hipdnn_resolve_flatc_command(_flatc_command _flatc_dependency) + + # graph.fbs pulls in the other schemas via `include`; flatc reads them all, so + # the .bfbs must regenerate when ANY schema in the include dir changes (a + # DEPENDS on graph.fbs alone would miss edits to included tables/attributes). + # CONFIGURE_DEPENDS re-runs the glob when a schema is added or removed. + file(GLOB _schema_deps CONFIGURE_DEPENDS "${ARG_SCHEMAS_DIR}/*.fbs") + set(_depends "${_schema_path}" ${_schema_deps}) + if(_flatc_dependency) + list(PREPEND _depends ${_flatc_dependency}) + endif() + + add_custom_command( + OUTPUT "${ARG_OUTPUT}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${_out_dir}" + COMMAND "${_flatc_command}" -b --schema + -o "${_out_dir}" + -I "${ARG_SCHEMAS_DIR}" + "${_schema_path}" + DEPENDS ${_depends} + COMMENT "Generating reflection schema ${ARG_OUTPUT}" + VERBATIM + ) +endfunction() diff --git a/projects/hipdnn/docs/Environment.md b/projects/hipdnn/docs/Environment.md index 6b426692e271..22659abdb63d 100644 --- a/projects/hipdnn/docs/Environment.md +++ b/projects/hipdnn/docs/Environment.md @@ -162,7 +162,7 @@ Controls graph structure logging. When set to a non-empty directory path, graphs | (unset) | Graph logging disabled (default) | | `` | Write graph structures as JSON files to the given directory | -Graph JSON files are written to the directory specified by `HIPDNN_LOG_GRAPH_DIR`. If the directory does not exist, it is created automatically. Relative paths are resolved against the current working directory. Files are named `graph_.json` where `` is derived from the graph content, ensuring identical graphs are not duplicated. +Graph JSON files are written to the directory specified by `HIPDNN_LOG_GRAPH_DIR`. If the directory does not exist, it is created automatically. Relative paths are resolved against the current working directory. Files are named `graph_.json` after the graph's own ID, so finalizing a graph again, or replaying a serialized one, reuses a single file. A graph rebuilt from scratch is a new graph with a new ID and is written separately; point the variable at a per-process directory if many processes share one output location. This variable is independent of `HIPDNN_LOG_LEVEL` and `HIPDNN_LOG_FILE`. diff --git a/projects/hipdnn/docs/images/umd_binding_model.svg b/projects/hipdnn/docs/images/umd_binding_model.svg new file mode 100644 index 000000000000..d6a37213c6a5 --- /dev/null +++ b/projects/hipdnn/docs/images/umd_binding_model.svg @@ -0,0 +1,68 @@ + + + + + + + + + UID-centric graph + Op-schema registry + Auto-bound symbol table + + + + Node { + op = sdpa_fwd + attributes(union): + q_tensor_uid = 7 + k_tensor_uid = 8 + v_tensor_uid = 9 + o_tensor_uid = 12 + } // no inputs[]/outputs[] + tensors[uid]: + 7: dims[..] strides[..] + data_type, virtual_ + stream -> getDeviceString() + = "gfx942" (out-of-band) + + + resolve + + + + sdpa_fwd schema: + operands (required): + q -> q_tensor_uid + k -> k_tensor_uid + v -> v_tensor_uid + results: + o -> o_tensor_uid + operands (optional): + attn_mask, page_table_* + attributes: + causal_mask, dropout_.. + generated from the + flatbuffer op schema + + + bind + + + + $q -> tensor(uid 7) + $k $v $o ... + $q.dims[0..3] + $q.strides[0..3] + $q.rank $q.dtype + $q.stride_order $q.packed + $q.batch, $q.num_heads, + $q.seqlen_q, $q.head_size + (named shape dims) + $sdpa_fwd.causal_mask + $sdpa_fwd.dropout_probability + consumed by the UDD's + launch + workspace formulas + + The graph has no edges; the op-schema registry reconstructs them from shared UIDs and auto-binds the full symbol table. + diff --git a/projects/hipdnn/docs/images/umd_matcher_pipeline.svg b/projects/hipdnn/docs/images/umd_matcher_pipeline.svg new file mode 100644 index 000000000000..5a0f695b0680 --- /dev/null +++ b/projects/hipdnn/docs/images/umd_matcher_pipeline.svg @@ -0,0 +1,79 @@ + + + + + + + + + Compile once at load; run per graph at plan build + + + + Compile (once) + + + text UMD + (JSONC) + + + + + resolve names + expand layouts + parse exprs, check + + + + + constraint IR + early-out + ordered + + + + + root-op + index + + + + Run (per graph) + + + live graph + root op + + + index lookup + + + candidate UMDs + for this opcode + + + eval ordered + + + accept + + bind symbols + + + + + provider + cache + + + + Applicability-time cache + catalog (kernels that passed) + bound token state + key: engine, graph, device + written by isApplicable; read by + getDetails / workspace / buildPlan + + + + Root-opcode indexing keeps match cost sublinear in descriptor count; the provider-owned cache matches each graph once (RFC 0017 §8). + diff --git a/projects/hipdnn/docs/images/umd_symbol_binding.svg b/projects/hipdnn/docs/images/umd_symbol_binding.svg new file mode 100644 index 000000000000..6d36347fe248 --- /dev/null +++ b/projects/hipdnn/docs/images/umd_symbol_binding.svg @@ -0,0 +1,66 @@ + + + + + + + + + Live graph + UMD constraints (declarative) + Declared / bound / used + + + + + Q + + K + + V + + sdpa_fwd + + O + + + + + + + match + + + + op: sdpa_fwd + operands {q:$q, k:$k, v:$v} + results {o:$o} + {"in": ["$q.dtype", ["BFLOAT16","FP8_E4M3"]]} + {"==": ["$k.dtype", "$q.dtype"]} + {"shape": ["$q", [b, h, sq, head_size]]} + {"shape": ["$k", [b, kvh, sk, head_size]]} + {"==": ["$q.head_size", 128]} + {"==": ["$q.stride_order", [3,2,1,0]]} + {"not_present": ["$attn_mask"]} + {"hipdnn.strides_fit_u32": [...]} + {"hipdnn.sdpa_mask_consistent": [...]} + + + bind + + + + $q $k $v $o + $q.dims[0..3] + $q.strides[0..3] + $q.batch, $q.num_heads + $k.kv_heads, $v.kv_heads + $q.seqlen_q, $k.seqlen_k + $q.head_size, $k.head_size + $sdpa_fwd.causal_mask + $sdpa_fwd.dropout_.. + used by the UDD's + grid / arg formulas + + The match auto-binds every tensor, dim, stride, and attribute; dim names are per-tensor, so a cross-tensor relation is an explicit criterion. + diff --git a/projects/hipdnn/docs/rfcs/0018_UMD_PoC_Plan.md b/projects/hipdnn/docs/rfcs/0018_UMD_PoC_Plan.md new file mode 100644 index 000000000000..5896dd02249f --- /dev/null +++ b/projects/hipdnn/docs/rfcs/0018_UMD_PoC_Plan.md @@ -0,0 +1,264 @@ +# RFC 0018 UMD — Proof-of-Concept Implementation Plan + +Scope: parse a Universal Match Descriptor JSON, build a `UniversalGraphMatcher` that +matches arbitrary graphs and returns a queryable bindings object, with automatic codegen +of the op-schema registry from the `umd_*` annotations. **SDPA forward only.** Driven by a +gtest suite (graphs built in code or from JSON→flatbuffer). Not plumbed as a provider engine. + +Proving out **the bindings** — the schema-generated, strongly-typed path→value layer that turns a +UMD variable reference (`$q.head_size`, `$sdpa_fwd.dropout_probability`, name→tensor) into a concrete +value read off the flatbuffer graph — is a first-class deliverable of this PoC, not a side effect of +matching. The phases below generate that layer at build time and prove it end-to-end. + +Source RFC: `projects/hipdnn/docs/rfcs/0018_UniversalMatchDescriptor.md`. + +--- + +## What already exists (reuse, don't rebuild) + +- **JsonLogic** — full compile-once/evaluate-many implementation at + `dnn-providers/hip-kernel-provider/include/hip_kernel_provider_common/JsonLogic.hpp`. + Already implements the RFC `$`-sigil variable convention, `Value` variant + (bool/int/double/string/array), operators `==`/`!=`/`<`/`<=`/`>`/`>=`/`in`/`and`/`or`/`!`/ + arithmetic/`min`/`max`/`ceil_div`/`abs`/`pow`/`log2`/`rsqrt`/`value_or_default`/`if`, and + `compile(rule)` → `Expression` over any `getData(path)->Value` data source. + Tests already include a `Umd0018ConstraintShapes` case (`src/tests/core/TestJsonLogic.cpp`). + **Gap: no `shape` short-hand (lowered by the UMD compiler, Phase 1). The operator set is closed — + there is no custom-operation hook to add; see Scope decisions.** +- **UMD annotations already in the schema** — the table-level `umd_opcode` shorthand (e.g. + `SdpaAttributes (umd_opcode: "sdpa_fwd")`) and the field-level `umd_input_tensor` / `umd_output_tensor` / + `umd_name` are declared once (`flatbuffers_sdk/schemas/data_types.fbs`) and applied on the SDPA + table + its UID fields (`flatbuffers_sdk/schemas/sdpa_attributes.fbs`). But flatc runs `--cpp` only + (`cmake/flatc_flags.txt`); no `.bfbs` is emitted, so the annotations are invisible at + build/runtime. **Gap: no `.bfbs` emit, no reflection-driven registry generator.** +- **Graph model** — `IGraph` / `GraphWrapper` + (`flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/flatbuffer_utilities/GraphWrapper.hpp`), + `INodeWrapper::attributesAs()`, `getTensorMap()` (uid→`TensorAttributes*`). + JSON→flatbuffer graph via `hipdnn_flatbuffers_sdk::json::to(builder, json)` + (`utilities/json/Graph.hpp`). In-code SDPA graph builder `createValidSdpaFwdGraph(...)` in + `test_sdk/include/hipdnn_test_sdk/utilities/FlatbufferGraphTestUtils.hpp:2218`. +- **Test infra** — GoogleTest only. Provider suite `hip_kernel_provider_tests` already links + `nlohmann_json` and the SDPA `plan_utils` (`getMaskType`, `byteStrideFitsU32`). + `src/tests/engines/asm_sdpa_engine/TestSdpaFwdPlanBuilder.cpp` is the closest template. +- **Parity oracle** — `SdpaFwdPlanBuilder::isApplicable` + (`dnn-providers/hip-kernel-provider/src/engines/asm_sdpa_engine/plans/SdpaFwdPlanBuilder.cpp:167-296`): + the declarative gates the UMD must reproduce. Arch (gfx942/gfx950) and kernel-table lookup are + **out** of the UMD per RFC §5/§8 (pack property + KDP Launch). + +--- + +## Home / layout + +Provider-local, under `dnn-providers/hip-kernel-provider` (reuses JsonLogic + SDPA `plan_utils`). +Registry generation lives in `flatbuffers_sdk` (schema owner), emitting a jlogic-agnostic header +the provider consumes. Matcher is exercised directly by tests — not registered as an engine. + +--- + +## Phase 0 — Op-schema registry codegen (RFC Appendix B) + +Goal: generate, from the `umd_*` annotations, a C++ registry keyed by opcode — the table's +`umd_opcode` shorthand (e.g. `sdpa_fwd`), falling back to the `NodeAttributes` union member name when +absent. Each entry also carries the table name (diagnostics) and the integer `NodeAttributes` value +(O(1) lookup against `Node::attributes_type()`), and lists input/output tensors (name, optionality, +UID reader) and scalar attributes (name, optionality, typed reader). + +1. **Emit `graph.bfbs`.** Add a dedicated flatc invocation (NOT via `flatc_flags.txt`, which is + `--cpp`-only) in `projects/hipdnn/cmake/FlatBuffersGenerate.cmake` — a sibling function + `hipdnn_generate_bfbs(graph.fbs)` running + `flatc -b --schema -I -o graph.fbs`; the custom `umd_*` attributes are retained in + the `.bfbs` with no extra flag. `graph.fbs` transitively covers every attribute table + the `umd_*` + declarations. Wire into `flatbuffers_sdk/CMakeLists.txt`. +2. **Generator tool.** Small standalone C++ executable `umd_registry_gen` linking flatbuffers + reflection (`flatbuffers/reflection.h`). Loads `graph.bfbs`, enumerates `NodeAttributes` union + members (opcode→table); reads the table's `umd_opcode` shorthand and applies B.3 classification + per field: + - `umd_input_tensor` + `umd_name` → input tensor name; `umd_output_tensor` + `umd_name` → output tensor name + (type MUST be `long`, `umd_name` non-empty). + - neither flag → scalar attribute, bind-named by field name. + - optionality derived from `= null` default (not re-annotated). + - **Fail the build** on B.3 violations (both flags on one field, `umd_name` without a flag, + flag on a non-integer field, duplicate `umd_name` within an op, a name colliding with a + reserved token `graph`/`kernel`/`device`, or a duplicate `umd_opcode` across ops). +3. **Emit `op_schema_registry_generated.hpp`** (header-only: inline per-op tables + inline lookup + functions, so `flatbuffers_sdk` stays INTERFACE-only) into the committed `flatbuffers_sdk` include + dir (`include/hipdnn_flatbuffers_sdk/umd/`). The neutral types live in a hand-written + `umd/OpSchemaRegistry.hpp`. + Neutral (jlogic-agnostic) shape so `flatbuffers_sdk` needn't depend on the provider: per opcode, + arrays of `{name, optional, UID-reader int64_t(const void*)}` and + `{attr-name, optional, AttrType, reader→neutral ScalarValue}`. Readers use the generated typed + accessors (`&SdpaAttributes::q_tensor_uid`, …) — **no runtime reflection** (RFC B.4). The + generator is general (it emits an entry for every `NodeAttributes` member); SDPA is the only + annotated op today, so the `SdpaAttributes` entry is the one carrying edges — that entry proves the + codegen. Each entry carries its `umd_opcode` shorthand (fallback: table name), the table name, and + the integer `NodeAttributes` value, so the matcher resolves it by `Node::attributes_type()` (O(1)), + and the UMD compiler resolves a node `op: "sdpa_fwd"` by the shorthand. + - Enum-typed scalar attributes surface as `AttrType::Dtype` carrying the enum-value name string + (via the generated `EnumName…`). Unannotated non-scalar fields (vectors/sub-tables) are not UMD + scalars and are skipped; SDPA's attribute table is all UID + scalar, so it is fully covered. + - Each attr entry's `AttrType` maps to a jlogic `ValueKind` (Int/Float/Bool/Dtype), consumed by the + Phase 1 compiler for compile-time criteria type-checking (A.10 §9). This carries strong typing + from the schema through to expression validation and is the payoff that makes the codegen worth + more than a hand-written table. +4. Wire `umd_registry_gen` as an `add_custom_command`/target; make it a dependency of + `hipdnn_flatbuffers_sdk` (the provider consumes the header transitively), gated on + `HIPDNN_GENERATE_SDK_HEADERS` (default ON) like the existing flatc header generation. + +**Verify:** registry unit test — `SdpaAttributes` entry lists Q/K/V (required operands), O (required +result), `attn_mask`/`page_table_k`/`page_table_v` (optional operands), and scalar attrs +(`dropout_probability` optional, `alibi_mask`/`padding_mask`/`causal_mask` with correct optionality). +A B.3-violation fixture fails generation. +- **Accessor value round-trip (proves the bindings, not just the shape):** build an `SdpaAttributes` + with known UIDs and scalars; assert every generated UID reader and every scalar reader returns the + exact value. Registry *shape* (names listed) is necessary but not sufficient — this proves the + generated accessors actually read the right field. +- **Optionality parity with the header:** the generator's `= null`-derived optionality matches the + generated header's `Optional` fields (an absent optional scalar reports not-present, a required + scalar always present). + +--- + +## Phase 1 — Bindings system + UMD compiler + matcher + +1. **`BindingContext`** (the queryable "bindings object" and the JsonLogic data source). Built per + graph after structural match; resolves all five namespaces via `getData`: + - **Tensor**: `$q` (→ uid int64), `$q.uid`, `$q.rank` (`dims->size()`), `$q.dtype` (enum name + string), `$q.dims[i]`, `$q.strides[i]`, `$q.` (via compiler-supplied name→index map), + `$q.stride_order` (IntArray in the RFC 0017 §5 outermost-first form, inverted from + `extractStrideOrder`'s stride-rank vector, `ShapeUtilities.hpp:146`), + `$q.packed` (bool, from dims+strides), `$q.virtual`, `$q.present` (optional operands). + Absent-optional field access resolves to unknown, which propagates rather than coercing, so the + criterion containing it declines (fail closed, A.4). + - **Graph**: `$graph.node_count`, `$graph.is_override_shape_enabled`. + - **Attributes**: `$.`, `$..present`, `$` (→ node index + for attribute reads). + - **Device**: `$device.` from `Handle` (`lds_size`, `warp_size`, …). + (Kernel namespace omitted — SDPA fwd criteria references none.) + - **Binding architecture (the two resolvers).** Edge and scalar-attr resolution use the Phase 0 + *generated* typed accessors — name→UID→tensor via `getTensorMap()`, and `$.` via the + typed reader. The Tensor-namespace path resolver (`dims[i]`/`strides[i]`/`rank`/`stride_order`/ + `packed`/`virtual`/`uid`) is hand-written **once** over the single `TensorAttributes` shape, since + there is exactly one tensor table. jlogic `Value` is the sole type-erasure boundary: strong typing + lives inside each accessor and is erased exactly once, at its return. +2. **UMD compiler.** Parse `{schema,id,name,allow_override_shape,nodes,criteria}`; validate the A.10 + subset (schema exact, single key per op-object, names resolve in registry, `?` ↔ registry + optionality, node-id/tvar disjoint from reserved roots, single-producer per variable). + JSON→JSON lowering pass: + - Expand layout aliases (`"nhwc"`→`[0,2,3,1]`, `"nchw"`→`[0,1,2,3]`, …, A.8). + - Expand `shape` (A.5): record per-tensor `{name→index, rank, capture}`; rewrite + `{"shape":["$q",[...]]}` → `{"==":["$q.rank",N]}` (no capture) / `{">=":["$q.rank",N-1]}` + (capture). Named-dim reads then resolve through `BindingContext`. + - `jlogic::compile(loweredCriteria)`; collect referenced symbols + (`Expression::variables()`) → published bound-symbol set; validate each resolves against + registry + shape maps (RFC §4 build check). + - **Static criteria type-check (A.10 §9).** Using the generated `AttrType`/tensor-field kinds from + Phase 0, verify each operator's argument types at compile (`$q.dtype` is `Dtype`, `$q.head_size` + is `Int`, `$q.stride_order` is `IntArray`); a type mismatch is a compile error, not a runtime + decline. This is where the generated strong typing pays off in the compiler. +3. **`UniversalGraphMatcher`.** Root-opcode index (opcode→compiled UMDs). `match(handle, graph)`: + - structural: locate node(s) by opcode (SDPA: the single node), build per-graph + uid→producer/consumer index, bind names (decline on missing required tensor), honor + `allow_override_shape` gate. + - construct `BindingContext`, evaluate criteria `Expression` → bool. + - return `MatchResult{ matched, bindings (BindingContext), umd_id }`. Bindings queryable + post-match (`bindings.get("$q.head_size")`). + +--- + +## Phase 2 — Test suite (the deliverable driver) + +New gtest source added via `target_sources` in +`dnn-providers/hip-kernel-provider/src/tests/engines/asm_sdpa_engine/CMakeLists.txt` (already links +nlohmann_json + plan_utils; add dep on the generated registry). + +- **Graph fixtures:** both paths — manual (`createValidSdpaFwdGraph(...)`, head_size=128, bf16) and + JSON (`json::to` from an inline JSON fixture) → `GraphWrapper`. +- **UMD fixture:** the §18 SDPA-forward descriptor JSON (head_size 128, bf16/fp8). The two gates that + need real C++ (`strides_fit_u32`, `sdpa_mask_consistent`) are not criteria at all — under RFC 0018 + §8 they belong to a native matcher beside the descriptor — so they are held constant in the graph + battery. Their native matcher is not written for this PoC, which is unplumbed. +- **Match-equivalence (RFC §16 primary):** battery of accepting/rejecting graphs through both + `SdpaFwdPlanBuilder::isApplicable` and the matcher; assert identical accept/reject on the + declarative gates. Hold device=gfx942, a valid kernel key, and the two non-declarative gates + (uint32 stride fit, mask self-consistency) constant across the battery — arch/kernel-table are + outside the UMD per §5/§8, and the custom-op gates are out of scope; all four are held explicitly, + not silently skipped. +- **Binding proof (RFC §4/§15, primary deliverable):** the auto-binding layer, proven end-to-end: + - *Completeness (§4 "complete symbol table for free"):* after a successful match the published + symbol table contains every operand/result tensor and, for each, `uid`/`rank`/`dtype`/`dims[i]`/ + `strides[i]`/`stride_order`/`packed`/`virtual`, plus every scalar attr of the matched node. Assert + against the enumerated expected set, not a sampled few. + - *Binding view (§15):* dump the full bound table as an inspectable artifact and assert its contents. + - *Path resolution per form:* `$q`, `$q.uid`, `$q.rank`, `$q.dtype`, `$q.dims[i]` (positional), + `$q.head_size` (named), `$q.strides[i]`, `$q.stride_order`, `$q.packed`, `$q.virtual`, + `$attn_mask.present`==false, `$sdpa_fwd.dropout_probability` + `.present`, `$graph.node_count`, + `$device.lds_size` — each resolves to the expected typed value. + - *Generated-accessor correctness through a live graph:* name→UID→`TensorAttributes`→field returns + the graph's actual values, tying the Phase 0 accessors to the Phase 2 resolver on real data. + - *Bad-path fail-closed:* out-of-range `dims[i]`, unknown dim-name, absent-optional field read → + decline, never a wrong value. +- **Fail-closed:** unknown symbol, absent-optional field access, `node_count`≠1 → decline (never + match by default). +- **Compiler validation:** bad schema version, `?` on a required name, unknown name → compile error / + quarantine. + +--- + +## Scope decisions & deferrals + +- **Only SDPA forward.** Registry generator is general but only SDPA is annotated, so it is the only + exercised opcode. +- **`shape` lowered in the UMD compiler**, not JsonLogic — keeps the shared language pure; matches + RFC §10 "compile expands short-hands." +- **JsonLogic core changes are limited to the presence predicates.** `present` / `not_present` are + added because RFC 0017 §5 lists them in the closed operator vocabulary and they are the only way to + write "absent, or present and constrained" — a bare field read on an absent operand declines rather + than answering. `all` / `rank`-op / `divisible` are unused by SDPA fwd — deferred (add if broader + §A.7 coverage is wanted). +- **The operator set is closed, permanently.** RFC 0018 §8 puts the C++ escape hatch in a native + matcher named by a `MatchDescriptor`'s `matchSymbol`, conjoined with the UMD's criteria by the + ingestor, rather than in a custom operation nested inside `criteria`. So a namespaced (dotted) + operator key needs no special handling: it is refused as an unrecognized operator like any other + (`TestUmdCompiler.RejectsUnknownOperator`) — that is the finished behavior, not a PoC stub awaiting + a predicate registry. +- **Not plumbed as a provider engine** — matcher is exercised directly by tests. +- **Deferred (RFC-acknowledged):** static/bytecode matcher (§11), fuzzing (§14), arbitration across + multiple UKDs (§12), UDD dispatch formulas, multi-node fusion patterns, and the whole + applicability-time catalog and cache (RFC 0017 §8) — the PoC matcher answers one UMD against one + graph and holds no catalog. +- **Deferred pack-level checks.** Two rules RFC 0018 §5 states are enforced by a KDP/KMD loader, not + by a single descriptor's compile: that every quantity a kernel bakes is a KMD field pinned by a + `$kernel.*` criterion, and that every pack carries an umbrella matcher checking the full graph + topology. Neither a KDP nor a KMD loader exists in this tree, so both are out of scope. The + compiler does its half: it publishes the `$kernel.*` fields a matcher reads, which is what such a + loader needs. +- **Version floors are structural only.** `version` and `sdk_version` parse and gate against a + compiled-in floor, but nothing yet publishes a real graph-schema version to compare against, so + the floor is a constant rather than a negotiated value. +- **Two non-declarative gates out of the UMD.** `strides_fit_u32` and `sdpa_mask_consistent` are a + native matcher's job (RFC §8), not a descriptor's; match-equivalence covers the declarative subset + with those two gates held constant, exactly as arch/kernel-table are. +- **Open follow-up: native-matcher bindings.** A `GraphMatcherFn` receives `MatchContext`, not the + bindings its pack's UMD published, and `BoundTokens` (`string -> int64_t`) cannot carry a tensor — + so a native matcher re-locates q/k/v/o by hand. Recorded as RFC 0018 §20 open question 2; it is a + change to the matcher-invocation contract, outside this PoC. + +## Primary risks + +- **Reflection availability/version** — flatbuffers 25.9.23 ships `reflection.h`; `graph.bfbs` must + compile cleanly with the pinned flatc. Verify early (Phase 0 gate). +- **Parity honesty** — the UMD deliberately omits arch, kernel-table, and (with the hook skipped) the + two custom-op gates; equivalence is over the declarative subset. Tests must fix those variables, + not fudge them. +- **`shape` name-binding ordering** — resolved by static compile-time lowering (name maps are + UMD-static, graph-independent), avoiding mutation during eval. + +## Open questions for iteration + +- Registry generator home: `flatbuffers_sdk` (schema owner, neutral output) vs. provider-local + (more contained for a PoC). Plan assumes `flatbuffers_sdk`. +- Include `all` / `divisible` / `rank`-op now for fuller A.7 coverage, or defer? +- Neutral `ScalarValue` type location for the generated registry (shared header vs. `flatbuffers_sdk`). +- Static criteria type-checking (A.10 §9) via the generated `AttrType` in the PoC, or defer to the + full matcher? Plan includes it as part of the binding proof, since it is the direct payoff of the + strongly-typed codegen. diff --git a/projects/hipdnn/docs/rfcs/0018_UniversalMatchDescriptor.md b/projects/hipdnn/docs/rfcs/0018_UniversalMatchDescriptor.md new file mode 100644 index 000000000000..c69668b2f4dd --- /dev/null +++ b/projects/hipdnn/docs/rfcs/0018_UniversalMatchDescriptor.md @@ -0,0 +1,1441 @@ +# RFC 0018: Universal Match Descriptor (UMD) and the Graph Matcher + +- Contributors: Brian Harrison + +> Follow-up to [RFC 0017 (Universal Kernel Descriptors)](0017_UniversalKernelDescriptor.md), +> the "UMD + graph matcher" row of its follow-up series ([RFC 0017 §14.2](0017_UniversalKernelDescriptor.md#142-follow-up-rfcs)). +> This RFC designs the match format and the matcher; the sibling formats (UDD, UED, UHD, KDP) and +> subsystems (packaging, drop-in, adapters) are designed in their own follow-ups and are referenced, +> not redesigned, here. The RFC number is provisional and is reconciled against the concurrent +> follow-up series at PR-open time. + +## Table of Contents + +1. [Overview](#1-overview) +2. [The Matcher's Input: hipDNN's Graph Model](#2-the-matchers-input-hipdnns-graph-model) +3. [Structural Pattern](#3-structural-pattern) +4. [Symbol Binding and the Auto-Binding Formula](#4-symbol-binding-and-the-auto-binding-formula) +5. [Constraint Vocabulary](#5-constraint-vocabulary) +6. [The Shared Expression Language](#6-the-shared-expression-language) +7. [Layout and Stride-Order Constraints](#7-layout-and-stride-order-constraints) +8. [The Native-Matcher Escape Hatch](#8-the-native-matcher-escape-hatch) +9. [Composite Constraints](#9-composite-constraints) +10. [The Matcher: Compilation, Indexing, and Caching](#10-the-matcher-compilation-indexing-and-caching) +11. [Static Matcher (Sketch)](#11-static-matcher-sketch) +12. [Arbitration](#12-arbitration) +13. [Serialization and Versioning](#13-serialization-and-versioning) +14. [Security and Hostile Input](#14-security-and-hostile-input) +15. [Observability and Diagnostics](#15-observability-and-diagnostics) +16. [Testing and Performance](#16-testing-and-performance) +17. [Migration](#17-migration) +18. [Worked Example: SDPA Forward](#18-worked-example-sdpa-forward) +19. [Risks](#19-risks) +20. [Open Questions](#20-open-questions) +21. [References and Prior Art](#21-references-and-prior-art) +22. [Glossary](#22-glossary) +23. [Appendix A: Schema Reference](#appendix-a-schema-reference) +24. [Appendix B: Op-Schema Registry Generation](#appendix-b-op-schema-registry-generation) + +--- + +## 1. Overview + +A **UMD (Universal Match Descriptor)** is the declarative data that decides whether a kernel applies to +an incoming problem graph, and, in the same pass, binds the named variables that the kernel's dispatch +and workspace formulas reference. It replaces the graph half of a hand-coded +`IPlanBuilder::isApplicable` check ([RFC 0017 §2](0017_UniversalKernelDescriptor.md#2-the-descriptors)). +One UMD is authored once and referenced by ID; a KDP lists a set of matcher IDs and a kernel applies +only when all pass ([RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd)), so a +family of near-identical kernels shares a handful of matchers rather than carrying a bespoke C++ check +each. + +A UMD has two parts, unchanged in intent from +[RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd): + +1. A **structural pattern**: named operation nodes and their named operand and result edges. Because + hipDNN op graphs are DAGs, the pattern is an explicit node-and-edge graph, not a nested expression. +2. A **criteria expression** over the pattern's bound variables: one JsonLogic boolean, typically an + `and` of the individual tests. + +This document turns that frame into a concrete format and a concrete matcher. It specifies the pattern +and criteria schema, the standard formula that auto-binds every tensor and attribute of a matched op, +JsonLogic as the shared expression language (boolean criteria here, dispatch formulas in the UDD), +the layout representation as a stride-order index array, the native-matcher escape hatch, +deterministic arbitration, and the compile-once matcher with its indexing and caching. The static +(compile-time) matcher is sketched as options, not fully designed, in this iteration. + +### 1.1 What This RFC Specifies Versus Defers + +| Capability | This RFC (day-one) | Deferred | +|---|---|---| +| Structural pattern: op nodes, named operand/result edges, single-op and bounded fused subgraphs that are the entire graph | Yes ([§3](#3-structural-pattern)) | Matching a fused pattern *inside* a larger graph: JIT follow-up | +| Auto-binding of every operand/result tensor, its dims and strides, and every op attribute | Yes ([§4](#4-symbol-binding-and-the-auto-binding-formula)) | None | +| Criteria as one JsonLogic expression: opcode, dtype (exact/set/relation), shape/rank, divisibility, stride order, packed, attribute, graph-structure (`node_count`, `virtual`), cross-tensor relation, optional operand, device property | Yes ([§5](#5-constraint-vocabulary)) | None | +| JsonLogic as the shared expression language (UMD boolean + UDD value), `$`-variable convention | Yes ([§6](#6-the-shared-expression-language)) | New operators as needed | +| Layout as a stride-order index array, with named aliases | Yes ([§7](#7-layout-and-stride-order-constraints)) | None | +| Escape hatch for checks needing C++, beside the descriptor rather than inside the expression language | Yes ([§8](#8-the-native-matcher-escape-hatch)) | None | +| Composite criteria: `(A AND B) OR C` as one criteria expression, via JsonLogic `and`/`or`/`!`/`if` | Yes ([§9](#9-composite-constraints)) | None | +| Compile-once matcher, root-opcode index, applicability-time match cache | Yes ([§10](#10-the-matcher-compilation-indexing-and-caching)) | None | +| Static (compile-time / AOT-lowered) matcher | Options sketched ([§11](#11-static-matcher-sketch)) | Full design | +| General N-ary commutative matching, unbounded variable-length chains | None | JIT follow-up ([RFC 0017 §9.3](0017_UniversalKernelDescriptor.md#93-future-jit-and-normalized-providers)) | + +--- + +## 2. The Matcher's Input: hipDNN's Graph Model + +The matcher reads an immutable graph through the existing `IGraph` interface +(`projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/flatbuffer_utilities/GraphWrapper.hpp`) +plus the HIP stream for device properties. Three properties of that model drive the design. + +**The graph is UID-centric, not edge-centric.** A `Node` carries only `{name, compute_data_type, +attributes_type, attributes}` (`graph_generated.h`, Node table). It has no input or output tensor +lists. A node's operands and results are UID fields inside its concrete attribute table, for example +`SdpaAttributes::q_tensor_uid()`, `k_tensor_uid()`, `o_tensor_uid()` (`sdpa_attributes_generated.h`). +Connectivity between nodes is implicit: two nodes are connected when a result UID of one appears as an +operand UID of the other. To resolve a node's edges, the matcher must know the op type, cast via +`attributesAs()`, and read the named UID fields. + +**Consequence: the matcher needs an op-schema registry.** For each op type, the registry declares which +attribute fields are operand UIDs, which are result UIDs, whether each is required or optional, and the +names of the op's scalar attributes. This registry is what lets a UMD reference operands and results +by name (`q`, `k`, `v`, `o`) and what powers the auto-binding formula of +[§4](#4-symbol-binding-and-the-auto-binding-formula). + +**The registry is generated from schema annotations, not name conventions.** A table-level FlatBuffers +attribute names the op, and field-level attributes on each op's attribute table declare the binding +contract next to the field they govern. `umd_opcode` on the attribute table gives the op's UMD-facing +opcode (`SdpaAttributes (umd_opcode: "sdpa_fwd")`); a UMD node's `op` names it, and the registry keys +on it (falling back to the table type name when the attribute is absent), so the schema is the single +source of truth for the opcode rather than an ad-hoc string. `umd_input_tensor` / `umd_output_tensor` +mark a UID field and `umd_name` names it, so SDPA's Q operand is +`q_tensor_uid: long (umd_input_tensor, umd_name: "q")` and its O result is +`o_tensor_uid: long (umd_output_tensor, umd_name: "o")`. Optionality is not re-annotated: +a UID field's `= null` default already encodes it (`attn_mask_tensor_uid: long = null`), and the +`NodeAttributes` union already maps each opcode to its table, so every unannotated *scalar* field is a +scalar attribute by elimination (unannotated non-scalar fields — vectors, sub-tables — are not +bindable scalars and are skipped). A build step emits the binary +reflection schema (`graph.bfbs`, which transitively covers every attribute table; custom attributes +surface only through reflection, not the generated headers) and a generator reads each field's +attributes to emit the registry, so it stays in lockstep with the graph definitions rather than being +hand-maintained ([Appendix B](#appendix-b-op-schema-registry-generation) specifies the annotation contract, the field-classification rules, and the generation pipeline). Names are never inferred from the `_tensor_uid` name suffix, which would misclassify +non-UID fields such as `PointwiseAttributes::axis_tensor_uid` (a plain axis index, not a tensor UID). + +**Tensors expose dims, strides, dtype, and a virtual flag, but no layout enum and no rank field.** +`TensorAttributes` (`tensor_attributes_generated.h`) offers `dims()`, `strides()` (both nullable +vectors), `data_type()`, `uid()`, and `virtual_()`. Rank is `dims()->size()`. Layout is not stored; it +is derived from the stride order, which is why the UMD represents layout as a stride-order index array +([§7](#7-layout-and-stride-order-constraints)). Quantities like `head_size`, `batch`, and `num_heads` +are **not** attributes; they are specific tensor dims (for SDPA, `q.dims[3]`, `q.dims[0]`, `q.dims[1]`). +The UMD binds them as named shape dims (`$q.head_size`), not as attribute reads, matching RFC 0017's +`shape` short-hand ([§4](#4-symbol-binding-and-the-auto-binding-formula)). + +**Device and arch are out-of-band.** The graph carries no device identity. Arch comes from the stream +via `getDeviceString(handle.getStream())` (`HipDeviceUtils.hpp:48`); for AOT it gates *pack selection*, +not a match criterion ([§5](#5-constraint-vocabulary)). Other device properties resolve against the +`Handle` the matcher receives alongside the graph, read as `$device.` rather than a graph field. + +**Graph guarantees the matcher may rely on.** Per the `IGraph` contract the graph is topologically +sorted, acyclic, fully connected, and has unique tensor UIDs. The matcher builds its own +UID-to-producer and UID-to-consumers index once per graph to walk edges and reconstruct connectivity, +since no adjacency query is provided; fusion legality reads each intermediate's `virtual` flag. + +![The matcher reads a UID-centric graph via an op-schema registry that reconstructs edges and auto-binds symbols](../images/umd_binding_model.svg) + +--- + +## 3. Structural Pattern + +The pattern is a set of op nodes and the named edges between them and the graph's tensors. Each node +declares its opcode and maps operand and result **names** (from the op-schema registry) to pattern +variables (`$q`, `$conv_out`). + +```jsonc +{ + "schema": "hipdnn.umd/v1", + "id": "9c3f5b2a-7d41-4e88-b6a0-1f2e3d4c5b6a", + "name": "SDPA forward (d128, bf16) match", + "nodes": [ + {"kind": "op", "id": "sdpa_fwd", "op": "sdpa_fwd", + "operands": {"q": "$q", "k": "$k", "v": "$v"}, "results": {"o": "$o"}} + ], + "criteria": { /* Section 6: one JsonLogic boolean expression */ } +} +``` + +- **Node identity.** Each node has a pattern-local `id` (`sdpa_fwd`, `conv`, `add`) used to read the + node's attributes (`$.`, [§4](#4-symbol-binding-and-the-auto-binding-formula)) and in + diagnostics. It is distinct from the descriptor's global `id`. +- **Opcode.** `op` names one opcode — the op's `umd_opcode` shorthand from the schema + ([§2](#2-the-matchers-input-hipdnns-graph-model), [Appendix B](#appendix-b-op-schema-registry-generation)), + e.g. `sdpa_fwd` — or `{"one_of": [...]}` for a small fixed set, or `"any"` for a wildcard node (used + only inside a bounded fused pattern). +- **Names.** Keys in `operands` and `results` are op-schema tensor names; values are pattern variables. + A name the schema marks optional is bound with a `?` suffix (`"attn_mask": "$attn_mask?"`), bound only + when the graph supplies it, and read with a default via `value_or_default` + ([§4](#4-symbol-binding-and-the-auto-binding-formula)). Names not listed are ignored for matching but + still auto-bound. +- **Edges are implicit through shared variables.** Two nodes are connected when the same variable + appears as a result of one and an operand of another. In the fused example below, `$conv_out` is + `conv`'s result and `add`'s operand, which is the edge. +- **Subgraph matching by default; exact match is explicit.** The pattern matches a connected subgraph, + which may sit inside a larger graph. A prebuilt kernel that is the whole graph pins the exact op count + with `{"==": ["$graph.node_count", N]}` ([§5](#5-constraint-vocabulary)): a single-op kernel checks + `node_count == 1`, replacing the `nodeWrappers().size() != 1` gate; a fused three-op kernel checks + `node_count == 3`. Matching a pattern inside a larger graph without a fixed count is the looser JIT + mode ([RFC 0017 §9.3](0017_UniversalKernelDescriptor.md#93-future-jit-and-normalized-providers)). +- **Override-shape graphs** are declined by default (mirroring today's gate); a UMD that supports them + sets `"allow_override_shape": true` at the top level. + +**Fusion matching is day-one, for a pattern that is the whole graph.** A multi-node pattern binds the +whole fused subgraph at once and hands it to one UKD. Legality — that the fused intermediates are +absorbed into the kernel, not consumed outside it — is expressed by requiring each intermediate +`virtual` and pinning the op count with `$graph.node_count`, which also confines a day-one fused +pattern to the entire graph. Matching a fused pattern *inside* a larger graph, with no fixed count, +is the JIT follow-up's looser mode. Fusion is distinct from composition — running one graph as +several kernels — which goes the opposite direction and is not in scope here: + +```jsonc +{ + "schema": "hipdnn.umd/v1", + "id": "2b7e1c44-9a0d-4f13-8c21-6e5d4a3b2c10", + "name": "Conv-Bias-ReLU (NHWC, f16) match", + "nodes": [ + {"kind": "op", "id": "conv", "op": "convolution_fwd", + "operands": {"x": "$x", "w": "$w"}, "results": {"y": "$conv_out"}}, + {"kind": "op", "id": "bias", "op": "pointwise_add", + "operands": {"a": "$conv_out", "b": "$bias"}, "results": {"y": "$bias_out"}}, + {"kind": "op", "id": "act", "op": "pointwise_relu", + "operands": {"a": "$bias_out"}, "results": {"y": "$y"}} + ], + "criteria": {"and": [ + {"==": ["$x.dtype", "FLOAT16"]}, {"==": ["$x.stride_order", [0, 2, 3, 1]]}, "$x.packed", // NHWC + {"==": ["$y.dtype", "FLOAT16"]}, {"==": ["$y.stride_order", [0, 2, 3, 1]]}, "$y.packed", // NHWC + {"shape": ["$y", ["batch", "out_channels", "out_h", "out_w"]]}, // logical dim order; layout is the stride_order above + {"shape": ["$bias", ["out_channels"]]}, + {"==": ["$graph.node_count", 3]}, // exactly these three ops + "$conv_out.virtual", "$bias_out.virtual" // intermediates absorbed by the fused kernel + ]} +} +``` + +Out of scope for this iteration, as in [RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd): +general N-ary commutative matching and unbounded variable-length chains. Bounded commutative pairs and +bounded optional slots cover the fusion cases a prebuilt kernel needs. + +--- + +## 4. Symbol Binding and the Auto-Binding Formula + +Matching does double duty: it decides applicability and it binds named variables. A symbol is +**declared** in the UMD, **bound** when the graph matches, and **used** by the UDD's dispatch and +workspace formulas ([RFC 0017 §6](0017_UniversalKernelDescriptor.md#6-dispatch-and-workspace)). Every +symbol a formula references must be bound by the match, so a formula can only read values the match +actually produces. The UMD publishes its bound-symbol set; each UKD that pairs this UMD with a UDD is +checked at build and at drop-in load, and a UDD that references an unbound symbol is rejected then +rather than failing closed later on a live graph. + +**Auto-binding is the default, and follows a standard formula (AICK-1698).** When a pattern names an +operand or result variable, the matcher, using the op-schema registry, automatically binds it and its +fields, so authors get a complete symbol table for free and never hand-declare each field. Every field a +criteria or dispatch expression may reference falls in one of **five namespaces**, and the hipDNN schema +declares them so the interpreter fails closed on anything undeclared: + +- **Tensor** — a bound operand/result and its fields: `$q` is the whole tensor (the matched + `TensorAttributes`) and `$q.uid` its graph UID; each + dim positionally as `$q.dims[i]` and by name as `$q.` once a `shape` names it (`$q.seqlen_q`); + each stride as `$q.strides[i]`; and the derived facts `$q.rank`, `$q.dtype`, `$q.stride_order` + ([§7](#7-layout-and-stride-order-constraints)), `$q.packed`, `$q.virtual` (an internal + intermediate between matched nodes, not a graph input or output), + `$q.is_runtime_pass_by_value` (its value arrives per execution rather than being baked into the + graph, [RFC 0016](0016_RuntimePassByValueTensors.md)), and the precomputed scalar `$q.value_f32` + (below). An optional operand also carries `$q.present`, true only when the graph supplies it. +- **Graph** — structural facts and graph-level flags of the matched graph: `$graph.node_count`, which + pins an exact match ([§3](#3-structural-pattern)), and `$graph.is_override_shape_enabled`, the + graph's own opt-in to execute-time override shapes. That flag is the graph's state and is distinct + from the descriptor's `allow_override_shape`, which is this matcher's opt-in to accepting such a + graph at all. +- **Attributes** — a matched node's scalar attributes, named by the node's pattern `id`: an + `{"id": "sdpa_fwd"}` node exposes `$sdpa_fwd.dropout_probability`, a `{"id": "conv"}` node + `$conv.dilation`. An optional attribute carries `$sdpa_fwd..present`. +- **Kernel metadata** — `$kernel.`, the values a UKD supplies for the fields its KMD declares + (tile and vector constants, the dtype it targets, [RFC 0017 §4](0017_UniversalKernelDescriptor.md#4-descriptor-formats)); + a matcher that reads them is evaluated per kernel ([§10](#10-the-matcher-compilation-indexing-and-caching)). + A `$kernel.*` field a matcher reads MUST exist in the engine's KMD; the loader checks this, so the + matcher publishes the set of `$kernel.*` fields it reads alongside its bound-symbol set. +- **Device properties** — `$device.` such as `$device.lds_size` or `$device.warp_size`, for a + check like an LDS budget. The device facts hipDNN carries today are narrower than this vocabulary + needs, so the device-property set is extended additively as the checks that need it land. + Architecture is **not** here for AOT: it is a pack property gated at + selection ([§2](#2-the-matchers-input-hipdnns-graph-model)); a JIT pack may reference `$device.arch`. + +**Precomputed fields.** Some tokens above are not stored on the graph: the schema layer derives them +once and publishes them as ordinary fields, so a matcher compares a value instead of re-deriving it. +`$q.packed` and `$q.stride_order` are the layout examples, standing in for `inferLayout`'s +contiguous-stride arithmetic. `$q.value_f32` is the other kind: a tensor's compile-time `value` is a +tagged union over eight differently-typed arms, and the expression language has no discriminator +syntax to unwrap one, so the schema layer coerces whichever arm is set to `f32` once and publishes it +as a single typed token — present only when the tensor carries a compile-time value at all, so a +criterion over it declines on a tensor that does not. A precomputed field is declared in the hipDNN +schema like any other field and versioned with it, so adding one is an additive schema change rather +than a per-pack extension point. Precomputed fields sit between the built-in operators and the +native-matcher escape hatch ([§8](#8-the-native-matcher-escape-hatch)): reach for one when a check +needs a derived fact, and for the hatch only when it needs real C++. + +**A `$` marks a reference.** Every reference to a bound field carries a leading `$`: tensors and +their fields (`$q`, `$q.uid`, `$q.dims[2]`, `$q.rank`, `$q.seqlen_q`), a node's attributes (`$sdpa_fwd.head_size` +— the node id `sdpa_fwd` is bare, the reference carries the `$`), `$graph.node_count`, `$kernel.tile_m`, +and `$device.lds_size`. Tokens without a `$` are literals: numbers, enum values (`"BFLOAT16"`, +`"gfx942"`), opcodes, and layout aliases. This is the JsonLogic variable rule of +[§6](#6-the-shared-expression-language): a `$`-string is a variable, everything else a literal, so no +`var` wrapper is needed and the same token reads identically wherever a criteria expression names a +bound field. + +**Named dims via the `shape` short-hand.** `shape` is a criteria short-hand that names a tensor's dims +and pins its rank to the list length: `{"shape": ["$q", ["batch", "num_heads", "seqlen_q", "head_size"]]}` +makes those dims readable as `$q.batch` … `$q.head_size` and requires `$q` to be rank 4. Names are +per-tensor, so a cross-tensor relation (same head dim across q/k) is an explicit criterion, +`{"==": ["$k.head_size", "$q.head_size"]}`. A dim position may be left anonymous +with `"_"`. When rank varies (NCHW vs NCDHW), name the fixed dims and capture the variable run as one +vector, `["n", "c", "$spatial"]`, so one matcher accepts both ranks and still reaches those dims through +`all` or a product; the exact `shape` grammar is specified in [Appendix A.5](#a5-the-shape-short-hand). + +![A live graph matched against a declarative pattern, auto-binding tensors, dims, strides, and attributes](../images/umd_symbol_binding.svg) + +--- + +## 5. Constraint Vocabulary + +The `criteria` field is a **single JsonLogic boolean expression** evaluated over the bound symbol table +([§4](#4-symbol-binding-and-the-auto-binding-formula), [§6](#6-the-shared-expression-language)). It is +normally an `and` of the individual tests, and reaches for `or` / `!` / `if` wherever a real +disjunction is needed ([§9](#9-composite-constraints)). The table below is not a set of criterion +*kinds* (there are none); it is the set of hand-written checks and the +JsonLogic sub-expression that expresses each. The residue — the handful of checks that need real C++ +— is not an operator at all: it is a **native matcher** the pack lists beside the descriptor +([§8](#8-the-native-matcher-escape-hatch)), so the expression language itself stays closed. + +| Hand-written check | JsonLogic criterion | Lowers from | +|---|---|---| +| **Opcode** | in the pattern node `op` (exact, `one_of`, `any`), not a criterion | node attribute-type gate | +| **Dtype (exact / set)** | `{"==": ["$q.dtype", "BFLOAT16"]}` / `{"in": ["$q.dtype", ["BFLOAT16", "FP8_E4M3"]]}` | `validateDataTypeIsSupported`, `validateFixedDataType` | +| **Dtype (relation)** | `{"==": ["$k.dtype", "$q.dtype"]}` | `validateConsistentDataTypes`, `q == k == v` | +| **Rank** | `{"==": ["$q.rank", 4]}`, or a `shape` short-hand that names four dims | `validateDimensionCount`, rank == 4 | +| **Shape (bind / relate)** | `{"shape": ["$q", ["batch", "num_heads", "seqlen_q", "head_size"]]}` binds `$q.`; relate with `{"==": ["$q.head_size", "$k.head_size"]}` | dim reads and cross-tensor dim relations | +| **Divisibility** | `{"divisible": [{"*": ["$y.n", "$y.ho", "$y.wo"]}, "$kernel.MPerBlock"]}` | tile-fit / GEMM-dim gates | +| **Layout** | `{"==": ["$q.stride_order", [0, 1, 2, 3]]}` ([§7](#7-layout-and-stride-order-constraints)) | `validateSupportedLayout` | +| **Packing** | `"$q.packed"` (a bound boolean) | `validatePackedTensors` | +| **Cross-tensor layout** | `{"==": ["$x.stride_order", "$y.stride_order"]}` (per pair) | `validateConsistentLayouts` | +| **Attribute (value)** | `{"==": ["$sdpa_fwd.causal_mask", false]}`; absent-or `{"or": [{"!": "$sdpa_fwd.dropout_probability.present"}, {"==": ["$sdpa_fwd.dropout_probability", 0.0]}]}` | per-attr value gates | +| **Attribute (one_of)** | `{"in": ["$sdpa_fwd.head_dim", [64, 128, 192]]}` | `head_dim in {...}` | +| **Optional operand present/absent** | operand `?` in the pattern; `{"not_present": ["$attn_mask"]}` (absent) / `{"present": ["$bias"]}` (present); one call takes a list, so a pack declines every optional operand it cannot serve at once | `attn_mask_tensor_uid()` absent gate | +| **Graph structure (exact / fusion)** | `{"==": ["$graph.node_count", 3]}`, and each intermediate `"$conv_out.virtual"` | node-count gate, fusion legality | +| **Cross-tensor / arithmetic** | `{"==": ["$q.dims[1]", "$k.dims[2]"]}`, `{"<": ["$q.head_size", 129]}`, `{"==": [{"%": ["$q.num_heads", "$k.kv_heads"]}, 0]}` | arithmetic and comparison gates | +| **Device property** | `{"<=": ["$kernel.lds_per_block", "$device.lds_size"]}` (arch is a pack property, not a criterion) | LDS/occupancy budgets; `getDeviceString` arch → pack `arch` | +| **Needs real C++** | not a criterion; a native matcher listed beside the descriptor ([§8](#8-the-native-matcher-escape-hatch)) | overflow guards, contradiction checks, derived-shape relations | + +Architecture gates *applicability* at pack selection via the KDP `arch` property and the per-arch +`kpack` manifest ([RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd), +[§12](0017_UniversalKernelDescriptor.md#12-packaging-and-delivery)), not as a match-time criterion for +AOT; other device properties (`$device.lds_size`, `$device.warp_size`) are read directly in criteria. + +**Exactness runs to the kernel, not just the graph.** `$graph.node_count` and the pattern's topology +pin the shape of the *graph*, but say nothing about whether a given candidate kernel can serve it. A +prebuilt kernel bakes quantities into its binary — a dtype, a head size, sometimes a sequence length — +and a graph that clears a pack's graph-level gates may still disagree with what one kernel baked. +**Every quantity a kernel bakes MUST therefore be a KMD field, and the pack's matcher MUST pin it +against the graph with a `$kernel.*` criterion.** Those are the clauses re-evaluated per candidate +([§10](#10-the-matcher-compilation-indexing-and-caching)), which is what turns one matcher plus a +kernel vector into a per-kernel applicability test. + +Getting this wrong fails silently rather than loudly: a matcher gating dtype only as +`{"in": ["$q.dtype", ["FLOAT16", "BFLOAT16"]]}` accepts an fp16 graph and may hand it to a bf16 +binary, which returns wrong numbers instead of an error. A field missing from the KMD also cannot be +pinned, so two kernels differing only in an unmodelled baked constant collide on the catalog key. The +check is mechanical, so the loader performs it: a UKD whose source declares a baked constant with no +corresponding KMD field is a load error. That check is a KDP/KMD-loader responsibility and is +specified by [RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd); the UMD's part +is to publish the `$kernel.*` fields it reads ([§4](#4-symbol-binding-and-the-auto-binding-formula)) +so the loader can perform it. + +**Every KDP needs one umbrella matcher.** At least one matcher in a pack MUST check the complete graph +topology the pack accepts — the same `node_count` and shape-defining criteria, applied to the whole +graph rather than a fragment. Other matchers in the pack may constrain any subset of what that matcher +binds. Without the rule, several matchers could each verify disjoint pieces of a graph while nothing +ever confirms the overall topology, producing loose or incorrect matches. Authors may write one large +matcher or split the work across focused ones; the only requirement is that the full graph shape is +checked explicitly somewhere in the pack. This is a **pack-level** rule: a single UMD cannot enforce +it, so it is validated by the KDP loader, not by A.10's per-descriptor checks. + +--- + +## 6. The Shared Expression Language + +Every UMD criterion and every UDD dispatch formula is a **JsonLogic** expression: a nested +`{"op": [args]}` tree whose arguments are themselves expressions or literals (the criteria-expression +form of [RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd)). Descriptors stay pure +data, and one parser, validator, and interpreter serve both subsystems. RFC 0017 §5 fixes the operator +vocabulary and the `$`-reference leaf convention; this RFC pins the concrete dialect, grammar, and +static typing those imply, and the UDD follow-up adopts the same core. + +**The `$`-variable convention.** Stock JsonLogic reads a bound value with `{"var": "path"}`. This RFC +replaces that with a single rule: **any string that begins with `$` is a variable reference** into the +bound symbol table ([§4](#4-symbol-binding-and-the-auto-binding-formula)) — `"$q"`, `"$q.dims[3]"`, +`"$q.seqlen_q"`, `"$q.dtype"`, `"$q.stride_order"`, `"$sdpa_fwd.head_size"`, `"$graph.node_count"`, +`"$kernel.tile_m"`, `"$device.lds_size"`. Every other JSON scalar is a literal: numbers (`128`), +booleans (`false`), and non-`$` strings (`"BFLOAT16"`, `"gfx942"`). There is no ambiguity, so no +`{"var": ...}` wrapper is used or accepted. A bare `$`-string is itself a valid boolean criterion when +it names a boolean field, e.g. `"$q.packed"`. + +**Two uses, one language.** + +- **Criteria (UMD).** A boolean-valued expression that decides applicability; a criterion that does not + evaluate to a boolean is a compile error. +- **Dispatch formulas (UDD).** A value-valued expression for grid, block, shared memory, and workspace + ([RFC 0017 §6](0017_UniversalKernelDescriptor.md#6-dispatch-and-workspace)), yielding a number. + +**Operators.** Logical `and`, `or`, `!`; comparison `==`, `!=`, `<`, `<=`, `>`, `>=`; membership `in`; +the per-element `all`; arithmetic `+`, `-`, `*`, `/`, `%`; the presence predicates `present` and +`not_present`; the pattern/short-hand ops `shape`, `rank`, `divisible`, and `value_or_default` (reads +a possibly-absent optional operand with a fallback); the conditional `if`; and the value-core +arithmetic the UDD needs (`ceil_div`, `min`, `max`, `abs`, `pow`, `log2`, `rsqrt`). The full table +with arities and types is [A.7](#a7-operator-reference); naming an operator outside it fails load +validation, and the set is closed — there is no registry or namespace by which a descriptor +introduces one ([A.9](#a9-the-operator-set-is-closed)). Anything the built-ins cannot express is a +**native matcher** beside the descriptor ([§8](#8-the-native-matcher-escape-hatch)). Adding a built-in +operator is additive ([§13](#13-serialization-and-versioning)); it never introduces a new criterion +*kind*, since the whole `criteria` field is one JsonLogic expression. + +**Operators nest, and either side of a comparison may be computed.** A leaf is a literal or a +`$`-prefixed reference, but either operand of a comparison can itself be an expression, so a check +like `{"divisible": [{"*": ["$y.n", "$y.ho", "$y.wo"]}, "$kernel.MPerBlock"]}` needs no temporary. + +```jsonc +// criteria (boolean): sub-expressions of the single top-level criteria expression +{"==": ["$q.head_size", 64]} // dim equality (named dim) +{"==": ["$q.dims[1]", "$k.dims[2]"]} // cross-tensor dim relation (positional) +{"<=": ["$q.head_size", 128]} // range bound +{"in": ["$q.head_size", [64, 128, 256]]} // set membership +{"divisible": ["$q.num_heads", "$k.kv_heads"]} // GQA divisibility +{"==": ["$q.stride_order", [0, 1, 2, 3]]}, "$q.packed" // layout + packed +{"==": ["$graph.node_count", 1]} // exact whole-graph match +{"or": [{"not_present": ["$attn_mask"]}, + {"==": ["$attn_mask.dtype", "$q.dtype"]}]} // composition (§10) + +// dispatch formulas (value), reused by the UDD +{"ceil_div": ["$q.seqlen_q", 16]} +{"*": [{"rsqrt": ["$q.head_size"]}, 1.4426950408889634]} +``` + +**Evaluation is a safe, bounded interpreter.** It fails closed on an unknown symbol, an out-of-range +axis, a type error, a non-boolean criterion result, or an invalid operation, and never executes +arbitrary code. Integer arithmetic uses checked-width integers and fails closed on overflow rather than +wrapping ([§14](#14-security-and-hostile-input)). The interpreter is bounded in recursion depth and step +count. Because JsonLogic is tiny and the evaluator is hand-written (no third-party parser), it is small +enough to audit and to lower into the static matcher ([§11](#11-static-matcher-sketch)). + +--- + +## 7. Layout and Stride-Order Constraints + +hipDNN tensors store no layout enum; layout is implied by stride order +([§2](#2-the-matchers-input-hipdnns-graph-model)). The UMD represents layout the way +[RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd) writes it: an **ordered list +of logical dimension indices, outermost (largest-stride) first**. Entry `i` names the logical +dimension stored at physical position `i`, so the array reads as the layout it describes: +`[0, 2, 3, 1]` over an `(n, c, h, w)` logical dim order spells N, H, W, C. + +```jsonc +{"==": ["$q.stride_order", [0, 1, 2, 3]]} // descending-stride packed (BHSD, rank-4) +{"==": ["$x.stride_order", [0, 2, 3, 1]]} // NHWC over an NCHW logical dim order +``` + +- The array is a permutation of `0..rank-1`. Entry `i` is the logical dimension at physical position + `i`, counting from the slowest-varying, so `[0,1,2,3]` is descending-stride packed and `[0,2,3,1]` + places the channel dim last, hence fastest-varying (NHWC). The `axis` used everywhere else (dims, + strides, `args_signature`) indexes the logical dimension order, independent of this physical + layout, consistent with RFC 0017 §6. +- **This is not the encoding `extractStrideOrder` returns.** That helper + (`data_sdk/include/hipdnn_data_sdk/utilities/ShapeUtilities.hpp:146`, called from + `ApplicabilityChecks.cpp:22`) produces the inverse: a per-dimension stride *rank*, entry `d` + giving the rank of logical dimension `d`, higher meaning slower-varying. The two forms are exact + inverses carrying identical information — `[0,2,3,1]` and `[3,0,2,1]` are the same NHWC layout — + so the binding layer converts once when it publishes `$q.stride_order`, and descriptors are + authored in the RFC 0017 form throughout. Nothing in the graph model changes; only the spelling a + matcher author writes. +- **Named aliases** are provided for the common cases and expand to the array literal at compile time, + so `{"==": ["$x.stride_order", "nhwc"]}` compiles to a comparison against `[0, 2, 3, 1]` + (A.8). The array remains the single canonical form. The four convolution aliases are exactly the + layouts `validateSupportedLayout` accepts today — NCHW/NHWC at rank 4, NCDHW/NDHWC at rank 5 + (`ApplicabilityChecks.cpp:76`); `bhsd` and `contiguous` are additions for the attention families, + which that oracle never covered. +- **Cross-tensor consistency** is a JsonLogic equality between stride orders, + `{"==": ["$x.stride_order", "$y.stride_order"]}` (one per pair, joined by the top-level `and`), + lowering `validateConsistentLayouts`; layout-agnostic tensors (rank-1 scalars, pass-by-value) are + skipped as they are today. Equality is convention-independent, so a cross-tensor check reads the + same under either form. +- **Packing** is the separate bound boolean `$q.packed` (written `"$q.packed"`), since a supported + stride order does not imply the tensor is gap-free; it lowers `validatePackedTensors`. +- `$q.stride_order` is an ordinary bound value ([§4](#4-symbol-binding-and-the-auto-binding-formula)), + so a `stride_order == [0,1,2,3]` gate is expressible directly. + +--- + +## 8. The Native-Matcher Escape Hatch + +Some checks cannot be stated with the built-in operators: they need real C++. **The expression +language has no extension point for them.** There is no custom operation, no namespaced operator key, +and no predicate registry the criteria tree resolves against; an operation key outside +[A.7](#a7-operator-reference) is simply unrecognized and refused at compile time +([Appendix A.10](#a10-compile-time-validation-normative)). + +Instead, such a check is a **native matcher**: an ordinary `GraphMatcherFn` registered in the +provider's `NativeRegistry` and named by a `MatchDescriptor`'s `matchSymbol` +([RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd)). A pack lists it alongside +the descriptor-backed matcher, and the ingestor conjoins their verdicts: a pack's graph-scoped +matchers all run and all must pass, so "this UMD's criteria **and** this C++ predicate" is expressed +by listing two matcher ids, not by nesting one inside the other. + +```jsonc +// pack.matcherIds: [ , <"hipdnn.sdpa.strides_fit_u32">, ... ] +``` + +**Why the hatch sits beside the expression language rather than inside it.** An escape hatch that +nests inside `criteria` has to be resolved by the compiler, which means the criteria language grows a +registry, a signature table, and a per-argument type contract — a second extension mechanism running +parallel to the one RFC 0017 already defines, differing only in grain. One hatch, at the matcher +level, keeps the expression language closed: every operator in [A.7](#a7-operator-reference) is +total, statically typed against the op-schema registry, and means the same thing in every provider +that ships a UMD. A descriptor is then fully interpretable from the schema alone, which is what makes +the drop-in path and the static-matcher lowering of [§11](#11-static-matcher-sketch) tractable. + +**What this costs, and the follow-up it implies.** A nested custom operation would have received +*bound variables* (`["$q", "$k", "$v", "$o"]`); a `GraphMatcherFn` receives `MatchContext` — the raw +graph and a device id — and must locate those tensors itself, repeating the structural search the UMD +already performed, in hand-written code that can drift from the registry-driven binding. Nor can the +existing plumbing hand them over: `BoundTokens` is `string -> int64_t`, which carries a dim but not a +tensor. Closing this gap — letting a pack's native matcher read the bindings a descriptor-backed +matcher in the same pack already published — is the natural follow-up, and it is a change to how +matchers are invoked, not a change to the expression language. + +The grounded cases that take this path: + +- **Integer-overflow guards.** `wouldFwdByteStridesFitUint32` / `byteStrideFitsU32` + (`SdpaFwdPlanBuilder.cpp:294`, `SdpaPlanUtils.hpp:159`): the kernarg struct stores byte strides as + `uint32`, so the check must be exact and fail closed. +- **Derived-quantity relations.** NumPy-broadcast affine-shape compatibility + (`BatchnormApplicabilityChecks.cpp:169`), layernorm normalized-dim reconciliation + (`LayernormApplicabilityChecks.cpp:68`), and RMSnorm `inv_rms` derived shape + (`RMSnormApplicabilityChecks.cpp:106`) each compute a shape and compare it, beyond the constraint + vocabulary. +- **Contradiction checks.** `getMaskType` throws when mask attributes contradict + (`SdpaFwdPlanBuilder.cpp:276`); a matcher encodes "the mask attributes are self-consistent". + +**GQA divisibility is not one of them.** `nhead_q % nhead_k == 0 && nhead_k != 0` +(`SdpaBwdPlanBuilder.cpp:548`) is expressible with `%` in +[§6](#6-the-shared-expression-language), and with no hatch inside the language there is no longer a +"centralize the zero-guard as a predicate" alternative to weigh: it stays declarative, and the +unknown-propagation rule of [A.7](#a7-operator-reference) supplies the fail-closed behavior. + +**Kernel-table lookups are a migration artifact, not a lasting matcher.** The +`getKernelNameKey` / CSV-registry lookups (`SdpaFwdPlanBuilder.cpp:287`, three in +`SdpaBwdPlanBuilder.cpp:660`) exist because today the builder resolves which prebuilt code object serves +a shape. Under the UKD model the KDP names the code object directly and the heuristic ranks candidates, +so these lookups mostly dissolve into ordinary constraints plus the Launch's kernel source. Where a +residual "is there a row for this exact combination" gate remains during coexistence, it is a native +matcher. + +**The registry a provider ships is part of its published contract**, unchanged from +[RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd): a pack naming a `matchSymbol` +the running provider does not ship fails to resolve, and fails closed. What changes is that this is +the *only* place a name resolves to C++, so the drop-in story is simple — **a UMD file is pure data +and always loads identically; a pack that needs C++ is not a drop-in.** + +Together with the UDD's custom-plan hatch +([RFC 0017 §6](0017_UniversalKernelDescriptor.md#6-dispatch-and-workspace)), these form the graded +ladder: fully declarative constraints, then a native matcher beside the descriptor for one gate that +needs C++, then a full provider. + +--- + +## 9. Composite Constraints + +The `criteria` field is a single JsonLogic expression, and JsonLogic supplies boolean composition +natively, so `(A AND B) OR C` is stated directly in the one tree with no extra mechanism: + +```jsonc +"criteria": + {"or": [ + {"and": [{"==": ["$q.head_size", 64]}, {"==": ["$q.num_heads", "$k.kv_heads"]}]}, // (A AND B) + {"==": ["$q.head_size", 128]} // OR C + ]} +``` + +`and`, `or`, `!`, and `if` compose the same comparison and membership tests used everywhere else, so +the deferral and reserved-shape +workaround of earlier drafts is gone: composition is day-one. A UMD whose tests all conjoin simply makes +the top-level expression an `and`, the common case. General N-ary commutative matching and unbounded +chains remain deferred to the JIT follow-up, as in RFC 0017 §5. + +--- + +## 10. The Matcher: Compilation, Indexing, and Caching + +A UMD is authored as text and **compiled once** into an in-memory matcher structure, on demand: nothing +is parsed until a graph needs it, and the parsed result is cached and reused +([RFC 0017 §3](0017_UniversalKernelDescriptor.md#3-how-it-works)). Compilation resolves op-schema +names, expands layout aliases, parses the criteria expression to an AST, and validates that every +referenced symbol is bound. The compiled form, not the text, is what runs against live graphs. + +**Root-opcode indexing.** The compiled matchers are indexed by the root node's opcode, so match cost +does not grow linearly with the number of descriptors: a graph whose root op is `sdpa_fwd` only +consults UMDs rooted at `sdpa_fwd`. This is the index RFC 0017 §16 calls for. Per-candidate cost (one +pass over the criteria expression) is separate from the index and bounded by short-circuit evaluation. + +**Shared matchers, evaluated once.** A KDP lists a set of matcher IDs and a kernel applies only when +all pass ([RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd)), so matchers are the +unit of sharing and of evaluation. A matcher that reads only graph fields (Tensor / Graph / Attributes / +Device, [§4](#4-symbol-binding-and-the-auto-binding-formula)) runs **once per graph**; on failure it +prunes every pack that lists it, so the most-shared checks (dtype, layout, shape) evaluated first shrink +the candidate set fast. A matcher that also reads `$kernel.*` is the **same** matcher re-evaluated +**once per distinct value of the `$kernel.*` fields it reads**, memoized on those, pruning per kernel +rather than per pack. The projection is what makes this pay: a kernel's full metadata tuple is unique +by construction, so memoizing on the whole tuple would save nothing, while a matcher reading one field +collapses an engine's catalog to that field's handful of distinct values. The compiler already +computes which `$kernel.*` fields a matcher reads ([§4](#4-symbol-binding-and-the-auto-binding-formula)), +so the memoization key costs nothing extra. Results are cached across queries. + +**Short-circuit evaluation.** The criteria expression evaluates with normal JsonLogic +short-circuiting: an `and` stops at its first false sub-expression, an `or` at its first true one, so a +non-match is rejected as early as the author's structure allows. The author orders the tree; the +compiler may hoist a cheap, highly selective sub-expression (a scalar attribute or dtype read) ahead of +an expensive one (a `shape`-wide relation) as an internal optimization, but this never changes the +result, only when a decision is reached. + +**Matching runs during applicability, and the provider owns the cache.** The order and the cache are +specified by [RFC 0017 §8](0017_UniversalKernelDescriptor.md#8-end-to-end-flow), which this RFC +follows rather than restates: matching happens inside `IEngine::isApplicable`, and its two products — +the **catalog** (the kernels whose full matcher set passed) and the **bound token state** (every +`$`-prefixed value the matchers resolved) — are cached together on the provider's shared container, +keyed on the engine, graph, and device that describe the problem, plus the descriptor-inventory +generation. Later phases read that cache instead of re-matching, so the +`isApplicable` / `getMaxWorkspaceSize` / `buildPlan` sequence that re-runs the loop today +(`AsmSdpaEngine.cpp:66,87`) matches a graph once. The compiled matcher itself is built once and shared +across every graph; only the binding result is per-problem. + +**Accepting is a promise.** Because the catalog is settled during applicability, a non-empty catalog +commits the engine to producing a launchable kernel: a later failure surfaces as a failed plan build, +not a fallback to another engine. This is RFC 0017 §8.6's base-path invariant, and it is what makes +match semantics load-bearing — a matcher that accepts a graph its kernel cannot serve turns a decline +into a user-visible error rather than a retry. It is the reason every quantity a kernel bakes must be +pinned by a `$kernel.*` criterion ([§5](#5-constraint-vocabulary)). + +**Device properties are constant per stream.** A `$device.` sub-expression (for example +`$device.lds_size`) is evaluated once per graph, since device properties do not vary across a stream. +Architecture is not a match-time criterion at all for AOT: it is a pack property gated at selection +([RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd)). + +![Compile-once pipeline: text UMD to constraint IR to a root-opcode-indexed matcher, with an applicability-time bind cache](../images/umd_matcher_pipeline.svg) + +--- + +## 11. Static Matcher (Sketch) + +AICK-1698 asks whether a UMD can be pre-compiled into a static matcher that further cuts the runtime +cost, while still supporting runtime (drop-in) matchers. This iteration does not commit to a design; it +records the options and the constraint they must satisfy. + +**The parity constraint.** However a static matcher is produced, it must be behaviorally identical to +the runtime matcher on the same UMD and graph. Build-time and drop-in descriptors run through one +generic engine ([RFC 0017 §3](0017_UniversalKernelDescriptor.md#3-how-it-works)), so a kernel that is +AOT-packed today and dropped in tomorrow must match the same graphs either way. Parity is testable as a +cross-path equivalence check ([§16](#16-testing-and-performance)). + +Options, from least to most build coupling: + +- **Interpreted compiled IR (baseline).** The [§10](#10-the-matcher-compilation-indexing-and-caching) + compiled form, interpreted. No codegen; identical on both paths by construction. This is the + fallback and the parity oracle. +- **Bytecode / flattened decision program.** Lower the constraint IR to a compact bytecode (a linear + program of typed ops over the bound symbol table) that a tiny VM executes. Serializable into the + KDP, so drop-in gets the same artifact AOT does. Faster than tree-walking, still data. +- **Generated C++ predicate, AOT only.** For build-time descriptors, emit a specialized C++ + applicability function per UMD and compile it into the provider (or a packed object). Closest to a + hand-written `isApplicable`, but unavailable to pure drop-in descriptors, which fall back to the + interpreted or bytecode path. Needs the generated function proven equivalent to the interpreter. +- **Shared decision tree across UMDs.** Combine many UMDs rooted at the same opcode into one decision + tree that tests shared constraints once (a discrimination net). An optimization layered on any of the + above; it changes throughput, not per-UMD semantics. + +Recommendation for a later iteration: make the interpreted compiled IR the contract and the parity +oracle, add the bytecode form as the shared AOT/drop-in fast path, and treat generated C++ and the +shared decision tree as opportunistic optimizations gated behind the parity test. The concrete choice +is deferred. + +--- + +## 12. Arbitration + +Today the first applicable plan builder wins, which is a documented latent bug when more than one +matches (`HipMlopsEngine.cpp:34`). The UMD makes overlap explicit and resolves it deterministically, +reusing the rule from RFC 0017 §5: + +1. When several UKDs match a graph, the engine's **heuristic** (the UHD) ranks them and the + top-scored kernel wins. +2. Ties break by explicit **`priority`** on the UKD. +3. Remaining ties break by the descriptor's stable **`id`**, compared as raw bytes, and the conflict + is logged to the warning log so an unintended overlap is visible. That byte order carries no + meaning; it is chosen for being stable across runs, load orders, and machines, not because a lower + id is better. + +Arbitration is a property of the generic engine over the set of matching UKDs; a UMD shared by several +UKDs contributes each of them as a candidate. This closes the mutual-exclusion-by-construction +requirement that the current engines depend on: overlap is allowed and resolved, not a correctness +hazard. + +--- + +## 13. Serialization and Versioning + +- **Authoring form.** Human-readable, diffable JSONC (the examples here): the JsonLogic criteria + expression of [§6](#6-the-shared-expression-language) with the `$`-variable convention. +- **Compiled form.** The compact binary the matcher runs ([§10](#10-the-matcher-compilation-indexing-and-caching)), + whose concrete bytes are defined with the KDP/packaging follow-up + ([RFC 0017 §14.2](0017_UniversalKernelDescriptor.md#142-follow-up-rfcs)); the schema those bytes + encode is specified in [Appendix A](#appendix-a-schema-reference). +- **Schema and version.** Every UMD carries `schema: "hipdnn.umd/v1"`, a stable `id` (a UUID), and a + mandatory `name` for diagnostics. A UMD whose schema version is newer than the runtime understands is + refused with a clear error, never silently reinterpreted, matching + [RFC 0017 §4](0017_UniversalKernelDescriptor.md#4-descriptor-formats). +- **A matcher carries two versions, gated in opposite directions.** `version` is the matcher format + itself, gated as a **ceiling**: a differing `major`, or a `minor` newer than the runtime's, is + refused, because the descriptor carries features that runtime cannot understand. An older minor + within the same major always loads — a file stamped `1.0` loads on a `1.1` runtime — so an author + stamps the lowest version their descriptor needs and it stays loadable on the oldest runtime that + can serve it. `sdk_version` is the hipDNN graph schema the matcher was authored against, and a + matcher is the only descriptor that needs it: it is the only one that reads graph fields, so it is + the only one a graph-schema change can silently invalidate. It is gated as a **floor the graph + sets**, not a constant: a graph reports the schema version its own contents require, and a matcher + declaring less is declined before it runs rather than matching on the fields it knows while + silently ignoring one that changes what the graph means. Both bounds hold at once — a + `sdk_version` newer than the runtime's own schema is still refused at compile + ([RFC 0017 §4](0017_UniversalKernelDescriptor.md#4-descriptor-formats)). Both compare numerically + by `(major, minor)`; both default to `1.0` when omitted, which is what every descriptor authored + against this revision means. +- **The graph's floor is an existing mechanism, not a new one.** hipDNN already computes the minimum + engine-plugin API version a graph requires from the optional features it uses and stamps it into + the serialized graph (`min_required_engine_api_version`); override shapes + ([RFC 0008](0008_OverridableTensorShapesDesign.md)) raise it to `1.1` and runtime pass-by-value + tensors ([RFC 0016](0016_RuntimePassByValueTensors.md)) to `1.2`. The matcher reads that field + rather than deriving a second floor of its own. A graph carrying no stamp reads as the `1.0` + baseline. +- **Additive evolution.** New JsonLogic operators and layout aliases are additive + within `v1` where they do not change the meaning of an existing descriptor; anything that would + reinterpret existing fields bumps the version. +- **Identity.** A UMD `id` is a **UUID**: globally unique with no central allocator, so descriptors + authored independently — including third-party drop-in files — do not collide by construction. + References are typed by field (a KDP's `matchers` versus `engine`), so a matcher id and an engine id + are never confused. A duplicate `id` seen on the drop-in path is logged and ignored rather than + taking down the provider ([RFC 0017 §16](0017_UniversalKernelDescriptor.md#16-risks)). + +--- + +## 14. Security and Hostile Input + +On the drop-in path the loader, the matcher, and the expression interpreter parse input that may be +untrusted or simply malformed, so they must be bounded and fail closed rather than crash +([RFC 0017 §16](0017_UniversalKernelDescriptor.md#16-risks)). + +- **Bounded parsing and matching.** Recursion depth, expression step count, node/constraint counts, and + descriptor size are capped; exceeding a cap quarantines the descriptor, it does not abort the + provider. +- **Checked arithmetic.** Shape, stride, and workspace arithmetic uses checked-width integers and fails + closed on overflow rather than under-allocating or wrapping. This is the same class of bug the + `strides_fit_u32` native matcher guards ([§8](#8-the-native-matcher-escape-hatch)). +- **Fail-closed evaluation.** An unknown symbol, an unrecognized operator key, an out-of-range axis, + or a type error declines the match; it never matches by default. +- **Quarantine, not cascade.** A bad descriptor is quarantined on load with a diagnostic; the rest load + ([RFC 0017 §12](0017_UniversalKernelDescriptor.md#12-packaging-and-delivery)). +- **Fuzzing.** A seed corpus of UMDs and graphs plus a fuzzer over the loader, matcher, and interpreter + run under the existing ASAN build ([§16](#16-testing-and-performance)), backing the fail-closed + requirement. + +--- + +## 15. Observability and Diagnostics + +Because matching is data-driven, it is inspectable. For the UMD the provider surfaces: + +- **A why-not trace.** For a graph and a candidate UMD, the sub-expression of `criteria` that + evaluated false and why (the concrete values compared), so an author can see exactly which test declined. +- **A binding view.** For a successful match, the full bound symbol table (tensors, dims, strides, + attributes) as the UDD will see it — the bound token state of + [RFC 0017 §8](0017_UniversalKernelDescriptor.md#8-end-to-end-flow). +- **An arbitration trace.** Which UKDs matched, how the heuristic scored them, and where a tie fell to + `priority` or stable `id` ([§12](#12-arbitration)). +- **Load diagnostics.** Which UMDs compiled, which were quarantined and why, and unresolved native + predicates by name. + +These reuse the diagnostic surface +[RFC 0017 §10](0017_UniversalKernelDescriptor.md#10-observability-and-diagnostics) defines rather than +adding a UMD-specific one. Authoring and validation tooling around the format is a separate, +first-class deliverable specified in +[RFC 0017 §11](0017_UniversalKernelDescriptor.md#11-tooling), where agentic authoring — agent-driven +skills that build and check descriptors from intent — is a committed first step. A UMD is a good fit +for it: the schema of [Appendix A](#appendix-a-schema-reference) and the compile-time checks of A.10 +are exactly what such a tool validates against. + +A matcher is also affected by the runtime opt-outs RFC 0017 §10 defines +(`HIPDNN_DISABLE_ENGINES`, `HIPDNN_DISABLE_KDPS`, `HIPDNN_DISABLE_UKDS`). Disabling a kernel carries a +risk specific to shared matchers: a matcher written around the kernel set it was meant to cover may no +longer be correct once one of those kernels is excluded, leaving the engine over-claiming +applicability for cases it no longer serves. The option is provided with that risk stated. + +--- + +## 16. Testing and Performance + +The UMD introduces no new testing strategy; it slots into hipDNN's existing tiers (`docs/Testing.md`, +`docs/testing/TestingStrategy.md`) as RFC 0017 §14.1 requires. A UMD-backed kernel runs through the +generic engine as an ordinary engine and produces the same graphs everything else consumes, so the +plugin-agnostic integration harness ([RFC 0006](0006_PluginAgnosticIntegrationTests.md)) validates it +against the CPU reference ([RFC 0001](0001_CpuGraphExecutorDesign.md)) with the golden-reference +tolerance chain ([RFC 0011](0011_GoldenReferenceValidation.md)). + +UMD-specific coverage: + +- **Match-equivalence against hand-written `isApplicable`.** For each converted engine, a test drives a + battery of graphs (accepting and rejecting) through both the hand-written builder and the UMD and + asserts identical accept/reject decisions and identical bound values. The SDPA-forward builder + ([§18](#18-worked-example-sdpa-forward)) is the first target. +- **Static/runtime parity.** The parity oracle of [§11](#11-static-matcher-sketch): the same UMD and + graph must decide identically on the interpreted and any lowered matcher. +- **Expression-language conformance.** A table-driven suite over the JsonLogic operator set (boolean + and value forms), including the AICK-1698 examples and the fail-closed cases (overflow, unknown + symbol, bad axis), shared with the UDD RFC's expression tests since the language is shared. +- **Fuzzing.** The corpus and fuzzer of [§14](#14-security-and-hostile-input). +- **Match overhead.** Plan-time match cost is measured against the hand-written baseline as + benchmarking matures (`tools/dnn-benchmarking`, [RFC 0013](0013_Autotune.md)); the compiled matcher, + root-opcode index, and applicability-time cache ([§10](#10-the-matcher-compilation-indexing-and-caching)) + keep it minimal, and the cost is paid once per graph and device. + +--- + +## 17. Migration + +Migration follows RFC 0017 §14: no engine is converted until a UMD-backed kernel runs end to end, and a +hand-written engine and its descriptor-backed replacement coexist until the generic one reaches parity +on the graphs that engine covers, at which point the hand-written code is retired. + +The **SDPA-forward** `isApplicable` (`SdpaFwdPlanBuilder.cpp:167`) is the first conversion, because it +exercises nearly the whole vocabulary (opcode, attribute gates, optional-operand absence, rank, dtype +relations, and cross-tensor dim relations) in one node, with its two non-declarative gates in the +paired native matcher. Its match-equivalence +test ([§16](#16-testing-and-performance)) gates the cutover. The mlops builders follow, reusing the +`IValidator` primitives (`dnn-providers/hip-kernel-provider/src/engines/hip_mlops_engine/plans/ApplicabilityChecks.cpp`) as the reference for their criteria +lowering. The kernel-table lookups dissolve into the KDP as described in +[§8](#8-the-native-matcher-escape-hatch). + +--- + +## 18. Worked Example: SDPA Forward + +The SDPA-forward check collapses into one UMD plus one native matcher. Compared to the hand-written +builder (`SdpaFwdPlanBuilder.cpp:167-296`), each C++ gate becomes a sub-expression, and only the two +genuinely non-declarative gates (uint32 stride fit, mask self-consistency) stay in C++ — as a native +matcher the pack lists beside this descriptor, not as criteria +([§8](#8-the-native-matcher-escape-hatch)). Note `$q.head_size` is bound from `$q`'s dim, not read as +an attribute ([§2](#2-the-matchers-input-hipdnns-graph-model)). + +This example is grounded on the asm-SDPA builder because that builder is this RFC's first migration +target ([§17](#17-migration)), so the mapping table below doubles as the cutover checklist. It is +deliberately a different example from +[RFC 0017 §13](0017_UniversalKernelDescriptor.md#13-worked-example-sdpa-as-a-ukd), which works the +`attention_dense` kernel family end to end across all six descriptor kinds; that one shows a UMD in +the context of a full UKD, this one shows one hand-written `isApplicable` becoming one UMD. + +```jsonc +{ + "schema": "hipdnn.umd/v1", + "id": "9c3f5b2a-7d41-4e88-b6a0-1f2e3d4c5b6a", + "name": "SDPA forward (d128, bf16/fp8) match", + "nodes": [ + {"kind": "op", "id": "sdpa_fwd", "op": "sdpa_fwd", + "operands": {"q": "$q", "k": "$k", "v": "$v", + "attn_mask": "$attn_mask?", "page_table_k": "$page_table_k?", + "page_table_v": "$page_table_v?"}, // optional operands carry a ? suffix + "results": {"o": "$o"}} + ], + "criteria": {"and": [ + {"==": ["$graph.node_count", 1]}, // exact: this kernel is the whole graph + {"in": ["$q.dtype", ["BFLOAT16", "FP8_E4M3"]]}, // supported dtype set + {"==": ["$k.dtype", "$q.dtype"]}, {"==": ["$v.dtype", "$q.dtype"]}, // q == k == v + {"shape": ["$q", ["batch", "num_heads", "seqlen_q", "head_size"]]}, // rank 4, binds dims + {"shape": ["$k", ["batch", "kv_heads", "seqlen_k", "head_size"]]}, + {"shape": ["$v", ["batch", "kv_heads", "seqlen_k", "head_size"]]}, + {"==": ["$o.rank", 4]}, + {"==": ["$q.head_size", 128]}, + {"==": ["$k.head_size", "$q.head_size"]}, // same head dim across q/k/v + {"or": [{"!": "$sdpa_fwd.dropout_probability.present"}, {"==": ["$sdpa_fwd.dropout_probability", 0.0]}]}, + {"==": ["$sdpa_fwd.alibi_mask", false]}, + {"==": ["$sdpa_fwd.padding_mask", false]}, + {"or": [{"!": "$sdpa_fwd.generate_stats.present"}, {"==": ["$sdpa_fwd.generate_stats", false]}]}, + // unsupported optional operands declined together; `not_present` always evaluates, + // unlike a field read on an absent operand + {"not_present": ["$attn_mask", "$page_table_k", "$page_table_v"]} + ]} + // arch is a pack property (KDP.arch), not a match criterion + // uint32 stride fit and mask self-consistency are a native matcher the pack lists + // alongside this descriptor (§8), conjoined with these criteria by the ingestor +} +``` + +Mapping to the hand-written code: + +| Hand-written (`SdpaFwdPlanBuilder.cpp`) | UMD field | +|---|---| +| `getDeviceString` gfx942/gfx950 (:186) | pack `arch` property (KDP), gated at selection | +| `nodeWrappers().size() != 1` (:199) | `{"==": ["$graph.node_count", 1]}` | +| `attributesType() != SdpaAttributes` (:200) | node `op: sdpa_fwd` | +| dropout / alibi / padding / stats gates (:205-224) | `$sdpa_fwd.*` criteria | +| `attn_mask` / `page_table_*` absent (:209-215) | `?` operands + one `{"not_present": [...]}` over all three | +| rank == 4 (:231-247) | `shape` short-hand (names four dims) | +| `q == k == v` dtype (:244) | `{"==": ["$k.dtype", "$q.dtype"]}` | +| `k.dims[1] == v.dims[1]` head count (:251) | shared `kv_heads` name in `$k`/`$v` `shape` | +| head dim == 128 | `{"==": ["$q.head_size", 128]}` | +| `getMaskType` throw-on-contradiction (:276) | native matcher ([§8](#8-the-native-matcher-escape-hatch)) | +| `wouldFwdByteStridesFitUint32` (:294) | native matcher ([§8](#8-the-native-matcher-escape-hatch)) | +| `getKernelNameKey` table lookup (:287) | dissolves into the KDP's Launch ([§8](#8-the-native-matcher-escape-hatch)) | + +The bound symbols (`$q..$o`, the named dims `$q.batch`, `$q.num_heads`, `$k.kv_heads`, `$q.seqlen_q`, +`$k.seqlen_k`, `$q.head_size`, and every auto-bound dim/stride) are exactly what the paired UDD's grid +and argument formulas reference ([RFC 0017 §6](0017_UniversalKernelDescriptor.md#6-dispatch-and-workspace)). + +--- + +## 19. Risks + +- **Op-schema registry coupling.** Auto-binding depends on a registry generated from the flatbuffer op + schema ([§2](#2-the-matchers-input-hipdnns-graph-model)). If it drifts from the graph definitions, + bindings are wrong. Mitigation: generate it from the schema's own `umd_opcode` table attribute and + `umd_input_tensor` / `umd_output_tensor` / `umd_name` field annotations (never from field-name conventions), so a + new or renamed operand carries + its binding contract in the same edit ([§2](#2-the-matchers-input-hipdnns-graph-model)), and fail + closed on an unknown op or name rather than binding a wrong field. +- **Expression language sharing.** JsonLogic is shared with the UDD ([§6](#6-the-shared-expression-language)). + A change made for one subsystem can affect the other. Mitigation: one parser/validator/interpreter, + a shared conformance suite, and a clear split (criteria are boolean JsonLogic, dispatch formulas + are value JsonLogic). +- **Native-matcher symbols as contract.** A pack's `matchSymbol` names C++ the provider must ship + ([§8](#8-the-native-matcher-escape-hatch)); a drop-in pack naming an unshipped symbol fails to + resolve. Mitigation: version and document the shipped symbol set, fail closed with a clear + diagnostic, and keep pure-UMD packs free of symbols so they remain true drop-ins. +- **Match overhead.** Per-candidate evaluation of the criteria expression is unbounded by the + root-opcode index ([§10](#10-the-matcher-compilation-indexing-and-caching)). Mitigation: short-circuit + evaluation, applicability-time caching, and the overhead test of [§16](#16-testing-and-performance). +- **Static-matcher parity.** A lowered matcher that diverges from the interpreter is a silent + correctness bug ([§11](#11-static-matcher-sketch)). Mitigation: the interpreter is the oracle and the + parity test gates any lowering. + +--- + +## 20. Open Questions + +1. **Expression language home.** This RFC pins JsonLogic as the shared expression language and defines + it here because the UMD needs it first; the UDD follow-up references this section. Confirm the split + rather than each subsystem defining its own, and decide where the shared conformance suite lives. +2. **Native-matcher bindings.** A native matcher gets `MatchContext`, not the bindings its pack's UMD + already published ([§8](#8-the-native-matcher-escape-hatch)), so it re-locates tensors by hand. + Extend the matcher-invocation contract to pass the published bindings, or accept the duplication? +3. **Static-matcher form.** Which of the [§11](#11-static-matcher-sketch) options becomes the AOT fast + path, and does it also serve drop-in via a serialized bytecode? +4. **Feature-vector overlap.** The bound symbol table overlaps the feature vector a UHD consumes + ([RFC 0017 §17 Q4](0017_UniversalKernelDescriptor.md#17-open-questions)); should the UMD's bindings + be the canonical feature source for kernel selection? +5. **Operators RFC 0017 omits.** `abs`, `pow`, `log2`, and `if` are in A.7 but not in the vocabulary + table of [RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd), which declares + itself complete. They are needed for dispatch formulas and precedence chains. Add them to RFC + 0017's table, or scope them in this RFC as dispatch-formula-only and keep criteria to RFC 0017's + set? + +--- + +## 21. References and Prior Art + +The design borrows established ideas; none is a dependency. These informed the UMD specifically. + +| System | Idea borrowed | +|---|---| +| **MLIR PDL / PDLL** | Two-layer design: a declarative pattern compiled once to a fast matcher; constraints inline on the binding; a named native-predicate escape hatch; pattern priority for arbitration | +| **TVM Relax DFPattern** | Constraint vocabulary (op, dtype, symbolic shape, wildcard); dataflow use-def constraints; cross-tensor same-shape relations | +| **XLA pattern matcher** | Exact-vs-compatible equality; a tensor virtual/internal flag gating fusion; layout as a distinct constraint; optional operands; capture-by-reference binding | +| **PyTorch Inductor / torch.library** | Node/edge pattern vocabulary; serialized precompiled patterns; duplicate-pattern detection | +| **LLVM ISel / discrimination nets** | Sharing common prefixes of many patterns rooted at one opcode into one decision structure ([§11](#11-static-matcher-sketch)) | +| **ONNX Runtime** | First-claim arbitration as the anti-pattern this RFC replaces with deterministic ranking; single-node versus fused-subgraph capability | + +--- + +## 22. Glossary + +- **UMD (Universal Match Descriptor) / matcher:** the declarative pattern and criteria that decide + whether a kernel applies to a graph and bind the variables its dispatch and workspace formulas use. + A KDP lists a set of matcher IDs; a kernel applies only when all pass. Reused across packs by ID. +- **Structural pattern:** the op nodes and the named operand/result edges of a UMD; edges are implicit + through shared pattern variables ([§3](#3-structural-pattern)). +- **Criteria expression:** the single JsonLogic `{"op": [args]}` boolean a UMD evaluates over its bound + symbol table, typically an `and` of the individual tests ([§5](#5-constraint-vocabulary)). +- **Symbol lifecycle:** a name is declared in the UMD, bound when the graph matches, and used by the UDD + ([§4](#4-symbol-binding-and-the-auto-binding-formula)). +- **Auto-binding formula:** the standard scheme that binds every operand/result tensor, its dims and + strides, and every op attribute of a matched node, without hand-declaration + ([§4](#4-symbol-binding-and-the-auto-binding-formula)). +- **Op-schema registry:** the generated table mapping each op type to its operand/result UID fields and + attributes, letting the matcher reconstruct edges and auto-bind + ([§2](#2-the-matchers-input-hipdnns-graph-model)). +- **JsonLogic:** the shared expression language; boolean-valued expressions are UMD criteria and + value-valued expressions are UDD dispatch formulas, both over one `$`-variable symbol table (the five + namespaces of [§4](#4-symbol-binding-and-the-auto-binding-formula)) with one evaluator + ([§6](#6-the-shared-expression-language)). +- **Stride-order layout:** layout represented as an ordered list of logical dimension indices, + outermost first, since tensors carry no layout enum ([§7](#7-layout-and-stride-order-constraints)). +- **Native matcher:** the escape hatch; a `GraphMatcherFn` named by a `MatchDescriptor`'s + `matchSymbol` and conjoined with a UMD's criteria by the ingestor, for logic the built-in operators + cannot state. It lives beside the expression language, never inside it + ([§8](#8-the-native-matcher-escape-hatch)). +- **Composite criteria:** any boolean combination of tests within the one `criteria` expression, + written directly with JsonLogic `and` / `or` / `!` / `if` ([§9](#9-composite-constraints)). +- **Arbitration:** the deterministic resolution when several UKDs match: heuristic (UHD) score, then + `priority`, then stable `id` compared as raw bytes ([§12](#12-arbitration)). +- **Catalog / bound token state:** the two products of matching a graph — the kernels whose full + matcher set passed, and every `$`-prefixed value the matchers resolved. Both are cached by the + provider during applicability and read by every later phase + ([RFC 0017 §8](0017_UniversalKernelDescriptor.md#8-end-to-end-flow)). +- **Root-opcode index:** the index of compiled matchers by root opcode that keeps match cost sublinear + in descriptor count ([§10](#10-the-matcher-compilation-indexing-and-caching)). + +--- + +## Appendix A: Schema Reference + +This appendix is the normative schema for `hipdnn.umd/v1`. Where the prose sections above describe a +construct by example, the grammar and tables here fix its exact form. A descriptor that violates a +**MUST** here is refused at compile ([§10](#10-the-matcher-compilation-indexing-and-caching)); it never +matches by default ([§14](#14-security-and-hostile-input)). Grammar is EBNF; quoted terminals are JSON +tokens. The value domain is `Int`, `Float`, `Bool`, `Dtype` (an enum name such as `"BFLOAT16"`), +`IntArray` (e.g. a stride order), and `Tensor` (an opaque bound operand/result); `Value` is any of +these. Criteria arithmetic is integer; `Float` arises only in UDD dispatch formulas +([§6](#6-the-shared-expression-language)). + +### A.1 Descriptor object + +| Field | Type | Required | Default | Rule | +|---|---|---|---|---| +| `schema` | string | yes | — | MUST equal `"hipdnn.umd/v1"`; a newer version is refused, never reinterpreted ([§13](#13-serialization-and-versioning)) | +| `id` | string (UUID) | yes | — | A UUID; stable, globally unique identity ([§13](#13-serialization-and-versioning)) | +| `name` | string | yes | — | Diagnostics only; not semantic | +| `version` | string | no | `"1.0"` | Matcher format version, `.`, gated at load as a **ceiling**: a differing `major`, or a `minor` newer than the runtime's, is refused; an older minor always loads ([§13](#13-serialization-and-versioning)) | +| `sdk_version` | string | no | `"1.0"` | The hipDNN graph schema version this matcher was authored against, `.`. Refused at load when newer than the runtime's own schema, and declined at match time against the **floor the graph sets** — a matcher below what the graph requires is skipped instead of asked ([§13](#13-serialization-and-versioning)) | +| `allow_override_shape` | bool | no | `false` | When `false`, override-shape graphs are declined ([§3](#3-structural-pattern)) | +| `nodes` | array<Node> | yes | — | Non-empty; A.2 | +| `criteria` | Expr | yes | — | A single expression whose static type is `Bool` (A.6) | + +No other top-level keys are permitted; an unknown key is refused. Both version fields compare +numerically by `(major, minor)`, so `1.10` is above `1.9`; a value that does not parse as exactly two +decimal components is refused. + +### A.2 Node object and opcode selector + +```ebnf +node = "{" , '"kind"' , ":" , '"op"' , "," + , '"id"' , ":" , string , "," + , '"op"' , ":" , op-selector + , [ "," , '"operands"' , ":" , name-map ] + , [ "," , '"results"' , ":" , name-map ] , "}" ; +op-selector = opcode | one-of | '"any"' ; +one-of = "{" , '"one_of"' , ":" , "[" , opcode , { "," , opcode } , "]" , "}" ; +opcode = string ; (* MUST resolve in the op-schema registry *) +name-map = "{" , [ name-bind , { "," , name-bind } ] , "}" ; +name-bind = string , ":" , bind ; (* key is an op-schema tensor name *) +bind = '"$' , ident , [ "?" ] , '"' ; +ident = letter , { letter | digit | "_" } ; +``` + +- `kind` MUST be `"op"` (the only kind in `v1`). +- Node `id` values MUST be unique within `nodes` and disjoint from every pattern-variable name (A.4). +- Every `opcode` and every name key MUST resolve in the op-schema registry for that opcode; an unknown + opcode or name is refused ([§2](#2-the-matchers-input-hipdnns-graph-model)). +- `"any"` is legal only inside a fused pattern (a descriptor whose match pins `node_count > 1`); it + MUST NOT be the sole or root node. +- A `?` suffix marks an optional binding and is legal only for a name the registry marks optional. A + `?` on a required name, or an optional name bound without `?`, is refused. +- Names not listed in `operands`/`results` are ignored for matching but still auto-bound (A.4). + +### A.3 Names and edges + +Name keys are drawn from the op-schema registry (`q`, `k`, `v`, `o`, …). An edge is implicit: two nodes +are connected when the same pattern variable is a `results` value of one and an `operands` value of +another ([§3](#3-structural-pattern)). A pattern variable MUST appear as a `results` value at most once +(single producer). + +### A.4 Variable references and the five namespaces + +Any JSON string beginning with `$` is a variable reference; every other JSON scalar is a literal — no +`{"var": …}` wrapper is used or accepted ([§6](#6-the-shared-expression-language)). + +```ebnf +var-ref = "$" , ( tensor-ref | graph-ref | attr-ref | kernel-ref | device-ref | element-ref ) ; +tensor-ref = tvar , [ "." , tensor-field ] ; +tvar = ident ; (* a pattern variable bound to a Tensor *) +tensor-field = "uid" | "rank" | "dtype" | "stride_order" | "packed" | "virtual" | "present" + | "is_runtime_pass_by_value" | "value_f32" + | "dims" , "[" , uint , "]" + | "strides" , "[" , uint , "]" + | dim-name ; (* a name a shape short-hand introduced (A.5) *) +graph-ref = "graph" , "." , ( "node_count" | "is_override_shape_enabled" ) ; +attr-ref = node-id , "." , attr-name , [ "." , "present" ] ; +kernel-ref = "kernel" , "." , ident ; +device-ref = "device" , "." , ident ; +element-ref = "_" ; (* the current element inside an `all` predicate *) +uint = digit , { digit } ; +``` + +| Namespace | Root | Fields | Type | +|---|---|---|---| +| Tensor | a pattern variable (`$q`) | `uid`, `rank`, `dtype`, `stride_order`, `packed`, `virtual`, `present`, `is_runtime_pass_by_value`, `value_f32`, `dims[i]`, `strides[i]`, `` | `Tensor` / `Int` / `Dtype` / `IntArray` / `Bool` / `Float` | +| Graph | `$graph` | `node_count`, `is_override_shape_enabled` | `Int` / `Bool` | +| Attributes | a node `id` (`$sdpa_fwd`) | ``, `.present` | scalar / `Bool` | +| Kernel | `$kernel` | `` a UKD supplies ([RFC 0017 §4](0017_UniversalKernelDescriptor.md#4-descriptor-formats)) | scalar | +| Device | `$device` | `` (`lds_size`, `warp_size`, …) | scalar | + +Rules: +- `graph`, `kernel`, and `device` are **reserved** namespace roots: a `tvar` and a node `id` MUST NOT + use them. +- `present` is bound only for an optional operand/attribute; reading it on a required one is refused. +- A field access on an **absent** optional operand or attribute (e.g. `$attn_mask.dtype` when + `attn_mask` is absent) resolves to **unknown**, which is neither true nor false: the criterion + containing it can no longer be satisfied, so the match declines (fail closed, + [§14](#14-security-and-hostile-input)). Unknown is not coerced — it never reads as `false`, `0`, or + "not equal" — because a coerced unknown would make a narrowing check such as + `{"!=": ["$attn_mask.dtype", "BFLOAT16"]}` silently **pass** on an operand the pack never saw. This + is what RFC 0017 §5 means by a check on an absent operand that "neither passes nor fails, it simply + does not run". +- **`and` and `or` are three-valued.** A definite `false` still decides an `and` and a definite `true` + still decides an `or`, even beside an unknown argument; only an otherwise-undecided result stays + unknown. This is what lets the "absent, or present and constrained" pair of + [§5](#5-the-constraint-vocabulary) accept a graph without the operand, where the second arm's field + reads cannot run. +- An out-of-range `dims[i]`/`strides[i]`, an unknown `dim-name`, or any unresolved reference declines + the match. `value_f32` resolves only when the tensor carries a compile-time value, so it declines + on one that does not. +- The `present` / `not_present` **operators** (A.7) are the exception to the fail-closed rule above: + they report whether a reference resolved and so always yield a boolean, on a required or an + optional reference alike. + +### A.5 The `shape` short-hand + +```ebnf +shape-op = "{" , '"shape"' , ":" , "[" , tensor-var , "," , entry-list , "]" , "}" ; +tensor-var = '"$' , ident , '"' ; +entry-list = "[" , entry , { "," , entry } , "]" ; +entry = dim-name | anon | capture ; +dim-name = '"' , ident , '"' ; (* not "_", not "$"-prefixed, not a reserved tensor-field *) +anon = '"_"' ; (* positional only; reachable via dims[i], not by name *) +capture = '"$' , ident , '"' ; (* at most one; MUST be the last entry *) +``` + +`shape` is a boolean-valued short-hand that binds names and pins rank: + +- With no `capture`, the entry count pins rank exactly (`rank == count`); each `dim-name` binds + `$.` to its positional dim, and `"_"` leaves that position name-less. +- With a trailing `capture`, the fixed entries bind positionally from the front and `$` binds + the remaining dims `[k .. rank-1]` (where `k` is the capture's index) as an ordered `Vector`, so one + matcher accepts a family of ranks (`["n", "c", "$spatial"]` matches NCHW rank 4 and NCDHW rank 5). + Rank MUST be `>= count - 1`. +- A `dim-name` MUST NOT equal a reserved `tensor-field` and MUST be unique within the entry list. +- A captured `Vector` is read by `all` (A.7), by arithmetic reduction (`*`/`+`), or positionally as + `$[i]`. It is not a scalar. + +```jsonc +{"shape": ["$q", ["batch", "num_heads", "seqlen_q", "head_size"]]} // rank 4; binds four names +{"shape": ["$x", ["n", "c", "$spatial"]]} // rank >= 2; $spatial = dims[2..] +{"all": ["$spatial", {">": ["$_", 0]}]} // every captured dim positive +``` + +### A.6 Criteria expression grammar + +```ebnf +expr = literal | var-ref-str | operation ; +operation = "{" , op-key , ":" , operand , "}" ; (* exactly one key *) +operand = expr | arg-array ; (* unary sugar, or an argument list *) +arg-array = "[" , [ expr , { "," , expr } ] , "]" ; +op-key = '"' , ( builtin-op | custom-op ) , '"' ; +custom-op = ident , "." , ident , { "." , ident } ; +literal = number | boolean | non-dollar-string | json-array ; +``` + +- An object used as an expression MUST have exactly one key, the operator; multi-key objects are + refused. `non-dollar-string` is any JSON string not beginning with `$` and is a literal (enum name, + opcode, or layout alias). +- The top-level `criteria` MUST be an `expr` whose static type is `Bool`; a non-boolean result is a + compile error. A bare `var-ref-str` is a valid criterion only when it resolves to `Bool` (`"$q.packed"`). +- Evaluation is strictly in written order with short-circuit; the engine never reorders operands (A.7, + [§10](#10-the-matcher-compilation-indexing-and-caching)). + +### A.7 Operator reference + +All integer arithmetic uses checked-width integers and fails closed on overflow +([§14](#14-security-and-hostile-input)). `n-ary` means two or more arguments. + +| Operator | Arity | Argument types | Result | Notes | +|---|---|---|---|---| +| `and` | n-ary | `Bool…` | `Bool` | Short-circuits at the first `false` | +| `or` | n-ary | `Bool…` | `Bool` | Short-circuits at the first `true` | +| `!` | 1 | `Bool` | `Bool` | Unary sugar accepts a bare operand | +| `==`, `!=` | 2 | `Value, Value` (same type) | `Bool` | Compares `Int`, `Bool`, `Dtype`, and `IntArray` (stride order) | +| `<`, `<=`, `>`, `>=` | 2 | `Int, Int` | `Bool` | | +| `in` | 2 | `Value, Array` | `Bool` | Element type MUST match the array element type | +| `all` | 2 | `Vector, Bool` | `Bool` | Predicate evaluated per element; the current element is `$_` | +| `+`, `*` | n-ary | `Number…` | `Number` | | +| `-`, `/` | 2 | `Number, Number` | `Number` | `/` fails closed on a zero divisor | +| `%` | 2 | `Int, Int` | `Int` | Fails closed on a zero divisor | +| `min`, `max` | n-ary | `Number…` | `Number` | | +| `abs` | 1 | `Number` | `Number` | | +| `pow` | 2 | `Number, Number` | `Number` | | +| `ceil_div` | 2 | `Int, Int` | `Int` | Fails closed on a zero divisor | +| `log2`, `rsqrt` | 1 | `Number` | `Float` | Dispatch-formula value core; fails closed on a non-positive argument | +| `rank` | 1 | `Tensor` | `Int` | Equal to `$t.rank` | +| `divisible` | 2 | `Int, Int` | `Bool` | `true` iff divisor `!= 0` and dividend `% divisor == 0`; a zero divisor yields `false`, not an error, giving uniform fail-closed zero-guarding (resolves [Open Question 2](#20-open-questions)) | +| `value_or_default` | 2 | `Ref, Value` | `Value` | The referenced optional operand/attribute value when present, else the fallback. The fallback is usually a literal, but it may be any expression of the same type, including a second field reference, so "this field, else that one" is one operator instead of a branch; both arms MUST be type-compatible against the schema's declared field types | +| `present`, `not_present` | n-ary | `Ref…` | `Bool` | Was this supplied? `present` is `true` iff every referenced operand/field is bound; `not_present` iff every one is absent. Unlike a field read, these **always evaluate** rather than declining on an absent reference, so one call declines a list of unsupported optional operands | +| `if` | 3 or 2n+1 | `Bool, Value [, Bool, Value]… , Value` | `Value` | `if`/`elif`/`else` chain; branch results MUST share a type | +| `shape` | 2 | `Tensor, EntryList` | `Bool` | A.5 | + +**Every operator except `present`, `not_present`, and `value_or_default` propagates an unknown** +(an unresolved reference) rather than coercing it, so an absent optional operand can never satisfy a +criterion (A.4). `and` and `or` are three-valued: a definite `false` still decides an `and` and a +definite `true` still decides an `or`, even beside an unknown argument. The three exceptions answer +"did this resolve?", so they always yield a real value. + +Adding an operator is additive within `v1` where it does not change the meaning of an existing +descriptor ([§13](#13-serialization-and-versioning)). + +`abs`, `pow`, `log2`, and `if` are not in the vocabulary table of +[RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd), which declares its list +complete. They are retained here because the UDD's dispatch formulas need them, and because +`if` is what encodes a precedence chain such as a mask classifier. They are additive under the rule +above; reconciling them into RFC 0017's table is tracked as +[Open Question 5](#20-open-questions). + +### A.8 `stride_order` values and layout aliases + +A `stride_order` comparison accepts either an integer array or an alias string; aliases expand to the +array at compile time, and the array is the single canonical form ([§7](#7-layout-and-stride-order-constraints)). +An array MUST be a permutation of `0 .. rank-1` listing logical dimension indices outermost +(largest-stride) first, matching +[RFC 0017 §5](0017_UniversalKernelDescriptor.md#5-matching-and-the-umd). + +| Alias | Array | | Alias | Array | +|---|---|---|---|---| +| `nchw` | `[0,1,2,3]` | | `ndhwc` | `[0,2,3,4,1]` | +| `nhwc` | `[0,2,3,1]` | | `bhsd` | `[0,1,2,3]` | +| `ncdhw` | `[0,1,2,3,4]` | | `contiguous` | ascending positions for the tensor's rank | + +`contiguous` needs a rank, so it is legal only on a tensor whose rank a `shape` short-hand has pinned +(A.5); a fixed-rank alias compared against a differently-pinned tensor is refused at compile rather +than declining silently at match time. + +### A.9 The operator set is closed + +A.7 is exhaustive. An operation key that is not listed there is refused at compile time, and there is +no registry, namespace, or provider hook by which a descriptor can introduce one +([§8](#8-the-native-matcher-escape-hatch)). + +A **namespaced (dotted) key** such as `{"hipdnn.strides_fit_u32": [...]}` is the form a custom +operation would have taken, and gets no special treatment: it is simply not in A.7, so it is refused +as an unrecognized operator. The check it expresses belongs in a native matcher listed beside this +descriptor in the pack, not inside `criteria`. Adding a built-in operator to A.7 is additive +([§13](#13-serialization-and-versioning)); adding one out-of-band is not possible, which is what +makes a UMD interpretable from the schema alone +([§14](#14-security-and-hostile-input)). + +### A.10 Compile-time validation (normative) + +A descriptor MUST pass every check below to compile; a failure refuses (and, on the drop-in path, +quarantines) the descriptor with a diagnostic ([§10](#10-the-matcher-compilation-indexing-and-caching), +[§14](#14-security-and-hostile-input)): + +1. `schema == "hipdnn.umd/v1"`, `id` is a well-formed UUID, and each of `version` / `sdk_version`, + when present, is a well-formed `.` string the runtime can honor: same `major`, and a + `minor` no newer than the runtime's ([§13](#13-serialization-and-versioning)). The `sdk_version` + floor is a match-time check against the graph, not a compile-time one. +2. Only the keys of A.1 at the top level; only the keys of A.2 on each node. +3. Every `opcode` and every name key resolves in the op-schema registry. +4. Every `?` suffix matches the registry's optionality for that name. +5. Node `id`s are unique and disjoint from every pattern-variable name; reserved roots + (`graph`, `kernel`, `device`) are unused as a `tvar` or node `id`. +6. Each pattern variable is a `results` value at most once (single producer, A.3). +7. At most one `shape` `capture` per tensor, in the last position; no `dim-name` collides with a + reserved `tensor-field`. +8. Every `$`-reference resolves to a bound symbol (the build and drop-in bound-symbol check of + [§4](#4-symbol-binding-and-the-auto-binding-formula)). +9. Every operation key appears in A.7 (A.9), every operation's arity and argument types satisfy A.7, + and every layout alias resolves. +10. `criteria` has static type `Bool`. + +--- + +## Appendix B: Op-Schema Registry Generation + +The op-schema registry ([§2](#2-the-matchers-input-hipdnns-graph-model)) is the table the matcher +consults to reconstruct a UID-centric graph's edges and to auto-bind symbols +([§4](#4-symbol-binding-and-the-auto-binding-formula), [Appendix A.4](#a4-variable-references-and-the-five-namespaces)). +It is **generated from FlatBuffers field annotations on the graph schema**, never hand-maintained and +never inferred from field-name conventions, so the binding contract for an operand lives in the same +`.fbs` edit that adds the operand and cannot silently drift from the graph definitions. + +### B.1 Attribute declarations + +Four custom attributes are declared once in the graph schema. FlatBuffers requires an `attribute` +declaration before an attribute may be used, and declared attributes — on a table or on a field — are +retained in the binary reflection schema (`.bfbs`), which is what the generator reads. + +```fbs +attribute "umd_opcode"; // table: the op's UMD-facing opcode (e.g. "sdpa_fwd") +attribute "umd_input_tensor"; // field flag: this field is an input (operand) tensor UID +attribute "umd_output_tensor"; // field flag: this field is an output (result) tensor UID +attribute "umd_name"; // field string: the name the UMD binds it by (e.g. "q", "o") +``` + +`umd_opcode` is a **table-level** attribute (applied in parentheses after the table name); the other +three are field-level. No separate operand/result type attribute is needed: an operand or result always +binds a `Tensor` — a tensor UID is the only edge kind in a UID-centric graph +([§2](#2-the-matchers-input-hipdnns-graph-model)) — and every scalar attribute is precisely an +unannotated scalar field (B.3). The `long` UID field type and the `umd_input_tensor`/`umd_output_tensor` flag +together already fix what the binding is. + +### B.2 Annotated schema + +Each op's attribute table annotates its UID fields next to the field they govern. Optionality is **not** +re-annotated: a UID field's `= null` default (an optional field) already encodes it, so the generator +derives required-vs-optional from the field's presence semantics rather than a fourth attribute. + +```fbs +table SdpaAttributes (umd_opcode: "sdpa_fwd") { // table-level opcode shorthand + q_tensor_uid:long (umd_input_tensor, umd_name: "q"); // required input + k_tensor_uid:long (umd_input_tensor, umd_name: "k"); + v_tensor_uid:long (umd_input_tensor, umd_name: "v"); + o_tensor_uid:long (umd_output_tensor, umd_name: "o"); // required output + attn_mask_tensor_uid:long = null (umd_input_tensor, umd_name: "attn_mask"); // optional input + // ... other optional UID operands, likewise annotated ... + dropout_probability:float = null; // unannotated -> scalar attribute + alibi_mask:bool = false; // unannotated -> scalar attribute + causal_mask:bool = false; +} +``` + +### B.3 Field classification (normative) + +For every table reachable from the `NodeAttributes` union (which already maps each opcode to its +attribute table), the generator reads the table's `umd_opcode` (the entry's opcode key; when absent it +falls back to the table type name) and classifies each field by these rules; a violation **fails the +build** rather than emitting a wrong registry: + +| Field carries | Classified as | Requirements | +|---|---|---| +| `umd_input_tensor` + `umd_name` | input (operand) edge for that name | field type MUST be an integer UID (`long`); `umd_name` MUST be non-empty | +| `umd_output_tensor` + `umd_name` | output (result) edge for that name | field type MUST be an integer UID (`long`); `umd_name` MUST be non-empty | +| neither flag, a **scalar** field | scalar attribute, named by the field name | — | +| neither flag, a **non-scalar** field (vector, sub-table, union, string) | skipped (not a UMD scalar) | — | + +- **Optionality** is derived, not annotated: a field with a `= null` default (an optional UID or an + optional scalar) is optional; it supplies the `?`-binding of [Appendix A.2](#a2-node-object-and-opcode-selector) + and the `.present` field of [Appendix A.4](#a4-variable-references-and-the-five-namespaces). +- **Build errors (fail closed):** `umd_input_tensor` and `umd_output_tensor` on the same field; `umd_name` without + either flag; `umd_input_tensor`/`umd_output_tensor` on a non-integer field; a duplicate `umd_name` within one op; + an input/output tensor whose name collides with a reserved token + ([Appendix A.4](#a4-variable-references-and-the-five-namespaces)); or a duplicate `umd_opcode` across + ops. +- **Scalar attribute value kind.** A scalar attribute carries its value kind for compile-time type + checking ([Appendix A.10](#a10-compile-time-validation-normative)): integer fields bind as `Int`, + float/double as `Float`, `bool` as `Bool`, and an **enum-typed** field as `Dtype`, carrying the + enum-value name string (e.g. `diagonal_alignment` → `"TOP_LEFT"`). This mirrors the tensor `dtype` + representation and lets a criterion compare an enum attribute against a literal enum name. +- **No name-suffix inference.** `PointwiseAttributes::axis_tensor_uid` is a plain axis index, not a + tensor UID; because it carries no `umd_input_tensor`/`umd_output_tensor` it is classified a scalar attribute, + exactly as intended — nothing keys off the `_tensor_uid` suffix. +- **Scalar attributes need no annotation, and are still fully bound.** An annotation carries the two + facts that cannot be inferred for an *edge*: that a `long` field is a tensor UID rather than a plain + integer (`q_tensor_uid` and `left_bound` are the same type, distinguishable only by the flag), and a + bind name distinct from the field name (`"q"` vs `q_tensor_uid`). A scalar needs neither: it is a + non-edge by elimination, and its bind name *is* its field name, which reflection already reports. So + every unannotated field is auto-bound in the Attributes namespace as `$.` + ([Appendix A.4](#a4-variable-references-and-the-five-namespaces)) with its reflected type and its + `= null`-derived optionality — `$sdpa_fwd.dropout_probability`, `$sdpa_fwd.alibi_mask`, + `$sdpa_fwd.left_bound` bind with no annotation (B.5). The one consequence is that a scalar's bind + name is coupled to its field name: renaming the field renames the symbol, whereas `umd_name` + insulates an operand from a field rename. If a scalar ever needs that insulation, the additive + extension is an optional `umd_name` on an unflagged field, used purely as a bind-name override; the + flag remains the operand/result discriminator. + +### B.4 Generation pipeline + +1. `flatc` compiles the graph schema and emits the binary reflection schema `graph.bfbs`, which + transitively covers every attribute table and carries the declared `umd_*` attributes (custom + attributes surface through reflection, not through the generated C++ headers). +2. A build-time generator loads `graph.bfbs` through the FlatBuffers reflection API, enumerates the + `NodeAttributes` union members (opcode → table), reads each table's `umd_opcode`, and applies the + B.3 rules to each table's fields. Because `graph.fbs` pulls the per-op tables in via `include`, the + `.bfbs` (and thus the registry) must regenerate when **any** included schema changes, not only when + the top-level schema does. +3. The generator emits the op-schema registry as generated C++ the provider compiles (a header-only + registry emitted into the schema-owning SDK, so it needs no dependency on the provider): a table + keyed by the `umd_opcode` shorthand, each entry also carrying its attribute-table name and the + integer `NodeAttributes` value, and listing its input and output tensors (name, optionality, and + the typed accessor for the UID field) and its scalar attributes (name, optionality, value kind, and + typed accessor). + +Reflection is used **only at build time**; the generated registry holds typed accessors, so the runtime +match path reads UID and attribute fields via `attributesAs()` +([§2](#2-the-matchers-input-hipdnns-graph-model)) with no per-match reflection cost. + +### B.5 Generated registry and how the matcher uses it + +The emitted entry for one opcode is, conceptually: + +```jsonc +// generated; illustrative shape, not a wire format +"sdpa_fwd": { // key: the table's umd_opcode (fallback: table name) + "attributes_type": "SdpaAttributes", // NodeAttributes union member; matched via Node::attributes_type() + "operands": [ + {"name": "q", "uid": "&SdpaAttributes::q_tensor_uid", "optional": false}, + {"name": "k", "uid": "&SdpaAttributes::k_tensor_uid", "optional": false}, + {"name": "v", "uid": "&SdpaAttributes::v_tensor_uid", "optional": false}, + {"name": "attn_mask", "uid": "&SdpaAttributes::attn_mask_tensor_uid", "optional": true} + ], + "results": [{"name": "o", "uid": "&SdpaAttributes::o_tensor_uid", "optional": false}], + "attributes": [ + {"name": "dropout_probability", "get": "&SdpaAttributes::dropout_probability", "optional": true}, + {"name": "alibi_mask", "get": "&SdpaAttributes::alibi_mask", "optional": false} + ] +} +``` + +At compile ([§10](#10-the-matcher-compilation-indexing-and-caching)) the matcher resolves each pattern +name against this entry. At match time, for a node of that opcode it reads each name's UID via the typed +accessor, resolves the UID against the per-graph UID→producer/consumer index +([§2](#2-the-matchers-input-hipdnns-graph-model)) to bind the tensor, and auto-binds the tensor's fields +and the node's scalar attributes into the five namespaces +([Appendix A.4](#a4-variable-references-and-the-five-namespaces)). An optional operand absent from the +graph binds `.present = false` and is read only through a guarded reference or `value_or_default`. + +### B.6 Lockstep and failure modes + +- **Lockstep.** Adding or renaming an operand is one `.fbs` edit that carries its `umd_input_tensor` / + `umd_name` with it; the next build regenerates the registry, so a UMD naming that name resolves and a + UMD referencing a name that no longer exists fails compile + ([Appendix A.10](#a10-compile-time-validation-normative)). +- **Unknown op or name at match compile.** The matcher fails closed: a pattern node whose opcode or + name is absent from the registry is refused, never bound to a guessed field + ([§19 Op-schema registry coupling](#19-risks)). +- **Generation is deterministic and diffable.** The generated registry is a build artifact; a schema + change that alters bindings shows up as a registry diff, which is the review surface for a binding + change. diff --git a/projects/hipdnn/flatbuffers_sdk/CMakeLists.txt b/projects/hipdnn/flatbuffers_sdk/CMakeLists.txt index 7f6fec723765..6f90169ed2eb 100644 --- a/projects/hipdnn/flatbuffers_sdk/CMakeLists.txt +++ b/projects/hipdnn/flatbuffers_sdk/CMakeLists.txt @@ -148,6 +148,37 @@ if(HIPDNN_GENERATE_SDK_HEADERS) GENERATED_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/hipdnn_flatbuffers_sdk/data_objects" OUTPUT_NAMESPACE hipdnn_flatbuffers_sdk ) + + # --- RFC 0018 UMD op-schema registry generation (Appendix B) --- + # 1. Emit graph.bfbs (reflection schema; retains the umd_* attributes the + # C++ headers drop). 2. Build the host tool umd_registry_gen. 3. Run it to + # emit the strongly-typed op-schema registry header into the committed + # include tree, in lockstep with the schema. + set(_umd_bfbs "${CMAKE_CURRENT_BINARY_DIR}/umd/graph.bfbs") + hipdnn_generate_bfbs( + SCHEMA schemas/graph.fbs + SCHEMAS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/schemas" + OUTPUT "${_umd_bfbs}" + ) + + add_executable(umd_registry_gen tools/umd_registry_gen/umd_registry_gen.cpp) + target_compile_features(umd_registry_gen PRIVATE cxx_std_17) + target_link_libraries(umd_registry_gen PRIVATE flatbuffers::flatbuffers) + + set(_umd_registry_hpp + "${CMAKE_CURRENT_SOURCE_DIR}/include/hipdnn_flatbuffers_sdk/umd/op_schema_registry_generated.hpp") + add_custom_command( + OUTPUT "${_umd_registry_hpp}" + COMMAND umd_registry_gen "${_umd_bfbs}" "${_umd_registry_hpp}" + DEPENDS umd_registry_gen "${_umd_bfbs}" + COMMENT "Generating UMD op-schema registry (RFC 0018 Appendix B)" + VERBATIM + ) + add_custom_target(generate_umd_registry + DEPENDS "${_umd_registry_hpp}" + COMMENT "UMD op-schema registry generated" + ) + add_dependencies(hipdnn_flatbuffers_sdk generate_umd_registry) endif() # --- Export and install --- diff --git a/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/data_objects/graph_generated.h b/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/data_objects/graph_generated.h index 58216f13fe0d..8c40f31b6b78 100644 --- a/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/data_objects/graph_generated.h +++ b/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/data_objects/graph_generated.h @@ -47,6 +47,8 @@ struct NodeT; struct EngineApiVersion; +struct Uuid; + struct Graph; struct GraphBuilder; struct GraphT; @@ -55,6 +57,8 @@ bool operator==(const NodeT &lhs, const NodeT &rhs); bool operator!=(const NodeT &lhs, const NodeT &rhs); bool operator==(const EngineApiVersion &lhs, const EngineApiVersion &rhs); bool operator!=(const EngineApiVersion &lhs, const EngineApiVersion &rhs); +bool operator==(const Uuid &lhs, const Uuid &rhs); +bool operator!=(const Uuid &lhs, const Uuid &rhs); bool operator==(const GraphT &lhs, const GraphT &rhs); bool operator!=(const GraphT &lhs, const GraphT &rhs); @@ -701,6 +705,36 @@ inline bool operator!=(const EngineApiVersion &lhs, const EngineApiVersion &rhs) } +FLATBUFFERS_MANUALLY_ALIGNED_STRUCT(1) Uuid FLATBUFFERS_FINAL_CLASS { + private: + uint8_t bytes_[16]; + + public: + Uuid() + : bytes_() { + } + Uuid(::flatbuffers::span _bytes) { + ::flatbuffers::CastToArray(bytes_).CopyFromSpan(_bytes); + } + const ::flatbuffers::Array *bytes() const { + return &::flatbuffers::CastToArray(bytes_); + } + ::flatbuffers::Array *mutable_bytes() { + return &::flatbuffers::CastToArray(bytes_); + } +}; +FLATBUFFERS_STRUCT_END(Uuid, 16); + +inline bool operator==(const Uuid &lhs, const Uuid &rhs) { + return + (*lhs.bytes() == *rhs.bytes()); +} + +inline bool operator!=(const Uuid &lhs, const Uuid &rhs) { + return !(lhs == rhs); +} + + struct NodeT : public ::flatbuffers::NativeTable { typedef Node TableType; std::string name{}; @@ -977,6 +1011,7 @@ struct GraphT : public ::flatbuffers::NativeTable { ::flatbuffers::Optional preferred_engine_id = ::flatbuffers::nullopt; bool is_override_shape_enabled = false; std::unique_ptr min_required_engine_api_version{}; + std::unique_ptr id{}; GraphT() = default; GraphT(const GraphT &o); GraphT(GraphT&&) FLATBUFFERS_NOEXCEPT = default; @@ -995,7 +1030,8 @@ struct Graph FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { VT_NODES = 14, VT_PREFERRED_ENGINE_ID = 16, VT_IS_OVERRIDE_SHAPE_ENABLED = 18, - VT_MIN_REQUIRED_ENGINE_API_VERSION = 20 + VT_MIN_REQUIRED_ENGINE_API_VERSION = 20, + VT_ID = 22 }; const ::flatbuffers::String *name() const { return GetPointer(VT_NAME); @@ -1051,6 +1087,12 @@ struct Graph FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { hipdnn_flatbuffers_sdk::data_objects::EngineApiVersion *mutable_min_required_engine_api_version() { return GetStruct(VT_MIN_REQUIRED_ENGINE_API_VERSION); } + const hipdnn_flatbuffers_sdk::data_objects::Uuid *id() const { + return GetStruct(VT_ID); + } + hipdnn_flatbuffers_sdk::data_objects::Uuid *mutable_id() { + return GetStruct(VT_ID); + } bool Verify(::flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_NAME) && @@ -1067,6 +1109,7 @@ struct Graph FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { VerifyField(verifier, VT_PREFERRED_ENGINE_ID, 8) && VerifyField(verifier, VT_IS_OVERRIDE_SHAPE_ENABLED, 1) && VerifyField(verifier, VT_MIN_REQUIRED_ENGINE_API_VERSION, 4) && + VerifyField(verifier, VT_ID, 1) && verifier.EndTable(); } GraphT *UnPack(const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; @@ -1105,6 +1148,9 @@ struct GraphBuilder { void add_min_required_engine_api_version(const hipdnn_flatbuffers_sdk::data_objects::EngineApiVersion *min_required_engine_api_version) { fbb_.AddStruct(Graph::VT_MIN_REQUIRED_ENGINE_API_VERSION, min_required_engine_api_version); } + void add_id(const hipdnn_flatbuffers_sdk::data_objects::Uuid *id) { + fbb_.AddStruct(Graph::VT_ID, id); + } explicit GraphBuilder(::flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -1126,9 +1172,11 @@ inline ::flatbuffers::Offset CreateGraph( ::flatbuffers::Offset<::flatbuffers::Vector<::flatbuffers::Offset>> nodes = 0, ::flatbuffers::Optional preferred_engine_id = ::flatbuffers::nullopt, bool is_override_shape_enabled = false, - const hipdnn_flatbuffers_sdk::data_objects::EngineApiVersion *min_required_engine_api_version = nullptr) { + const hipdnn_flatbuffers_sdk::data_objects::EngineApiVersion *min_required_engine_api_version = nullptr, + const hipdnn_flatbuffers_sdk::data_objects::Uuid *id = nullptr) { GraphBuilder builder_(_fbb); if(preferred_engine_id) { builder_.add_preferred_engine_id(*preferred_engine_id); } + builder_.add_id(id); builder_.add_min_required_engine_api_version(min_required_engine_api_version); builder_.add_nodes(nodes); builder_.add_tensors(tensors); @@ -1150,7 +1198,8 @@ inline ::flatbuffers::Offset CreateGraphDirect( const std::vector<::flatbuffers::Offset> *nodes = nullptr, ::flatbuffers::Optional preferred_engine_id = ::flatbuffers::nullopt, bool is_override_shape_enabled = false, - const hipdnn_flatbuffers_sdk::data_objects::EngineApiVersion *min_required_engine_api_version = nullptr) { + const hipdnn_flatbuffers_sdk::data_objects::EngineApiVersion *min_required_engine_api_version = nullptr, + const hipdnn_flatbuffers_sdk::data_objects::Uuid *id = nullptr) { auto name__ = name ? _fbb.CreateString(name) : 0; auto tensors__ = tensors ? _fbb.CreateVector<::flatbuffers::Offset>(*tensors) : 0; auto nodes__ = nodes ? _fbb.CreateVector<::flatbuffers::Offset>(*nodes) : 0; @@ -1164,7 +1213,8 @@ inline ::flatbuffers::Offset CreateGraphDirect( nodes__, preferred_engine_id, is_override_shape_enabled, - min_required_engine_api_version); + min_required_engine_api_version, + id); } ::flatbuffers::Offset CreateGraph(::flatbuffers::FlatBufferBuilder &_fbb, const GraphT *_o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); @@ -1228,7 +1278,8 @@ inline bool operator==(const GraphT &lhs, const GraphT &rhs) { (lhs.nodes.size() == rhs.nodes.size() && std::equal(lhs.nodes.cbegin(), lhs.nodes.cend(), rhs.nodes.cbegin(), [](std::unique_ptr const &a, std::unique_ptr const &b) { return (a == b) || (a && b && *a == *b); })) && (lhs.preferred_engine_id == rhs.preferred_engine_id) && (lhs.is_override_shape_enabled == rhs.is_override_shape_enabled) && - ((lhs.min_required_engine_api_version == rhs.min_required_engine_api_version) || (lhs.min_required_engine_api_version && rhs.min_required_engine_api_version && *lhs.min_required_engine_api_version == *rhs.min_required_engine_api_version)); + ((lhs.min_required_engine_api_version == rhs.min_required_engine_api_version) || (lhs.min_required_engine_api_version && rhs.min_required_engine_api_version && *lhs.min_required_engine_api_version == *rhs.min_required_engine_api_version)) && + ((lhs.id == rhs.id) || (lhs.id && rhs.id && *lhs.id == *rhs.id)); } inline bool operator!=(const GraphT &lhs, const GraphT &rhs) { @@ -1243,7 +1294,8 @@ inline GraphT::GraphT(const GraphT &o) io_data_type(o.io_data_type), preferred_engine_id(o.preferred_engine_id), is_override_shape_enabled(o.is_override_shape_enabled), - min_required_engine_api_version((o.min_required_engine_api_version) ? new hipdnn_flatbuffers_sdk::data_objects::EngineApiVersion(*o.min_required_engine_api_version) : nullptr) { + min_required_engine_api_version((o.min_required_engine_api_version) ? new hipdnn_flatbuffers_sdk::data_objects::EngineApiVersion(*o.min_required_engine_api_version) : nullptr), + id((o.id) ? new hipdnn_flatbuffers_sdk::data_objects::Uuid(*o.id) : nullptr) { tensors.reserve(o.tensors.size()); for (const auto &tensors_ : o.tensors) { tensors.emplace_back((tensors_) ? new hipdnn_flatbuffers_sdk::data_objects::TensorAttributesT(*tensors_) : nullptr); } nodes.reserve(o.nodes.size()); @@ -1260,6 +1312,7 @@ inline GraphT &GraphT::operator=(GraphT o) FLATBUFFERS_NOEXCEPT { std::swap(preferred_engine_id, o.preferred_engine_id); std::swap(is_override_shape_enabled, o.is_override_shape_enabled); std::swap(min_required_engine_api_version, o.min_required_engine_api_version); + std::swap(id, o.id); return *this; } @@ -1281,6 +1334,7 @@ inline void Graph::UnPackTo(GraphT *_o, const ::flatbuffers::resolver_function_t { auto _e = preferred_engine_id(); _o->preferred_engine_id = _e; } { auto _e = is_override_shape_enabled(); _o->is_override_shape_enabled = _e; } { auto _e = min_required_engine_api_version(); if (_e) _o->min_required_engine_api_version = std::unique_ptr(new hipdnn_flatbuffers_sdk::data_objects::EngineApiVersion(*_e)); } + { auto _e = id(); if (_e) _o->id = std::unique_ptr(new hipdnn_flatbuffers_sdk::data_objects::Uuid(*_e)); } } inline ::flatbuffers::Offset Graph::Pack(::flatbuffers::FlatBufferBuilder &_fbb, const GraphT* _o, const ::flatbuffers::rehasher_function_t *_rehasher) { @@ -1300,6 +1354,7 @@ inline ::flatbuffers::Offset CreateGraph(::flatbuffers::FlatBufferBuilder auto _preferred_engine_id = _o->preferred_engine_id; auto _is_override_shape_enabled = _o->is_override_shape_enabled; auto _min_required_engine_api_version = _o->min_required_engine_api_version ? _o->min_required_engine_api_version.get() : nullptr; + auto _id = _o->id ? _o->id.get() : nullptr; return hipdnn_flatbuffers_sdk::data_objects::CreateGraph( _fbb, _name, @@ -1310,7 +1365,8 @@ inline ::flatbuffers::Offset CreateGraph(::flatbuffers::FlatBufferBuilder _nodes, _preferred_engine_id, _is_override_shape_enabled, - _min_required_engine_api_version); + _min_required_engine_api_version, + _id); } inline bool VerifyNodeAttributes(::flatbuffers::Verifier &verifier, const void *obj, NodeAttributes type) { diff --git a/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/umd/OpSchemaRegistry.hpp b/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/umd/OpSchemaRegistry.hpp new file mode 100644 index 000000000000..07f39742e48b --- /dev/null +++ b/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/umd/OpSchemaRegistry.hpp @@ -0,0 +1,121 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +// +// Neutral (JsonLogic-agnostic) op-schema registry types for RFC 0018 (UMD). +// +// This header defines the data shapes the generated registry populates. It is +// hand-written and stable; the per-op tables and lookup functions live in the +// build-time-generated op_schema_registry_generated.hpp, emitted by the +// umd_registry_gen tool from the FlatBuffers `.bfbs` reflection schema +// (RFC 0018 Appendix B). Keeping these types free of any provider / JsonLogic +// dependency lets flatbuffers_sdk own the registry without depending on a +// consumer. +// +// The registry reconstructs a UID-centric graph's edges and auto-binds symbols: +// each op entry lists its input-tensor and output-tensor names (each with a typed +// reader) and its scalar attributes (each with a typed value reader). Readers +// use the generated FlatBuffers accessors directly -- no runtime reflection. + +#pragma once + +#include +#include +#include + +// Opaque forward declaration of the FlatBuffers NodeAttributes union discriminant +// (fully defined in the generated graph_generated.h). Declaring it here with its +// fixed underlying type makes it a complete type, so OpSchemaEntry can carry the +// real enum value -- no int laundering -- while this neutral header stays free of +// the generated schema include. The generated registry TU sees both this +// declaration and the real definition, so any underlying-type mismatch is a +// compile error there. +namespace hipdnn_flatbuffers_sdk::data_objects +{ +enum class NodeAttributes : std::uint8_t; +} // namespace hipdnn_flatbuffers_sdk::data_objects + +namespace hipdnn_flatbuffers_sdk::umd +{ + +// The value kind of a scalar attribute. Mirrors the JsonLogic value domain a +// criteria expression type-checks against (RFC 0018 Appendix A.7): integer, +// float, boolean, or an enum name (`DTYPE`). Enum-typed attributes surface as +// `DTYPE`, carrying the enum-value name string. +enum class AttrType : std::uint8_t +{ + INT, + FLOAT, + BOOL, + DTYPE, +}; + +// A resolved scalar attribute value. `present` is false only for an optional +// attribute the graph omitted; a present value populates exactly the member +// selected by `type`. `dtype` points at a static enum-name string (no +// ownership) when `type == DTYPE`. +struct ScalarValue +{ + AttrType type = AttrType::INT; + bool present = false; + std::int64_t i = 0; + double f = 0.0; + bool b = false; + const char* dtype = nullptr; +}; + +// Reads a bound tensor's UID out of a concrete attribute table (passed as a +// `const void*` to keep this header type-erased). Returns false when the tensor +// is optional and absent from the graph; returns true and writes `out` +// otherwise. A required tensor always returns true. +using UidReader = bool (*)(const void* attributes, std::int64_t& out); + +// Reads a scalar attribute value out of a concrete attribute table. +using ScalarReader = ScalarValue (*)(const void* attributes); + +struct InputTensorBinding +{ + std::string_view name; + bool optional = false; + UidReader read = nullptr; +}; + +struct OutputTensorBinding +{ + std::string_view name; + bool optional = false; + UidReader read = nullptr; +}; + +struct AttrBinding +{ + std::string_view name; + bool optional = false; + AttrType type = AttrType::INT; + ScalarReader read = nullptr; +}; + +// One op's schema: its `opcode` (the UMD-facing shorthand from the table's +// `umd_opcode` attribute, e.g. "sdpa_fwd", falling back to the table type name +// when the attribute is absent), the `tableName` (the NodeAttributes union member, +// e.g. "SdpaAttributes", for diagnostics), and the `attributesType` (the +// NodeAttributes enum value, for O(1) lookup against Node::attributes_type()), +// plus its input-tensor, output-tensor, and scalar bindings. +struct OpSchemaEntry +{ + std::string_view opcode; + std::string_view tableName; + data_objects::NodeAttributes attributesType{}; + const InputTensorBinding* inputTensors = nullptr; + std::size_t inputTensorCount = 0; + const OutputTensorBinding* outputTensors = nullptr; + std::size_t outputTensorCount = 0; + const AttrBinding* attributes = nullptr; + std::size_t attributeCount = 0; +}; + +// Defined (inline) in the generated header. +// const OpSchemaEntry* opSchemaEntries(std::size_t& count); +// const OpSchemaEntry* lookupOpByName(std::string_view opcode); +// const OpSchemaEntry* lookupOpByType(data_objects::NodeAttributes attributesType); + +} // namespace hipdnn_flatbuffers_sdk::umd diff --git a/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/umd/op_schema_registry_generated.hpp b/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/umd/op_schema_registry_generated.hpp new file mode 100644 index 000000000000..96050da6fe48 --- /dev/null +++ b/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/umd/op_schema_registry_generated.hpp @@ -0,0 +1,391 @@ +// AUTOMATICALLY GENERATED by umd_registry_gen -- do not modify. +// Source: FlatBuffers reflection schema (graph.bfbs). See RFC 0018 Appendix B. +#pragma once + +#include +#include +#include + +#include +#include + +namespace hipdnn_flatbuffers_sdk::umd +{ +namespace generated +{ + +// BatchnormInferenceAttributes +namespace op0 { +inline const AttrBinding attributes[] = { + {"bias_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->bias_tensor_uid()); return s; }}, + {"inv_variance_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->inv_variance_tensor_uid()); return s; }}, + {"mean_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->mean_tensor_uid()); return s; }}, + {"scale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->scale_tensor_uid()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, + {"y_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->y_tensor_uid()); return s; }}, +}; +} // namespace op0 + +// PointwiseAttributes +namespace op1 { +inline const AttrBinding attributes[] = { + {"axis_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->axis_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"elu_alpha", true, AttrType::FLOAT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::FLOAT; auto v = static_cast(a)->elu_alpha(); s.present = v.has_value(); if(v) s.f = static_cast(*v); return s; }}, + {"in_0_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->in_0_tensor_uid()); return s; }}, + {"in_1_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->in_1_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"in_2_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->in_2_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"operation", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNamePointwiseMode(static_cast(a)->operation()); return s; }}, + {"out_0_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->out_0_tensor_uid()); return s; }}, + {"relu_lower_clip", true, AttrType::FLOAT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::FLOAT; auto v = static_cast(a)->relu_lower_clip(); s.present = v.has_value(); if(v) s.f = static_cast(*v); return s; }}, + {"relu_lower_clip_slope", true, AttrType::FLOAT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::FLOAT; auto v = static_cast(a)->relu_lower_clip_slope(); s.present = v.has_value(); if(v) s.f = static_cast(*v); return s; }}, + {"relu_upper_clip", true, AttrType::FLOAT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::FLOAT; auto v = static_cast(a)->relu_upper_clip(); s.present = v.has_value(); if(v) s.f = static_cast(*v); return s; }}, + {"softplus_beta", true, AttrType::FLOAT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::FLOAT; auto v = static_cast(a)->softplus_beta(); s.present = v.has_value(); if(v) s.f = static_cast(*v); return s; }}, + {"swish_beta", true, AttrType::FLOAT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::FLOAT; auto v = static_cast(a)->swish_beta(); s.present = v.has_value(); if(v) s.f = static_cast(*v); return s; }}, +}; +} // namespace op1 + +// BatchnormBackwardAttributes +namespace op2 { +inline const AttrBinding attributes[] = { + {"dbias_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dbias_tensor_uid()); return s; }}, + {"dscale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dscale_tensor_uid()); return s; }}, + {"dx_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dx_tensor_uid()); return s; }}, + {"dy_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dy_tensor_uid()); return s; }}, + {"inv_variance_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->inv_variance_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"mean_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->mean_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"scale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->scale_tensor_uid()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, +}; +} // namespace op2 + +// BatchnormAttributes +namespace op3 { +inline const AttrBinding attributes[] = { + {"bias_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->bias_tensor_uid()); return s; }}, + {"epsilon_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->epsilon_tensor_uid()); return s; }}, + {"inv_variance_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->inv_variance_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"mean_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->mean_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"momentum_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->momentum_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"next_running_mean_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->next_running_mean_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"next_running_variance_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->next_running_variance_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"prev_running_mean_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->prev_running_mean_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"prev_running_variance_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->prev_running_variance_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"scale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->scale_tensor_uid()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, + {"y_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->y_tensor_uid()); return s; }}, +}; +} // namespace op3 + +// ConvolutionFwdAttributes +namespace op4 { +inline const AttrBinding attributes[] = { + {"conv_mode", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameConvMode(static_cast(a)->conv_mode()); return s; }}, + {"w_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->w_tensor_uid()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, + {"y_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->y_tensor_uid()); return s; }}, +}; +} // namespace op4 + +// ConvolutionBwdAttributes +namespace op5 { +inline const AttrBinding attributes[] = { + {"conv_mode", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameConvMode(static_cast(a)->conv_mode()); return s; }}, + {"dx_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dx_tensor_uid()); return s; }}, + {"dy_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dy_tensor_uid()); return s; }}, + {"w_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->w_tensor_uid()); return s; }}, +}; +} // namespace op5 + +// ConvolutionWrwAttributes +namespace op6 { +inline const AttrBinding attributes[] = { + {"conv_mode", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameConvMode(static_cast(a)->conv_mode()); return s; }}, + {"dw_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dw_tensor_uid()); return s; }}, + {"dy_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dy_tensor_uid()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, +}; +} // namespace op6 + +// BatchnormInferenceAttributesVarianceExt +namespace op7 { +inline const AttrBinding attributes[] = { + {"bias_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->bias_tensor_uid()); return s; }}, + {"epsilon_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->epsilon_tensor_uid()); return s; }}, + {"mean_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->mean_tensor_uid()); return s; }}, + {"scale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->scale_tensor_uid()); return s; }}, + {"variance_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->variance_tensor_uid()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, + {"y_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->y_tensor_uid()); return s; }}, +}; +} // namespace op7 + +// MatmulAttributes +namespace op8 { +inline const AttrBinding attributes[] = { + {"a_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->a_tensor_uid()); return s; }}, + {"b_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->b_tensor_uid()); return s; }}, + {"c_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->c_tensor_uid()); return s; }}, +}; +} // namespace op8 + +// RMSNormAttributes +namespace op9 { +inline const AttrBinding attributes[] = { + {"bias_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->bias_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"epsilon_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->epsilon_tensor_uid()); return s; }}, + {"forward_phase", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameNormFwdPhase(static_cast(a)->forward_phase()); return s; }}, + {"inv_rms_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->inv_rms_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"scale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->scale_tensor_uid()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, + {"y_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->y_tensor_uid()); return s; }}, +}; +} // namespace op9 + +// LayernormAttributes +namespace op10 { +inline const AttrBinding attributes[] = { + {"bias_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->bias_tensor_uid()); return s; }}, + {"epsilon_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->epsilon_tensor_uid()); return s; }}, + {"forward_phase", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameNormFwdPhase(static_cast(a)->forward_phase()); return s; }}, + {"inv_variance_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->inv_variance_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"mean_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->mean_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"normalized_dim_count", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->normalized_dim_count()); return s; }}, + {"scale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->scale_tensor_uid()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, + {"y_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->y_tensor_uid()); return s; }}, +}; +} // namespace op10 + +// SdpaAttributes +namespace op11 { +inline const InputTensorBinding inputTensors[] = { + {"attn_mask", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->attn_mask_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"block_mask", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->block_mask_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"descale_k", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->descale_k_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"descale_q", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->descale_q_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"descale_s", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->descale_s_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"descale_v", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->descale_v_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"dropout_mask", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->dropout_mask_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"dropout_scale", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->dropout_scale_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"k", false, +[](const void* a, std::int64_t& out) -> bool { out = static_cast(a)->k_tensor_uid(); return true; }}, + {"offset", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->offset_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"page_table_k", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->page_table_k_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"page_table_v", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->page_table_v_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"q", false, +[](const void* a, std::int64_t& out) -> bool { out = static_cast(a)->q_tensor_uid(); return true; }}, + {"scale_o", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->scale_o_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"scale_s", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->scale_s_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"scale", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->scale_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"seed", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->seed_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"seq_len_kv", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->seq_len_kv_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"seq_len_q", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->seq_len_q_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"sink_token", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->sink_token_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"v", false, +[](const void* a, std::int64_t& out) -> bool { out = static_cast(a)->v_tensor_uid(); return true; }}, +}; +inline const OutputTensorBinding outputTensors[] = { + {"amax_o", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->amax_o_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"amax_s", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->amax_s_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"max", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->max_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"o", false, +[](const void* a, std::int64_t& out) -> bool { out = static_cast(a)->o_tensor_uid(); return true; }}, + {"rng_dump", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->rng_dump_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"stats", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->stats_tensor_uid(); if(!v) return false; out = *v; return true; }}, + {"sum_exp", true, +[](const void* a, std::int64_t& out) -> bool { auto v = static_cast(a)->sum_exp_tensor_uid(); if(!v) return false; out = *v; return true; }}, +}; +inline const AttrBinding attributes[] = { + {"alibi_mask", false, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; s.present = true; s.b = static_cast(a)->alibi_mask(); return s; }}, + {"attn_scale_value", true, AttrType::FLOAT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::FLOAT; auto v = static_cast(a)->attn_scale_value(); s.present = v.has_value(); if(v) s.f = static_cast(*v); return s; }}, + {"causal_mask", false, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; s.present = true; s.b = static_cast(a)->causal_mask(); return s; }}, + {"causal_mask_bottom_right", false, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; s.present = true; s.b = static_cast(a)->causal_mask_bottom_right(); return s; }}, + {"diagonal_alignment", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameDiagonalAlignment(static_cast(a)->diagonal_alignment()); return s; }}, + {"dropout_probability", true, AttrType::FLOAT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::FLOAT; auto v = static_cast(a)->dropout_probability(); s.present = v.has_value(); if(v) s.f = static_cast(*v); return s; }}, + {"generate_stats", true, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; auto v = static_cast(a)->generate_stats(); s.present = v.has_value(); if(v) s.b = *v; return s; }}, + {"implementation", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameAttentionImplementation(static_cast(a)->implementation()); return s; }}, + {"left_bound", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->left_bound(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"max_seq_len_kv", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->max_seq_len_kv(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"mma_core_mode", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameDataType(static_cast(a)->mma_core_mode()); return s; }}, + {"padding_mask", false, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; s.present = true; s.b = static_cast(a)->padding_mask(); return s; }}, + {"right_bound", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->right_bound(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, +}; +} // namespace op11 + +// BlockScaleDequantizeAttributes +namespace op12 { +inline const AttrBinding attributes[] = { + {"is_negative_scale", false, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; s.present = true; s.b = static_cast(a)->is_negative_scale(); return s; }}, + {"scale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->scale_tensor_uid()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, + {"y_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->y_tensor_uid()); return s; }}, +}; +} // namespace op12 + +// BlockScaleQuantizeAttributes +namespace op13 { +inline const AttrBinding attributes[] = { + {"axis", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->axis(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"block_size", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->block_size()); return s; }}, + {"scale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->scale_tensor_uid()); return s; }}, + {"transpose", false, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; s.present = true; s.b = static_cast(a)->transpose(); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, + {"y_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->y_tensor_uid()); return s; }}, +}; +} // namespace op13 + +// SdpaBackwardAttributes +namespace op14 { +inline const AttrBinding attributes[] = { + {"alibi_mask", false, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; s.present = true; s.b = static_cast(a)->alibi_mask(); return s; }}, + {"attn_mask_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->attn_mask_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"attn_scale_value", true, AttrType::FLOAT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::FLOAT; auto v = static_cast(a)->attn_scale_value(); s.present = v.has_value(); if(v) s.f = static_cast(*v); return s; }}, + {"causal_mask", false, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; s.present = true; s.b = static_cast(a)->causal_mask(); return s; }}, + {"causal_mask_bottom_right", false, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; s.present = true; s.b = static_cast(a)->causal_mask_bottom_right(); return s; }}, + {"dbias_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->dbias_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"diagonal_alignment", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameDiagonalAlignment(static_cast(a)->diagonal_alignment()); return s; }}, + {"dk_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dk_tensor_uid()); return s; }}, + {"do_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->do_tensor_uid()); return s; }}, + {"dq_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dq_tensor_uid()); return s; }}, + {"dropout_mask_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->dropout_mask_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"dropout_probability", true, AttrType::FLOAT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::FLOAT; auto v = static_cast(a)->dropout_probability(); s.present = v.has_value(); if(v) s.f = static_cast(*v); return s; }}, + {"dropout_scale_inv_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->dropout_scale_inv_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"dropout_scale_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->dropout_scale_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"dv_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dv_tensor_uid()); return s; }}, + {"k_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->k_tensor_uid()); return s; }}, + {"left_bound", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->left_bound(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"o_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->o_tensor_uid()); return s; }}, + {"offset_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->offset_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"padding_mask", false, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; s.present = true; s.b = static_cast(a)->padding_mask(); return s; }}, + {"q_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->q_tensor_uid()); return s; }}, + {"right_bound", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->right_bound(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"scale_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->scale_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"seed_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->seed_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"seq_len_kv_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->seq_len_kv_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"seq_len_q_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->seq_len_q_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"stats_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->stats_tensor_uid()); return s; }}, + {"v_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->v_tensor_uid()); return s; }}, +}; +} // namespace op14 + +// RMSNormBackwardAttributes +namespace op16 { +inline const AttrBinding attributes[] = { + {"dbias_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->dbias_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"dscale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dscale_tensor_uid()); return s; }}, + {"dx_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dx_tensor_uid()); return s; }}, + {"dy_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dy_tensor_uid()); return s; }}, + {"inv_rms_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->inv_rms_tensor_uid()); return s; }}, + {"scale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->scale_tensor_uid()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, +}; +} // namespace op16 + +// ReductionAttributes +namespace op17 { +inline const AttrBinding attributes[] = { + {"in_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->in_tensor_uid()); return s; }}, + {"is_deterministic", false, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; s.present = true; s.b = static_cast(a)->is_deterministic(); return s; }}, + {"mode", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameReductionMode(static_cast(a)->mode()); return s; }}, + {"out_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->out_tensor_uid()); return s; }}, +}; +} // namespace op17 + +// ResampleFwdAttributes +namespace op18 { +inline const AttrBinding attributes[] = { + {"generate_index", true, AttrType::BOOL, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::BOOL; auto v = static_cast(a)->generate_index(); s.present = v.has_value(); if(v) s.b = *v; return s; }}, + {"index_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->index_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"padding_mode", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNamePaddingMode(static_cast(a)->padding_mode()); return s; }}, + {"resample_mode", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameResampleMode(static_cast(a)->resample_mode()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, + {"y_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->y_tensor_uid()); return s; }}, +}; +} // namespace op18 + +// LayernormBackwardAttributes +namespace op19 { +inline const AttrBinding attributes[] = { + {"dbias_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dbias_tensor_uid()); return s; }}, + {"dscale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dscale_tensor_uid()); return s; }}, + {"dx_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dx_tensor_uid()); return s; }}, + {"dy_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dy_tensor_uid()); return s; }}, + {"epsilon_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->epsilon_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"inv_variance_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->inv_variance_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"mean_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->mean_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"normalized_dim_count", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->normalized_dim_count()); return s; }}, + {"scale_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->scale_tensor_uid()); return s; }}, + {"x_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->x_tensor_uid()); return s; }}, +}; +} // namespace op19 + +// ResampleBwdAttributes +namespace op20 { +inline const AttrBinding attributes[] = { + {"dx_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dx_tensor_uid()); return s; }}, + {"dy_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->dy_tensor_uid()); return s; }}, + {"index_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->index_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"padding_mode", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNamePaddingMode(static_cast(a)->padding_mode()); return s; }}, + {"resample_mode", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameResampleMode(static_cast(a)->resample_mode()); return s; }}, +}; +} // namespace op20 + +// MoeGroupedMatmulAttributes +namespace op21 { +inline const AttrBinding attributes[] = { + {"first_token_offset_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->first_token_offset_tensor_uid()); return s; }}, + {"mode", false, AttrType::DTYPE, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::DTYPE; s.present = true; s.dtype = ::hipdnn_flatbuffers_sdk::data_objects::EnumNameMoeGroupedMatmulMode(static_cast(a)->mode()); return s; }}, + {"output_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->output_tensor_uid()); return s; }}, + {"token_index_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->token_index_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"token_ks_tensor_uid", true, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; auto v = static_cast(a)->token_ks_tensor_uid(); s.present = v.has_value(); if(v) s.i = static_cast(*v); return s; }}, + {"token_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->token_tensor_uid()); return s; }}, + {"top_k", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->top_k()); return s; }}, + {"weight_tensor_uid", false, AttrType::INT, +[](const void* a) -> ScalarValue { ScalarValue s; s.type = AttrType::INT; s.present = true; s.i = static_cast(static_cast(a)->weight_tensor_uid()); return s; }}, +}; +} // namespace op21 + +inline const OpSchemaEntry entries[] = { + {"BatchnormInferenceAttributes", "BatchnormInferenceAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::BatchnormInferenceAttributes, nullptr, 0u, nullptr, 0u, op0::attributes, 6u}, + {"PointwiseAttributes", "PointwiseAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::PointwiseAttributes, nullptr, 0u, nullptr, 0u, op1::attributes, 12u}, + {"BatchnormBackwardAttributes", "BatchnormBackwardAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::BatchnormBackwardAttributes, nullptr, 0u, nullptr, 0u, op2::attributes, 8u}, + {"BatchnormAttributes", "BatchnormAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::BatchnormAttributes, nullptr, 0u, nullptr, 0u, op3::attributes, 12u}, + {"ConvolutionFwdAttributes", "ConvolutionFwdAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionFwdAttributes, nullptr, 0u, nullptr, 0u, op4::attributes, 4u}, + {"ConvolutionBwdAttributes", "ConvolutionBwdAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionBwdAttributes, nullptr, 0u, nullptr, 0u, op5::attributes, 4u}, + {"ConvolutionWrwAttributes", "ConvolutionWrwAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ConvolutionWrwAttributes, nullptr, 0u, nullptr, 0u, op6::attributes, 4u}, + {"BatchnormInferenceAttributesVarianceExt", "BatchnormInferenceAttributesVarianceExt", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::BatchnormInferenceAttributesVarianceExt, nullptr, 0u, nullptr, 0u, op7::attributes, 7u}, + {"MatmulAttributes", "MatmulAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::MatmulAttributes, nullptr, 0u, nullptr, 0u, op8::attributes, 3u}, + {"RMSNormAttributes", "RMSNormAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::RMSNormAttributes, nullptr, 0u, nullptr, 0u, op9::attributes, 7u}, + {"LayernormAttributes", "LayernormAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::LayernormAttributes, nullptr, 0u, nullptr, 0u, op10::attributes, 9u}, + {"sdpa_fwd", "SdpaAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::SdpaAttributes, op11::inputTensors, 21u, op11::outputTensors, 7u, op11::attributes, 13u}, + {"BlockScaleDequantizeAttributes", "BlockScaleDequantizeAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::BlockScaleDequantizeAttributes, nullptr, 0u, nullptr, 0u, op12::attributes, 4u}, + {"BlockScaleQuantizeAttributes", "BlockScaleQuantizeAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::BlockScaleQuantizeAttributes, nullptr, 0u, nullptr, 0u, op13::attributes, 6u}, + {"SdpaBackwardAttributes", "SdpaBackwardAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::SdpaBackwardAttributes, nullptr, 0u, nullptr, 0u, op14::attributes, 28u}, + {"CustomOpAttributes", "CustomOpAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::CustomOpAttributes, nullptr, 0u, nullptr, 0u, nullptr, 0u}, + {"RMSNormBackwardAttributes", "RMSNormBackwardAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::RMSNormBackwardAttributes, nullptr, 0u, nullptr, 0u, op16::attributes, 7u}, + {"ReductionAttributes", "ReductionAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ReductionAttributes, nullptr, 0u, nullptr, 0u, op17::attributes, 4u}, + {"ResampleFwdAttributes", "ResampleFwdAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ResampleFwdAttributes, nullptr, 0u, nullptr, 0u, op18::attributes, 6u}, + {"LayernormBackwardAttributes", "LayernormBackwardAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::LayernormBackwardAttributes, nullptr, 0u, nullptr, 0u, op19::attributes, 10u}, + {"ResampleBwdAttributes", "ResampleBwdAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::ResampleBwdAttributes, nullptr, 0u, nullptr, 0u, op20::attributes, 5u}, + {"MoeGroupedMatmulAttributes", "MoeGroupedMatmulAttributes", ::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes::MoeGroupedMatmulAttributes, nullptr, 0u, nullptr, 0u, op21::attributes, 8u}, +}; + +} // namespace generated + +inline const OpSchemaEntry* opSchemaEntries(std::size_t& count) +{ + count = sizeof(generated::entries) / sizeof(generated::entries[0]); + return generated::entries; +} + +inline const OpSchemaEntry* lookupOpByName(std::string_view opcode) +{ + for(const auto& e : generated::entries) + if(e.opcode == opcode) + return &e; + return nullptr; +} + +inline const OpSchemaEntry* lookupOpByType(::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes attributesType) +{ + for(const auto& e : generated::entries) + if(e.attributesType == attributesType) + return &e; + return nullptr; +} + +} // namespace hipdnn_flatbuffers_sdk::umd diff --git a/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/utilities/Uuid.hpp b/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/utilities/Uuid.hpp new file mode 100644 index 000000000000..cd8644b1e38b --- /dev/null +++ b/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/utilities/Uuid.hpp @@ -0,0 +1,93 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace hipdnn_flatbuffers_sdk::utilities +{ + +using UuidBytes = std::array; + +inline UuidBytes toUuidBytes(const data_objects::Uuid& uuid) +{ + UuidBytes bytes{}; + for(size_t i = 0; i < bytes.size(); ++i) + { + bytes[i] = uuid.bytes()->Get(static_cast(i)); + } + return bytes; +} + +inline data_objects::Uuid toFlatbufferUuid(const UuidBytes& bytes) +{ + return {flatbuffers::span(bytes)}; +} + +inline bool isUuidV4(const UuidBytes& bytes) +{ + return (bytes[6] & 0xf0U) == 0x40U && (bytes[8] & 0xc0U) == 0x80U; +} + +inline std::string formatUuid(const UuidBytes& bytes) +{ + static constexpr std::string_view HEX = "0123456789abcdef"; + std::string result(36, '-'); + size_t output = 0; + for(const auto byte : bytes) + { + if(output == 8 || output == 13 || output == 18 || output == 23) + { + ++output; + } + result[output++] = HEX[byte >> 4U]; + result[output++] = HEX[byte & 0x0fU]; + } + return result; +} + +inline UuidBytes parseUuid(std::string_view text) +{ + if(text.size() != 36 || text[8] != '-' || text[13] != '-' || text[18] != '-' || text[23] != '-') + { + throw std::invalid_argument("UUID must use xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx format"); + } + const auto parseHexDigit = [](char value) -> uint8_t { + if(value >= '0' && value <= '9') + { + return static_cast(value - '0'); + } + if(value >= 'a' && value <= 'f') + { + return static_cast(value - 'a' + 10); + } + if(value >= 'A' && value <= 'F') + { + return static_cast(value - 'A' + 10); + } + throw std::invalid_argument("UUID contains a non-hexadecimal character"); + }; + + UuidBytes bytes{}; + size_t input = 0; + for(auto& byte : bytes) + { + if(input == 8 || input == 13 || input == 18 || input == 23) + { + ++input; + } + const auto high = parseHexDigit(text[input++]); + const auto low = parseHexDigit(text[input++]); + byte = static_cast((high << 4U) | low); + } + return bytes; +} + +} // namespace hipdnn_flatbuffers_sdk::utilities diff --git a/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/utilities/json/Graph.hpp b/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/utilities/json/Graph.hpp index 414a192b6361..ee9ee493b408 100644 --- a/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/utilities/json/Graph.hpp +++ b/projects/hipdnn/flatbuffers_sdk/include/hipdnn_flatbuffers_sdk/utilities/json/Graph.hpp @@ -5,6 +5,7 @@ #ifndef HIPDNN_FLATBUFFERS_SDK_SKIP_JSON_LIB #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include #include +#include namespace hipdnn_flatbuffers_sdk::data_objects { @@ -160,6 +162,10 @@ inline void to_json(nlohmann::json& graphJson, const data_objects::Graph& graph) { graphJson["preferred_engine_id"] = graph.preferred_engine_id().value(); } + if(graph.id() != nullptr) + { + graphJson["id"] = utilities::formatUuid(utilities::toUuidBytes(*graph.id())); + } if(const auto* version = graph.min_required_engine_api_version()) { graphJson["min_required_engine_api_version"] = { @@ -254,6 +260,16 @@ inline auto to(flatbuffers::FlatBufferBuilder& builder, preferredEngineId = entry["preferred_engine_id"].get(); } const bool isOverrideShapeEnabled = entry.value("is_override_shape_enabled", false); + std::optional id; + if(entry.contains("id")) + { + if(!entry["id"].is_string()) + { + throw std::runtime_error("Graph id must be a canonical UUID string"); + } + id.emplace( + utilities::toFlatbufferUuid(utilities::parseUuid(entry["id"].get()))); + } const data_objects::EngineApiVersion* minRequiredEngineApiVersion = nullptr; data_objects::EngineApiVersion parsedMinRequiredEngineApiVersion; @@ -278,7 +294,8 @@ inline auto to(flatbuffers::FlatBufferBuilder& builder, &nodes, preferredEngineId, isOverrideShapeEnabled, - minRequiredEngineApiVersion); + minRequiredEngineApiVersion, + id ? &*id : nullptr); } } diff --git a/projects/hipdnn/flatbuffers_sdk/schemas/data_types.fbs b/projects/hipdnn/flatbuffers_sdk/schemas/data_types.fbs index 344cc2d879d0..6c24d53118b7 100644 --- a/projects/hipdnn/flatbuffers_sdk/schemas/data_types.fbs +++ b/projects/hipdnn/flatbuffers_sdk/schemas/data_types.fbs @@ -24,3 +24,16 @@ enum DataType : byte { FP8_E4M3_FNUZ = 17, FP8_E5M2_FNUZ = 18, } + +// UMD binding annotations (RFC 0018 Appendix B). Declared once here, in the shared +// include, and applied to each op's attribute table. `umd_opcode` is a table-level +// attribute giving the op's UMD-facing shorthand (e.g. "sdpa_fwd"); a UMD file's +// node `op` names it, and the registry keys on it (falling back to the table type +// name when absent). `umd_input_tensor` / `umd_output_tensor` mark a tensor-UID +// edge and `umd_name` gives the name a UMD binds it by; the input/output flag +// already fixes that the binding is a tensor, so no separate type attribute is +// carried. Unannotated fields are scalar attributes. +attribute "umd_opcode"; +attribute "umd_input_tensor"; +attribute "umd_output_tensor"; +attribute "umd_name"; diff --git a/projects/hipdnn/flatbuffers_sdk/schemas/graph.fbs b/projects/hipdnn/flatbuffers_sdk/schemas/graph.fbs index e02d55a36877..f84e03cbd7ca 100644 --- a/projects/hipdnn/flatbuffers_sdk/schemas/graph.fbs +++ b/projects/hipdnn/flatbuffers_sdk/schemas/graph.fbs @@ -71,6 +71,11 @@ struct EngineApiVersion { patch: uint; } +// Opaque 128-bit value used for graph identities. +struct Uuid { + bytes: [ubyte:16]; +} + // Hipdnn expects that the serialized graph is sorted in topological order, has no cycles, // and is fully connected (no orphan nodes). Additionally, all tensors in the graph must have unique uids. table Graph { @@ -88,6 +93,13 @@ table Graph { // graphs) -- callers must treat a missing field as the baseline "1.0.0" floor, // exactly like the field's absence used to mean min_reader_version == 0. min_required_engine_api_version: EngineApiVersion; + // Stable logical graph identity across serialization round trips. hipDNN mints UUID v4 + // values at finalization; readers preserve any 128-bit value without enforcing a UUID + // version or variant. It is neither a content hash nor a unique live-object token: + // distinct IDs may have identical content, while one ID must not represent different + // immutable graph content. Absent on legacy and locally constructed, unfinalized graphs; + // an ID inherited through deserialization may be present before local finalization. + id: Uuid; } root_type Graph; diff --git a/projects/hipdnn/flatbuffers_sdk/schemas/sdpa_attributes.fbs b/projects/hipdnn/flatbuffers_sdk/schemas/sdpa_attributes.fbs index a3cc168c87b5..26a2538d9295 100644 --- a/projects/hipdnn/flatbuffers_sdk/schemas/sdpa_attributes.fbs +++ b/projects/hipdnn/flatbuffers_sdk/schemas/sdpa_attributes.fbs @@ -16,46 +16,46 @@ enum AttentionImplementation : byte { UNIFIED = 2, } -table SdpaAttributes { +table SdpaAttributes (umd_opcode: "sdpa_fwd") { // Required input tensor UIDs - q_tensor_uid: long; - k_tensor_uid: long; - v_tensor_uid: long; + q_tensor_uid: long (umd_input_tensor, umd_name: "q"); + k_tensor_uid: long (umd_input_tensor, umd_name: "k"); + v_tensor_uid: long (umd_input_tensor, umd_name: "v"); // Required output tensor UID - o_tensor_uid: long; + o_tensor_uid: long (umd_output_tensor, umd_name: "o"); // Optional input tensor UIDs - attn_mask_tensor_uid: long = null; // Additive attention bias (Bias) - scale_tensor_uid: long = null; // Attention scale tensor (Attn_scale) - seq_len_q_tensor_uid: long = null; - seq_len_kv_tensor_uid: long = null; - seed_tensor_uid: long = null; // Dropout seed - offset_tensor_uid: long = null; // Dropout offset - dropout_mask_tensor_uid: long = null; - dropout_scale_tensor_uid: long = null; - page_table_k_tensor_uid: long = null; - page_table_v_tensor_uid: long = null; - block_mask_tensor_uid: long = null; - sink_token_tensor_uid: long = null; + attn_mask_tensor_uid: long = null (umd_input_tensor, umd_name: "attn_mask"); // Additive attention bias (Bias) + scale_tensor_uid: long = null (umd_input_tensor, umd_name: "scale"); // Attention scale tensor (Attn_scale) + seq_len_q_tensor_uid: long = null (umd_input_tensor, umd_name: "seq_len_q"); + seq_len_kv_tensor_uid: long = null (umd_input_tensor, umd_name: "seq_len_kv"); + seed_tensor_uid: long = null (umd_input_tensor, umd_name: "seed"); // Dropout seed + offset_tensor_uid: long = null (umd_input_tensor, umd_name: "offset"); // Dropout offset + dropout_mask_tensor_uid: long = null (umd_input_tensor, umd_name: "dropout_mask"); + dropout_scale_tensor_uid: long = null (umd_input_tensor, umd_name: "dropout_scale"); + page_table_k_tensor_uid: long = null (umd_input_tensor, umd_name: "page_table_k"); + page_table_v_tensor_uid: long = null (umd_input_tensor, umd_name: "page_table_v"); + block_mask_tensor_uid: long = null (umd_input_tensor, umd_name: "block_mask"); + sink_token_tensor_uid: long = null (umd_input_tensor, umd_name: "sink_token"); // FP8-specific input tensor UIDs - descale_q_tensor_uid: long = null; - descale_k_tensor_uid: long = null; - descale_v_tensor_uid: long = null; - descale_s_tensor_uid: long = null; - scale_s_tensor_uid: long = null; - scale_o_tensor_uid: long = null; + descale_q_tensor_uid: long = null (umd_input_tensor, umd_name: "descale_q"); + descale_k_tensor_uid: long = null (umd_input_tensor, umd_name: "descale_k"); + descale_v_tensor_uid: long = null (umd_input_tensor, umd_name: "descale_v"); + descale_s_tensor_uid: long = null (umd_input_tensor, umd_name: "descale_s"); + scale_s_tensor_uid: long = null (umd_input_tensor, umd_name: "scale_s"); + scale_o_tensor_uid: long = null (umd_input_tensor, umd_name: "scale_o"); // Optional output tensor UIDs - stats_tensor_uid: long = null; - max_tensor_uid: long = null; // Logit max (Max) - sum_exp_tensor_uid: long = null; // Score sum exp (Sum_exp) - rng_dump_tensor_uid: long = null; // RNG dump for dropout debugging + stats_tensor_uid: long = null (umd_output_tensor, umd_name: "stats"); + max_tensor_uid: long = null (umd_output_tensor, umd_name: "max"); // Logit max (Max) + sum_exp_tensor_uid: long = null (umd_output_tensor, umd_name: "sum_exp"); // Score sum exp (Sum_exp) + rng_dump_tensor_uid: long = null (umd_output_tensor, umd_name: "rng_dump"); // RNG dump for dropout debugging // FP8-specific output tensor UIDs - amax_s_tensor_uid: long = null; // Absolute max for attention scores - amax_o_tensor_uid: long = null; // Absolute max for output tensor + amax_s_tensor_uid: long = null (umd_output_tensor, umd_name: "amax_s"); // Absolute max for attention scores + amax_o_tensor_uid: long = null (umd_output_tensor, umd_name: "amax_o"); // Absolute max for output tensor // Boolean flags generate_stats: bool = null; diff --git a/projects/hipdnn/flatbuffers_sdk/tests/CMakeLists.txt b/projects/hipdnn/flatbuffers_sdk/tests/CMakeLists.txt index c3e5a55d102b..3f0af1241a78 100644 --- a/projects/hipdnn/flatbuffers_sdk/tests/CMakeLists.txt +++ b/projects/hipdnn/flatbuffers_sdk/tests/CMakeLists.txt @@ -29,6 +29,8 @@ add_executable( utilities/TestFlatbufferUtils.cpp utilities/TestJson.cpp utilities/TestPointwiseValidation.cpp + utilities/TestUuid.cpp + umd/TestOpSchemaRegistry.cpp ) target_compile_options(hipdnn_flatbuffers_sdk_tests PRIVATE ${HIPDNN_WARNING_COMPILE_OPTIONS} ${HIPDNN_NO_RTTI_OPTIONS}) @@ -74,3 +76,27 @@ if(ENABLE_CLANG_TIDY AND _fb_include_dirs) endif() add_hipdnn_test(hipdnn_flatbuffers_sdk_tests ${CMAKE_CURRENT_BINARY_DIR}) + +# RFC 0018 B.3: registry generation must fail closed on an annotation violation +# rather than emit a wrong binding. flatc accepts the fixture (it is valid +# FlatBuffers); umd_registry_gen must reject it (non-zero exit). Gated on the +# generator target, which exists only when HIPDNN_GENERATE_SDK_HEADERS is ON. +if(TARGET umd_registry_gen) + set(_umd_bad_bfbs "${CMAKE_CURRENT_BINARY_DIR}/umd/bad_nonlong.bfbs") + hipdnn_generate_bfbs( + SCHEMA "${CMAKE_CURRENT_SOURCE_DIR}/umd/fixtures/bad_nonlong.fbs" + SCHEMAS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/umd/fixtures" + OUTPUT "${_umd_bad_bfbs}" + ) + add_custom_target(generate_umd_bad_bfbs + DEPENDS "${_umd_bad_bfbs}" + COMMENT "UMD B.3 negative fixture schema generated" + ) + add_dependencies(hipdnn_flatbuffers_sdk_tests generate_umd_bad_bfbs umd_registry_gen) + add_test( + NAME umd_registry_gen_rejects_bad_schema + COMMAND $ "${_umd_bad_bfbs}" + "${CMAKE_CURRENT_BINARY_DIR}/umd/bad_nonlong_out.hpp" + ) + set_tests_properties(umd_registry_gen_rejects_bad_schema PROPERTIES WILL_FAIL TRUE) +endif() diff --git a/projects/hipdnn/flatbuffers_sdk/tests/umd/TestOpSchemaRegistry.cpp b/projects/hipdnn/flatbuffers_sdk/tests/umd/TestOpSchemaRegistry.cpp new file mode 100644 index 000000000000..b380afd0b4bb --- /dev/null +++ b/projects/hipdnn/flatbuffers_sdk/tests/umd/TestOpSchemaRegistry.cpp @@ -0,0 +1,238 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +// +// RFC 0018 Phase 0: proves the generated op-schema registry. Two claims: +// 1. Shape + optionality parity -- the registry lists SDPA's input/output tensor +// names and scalar attributes with the optionality the schema `= null` +// defaults imply (matching the generated header's Optional fields). +// 2. Accessor value round-trip -- the generated typed readers, driven off a +// live SdpaAttributes flatbuffer, return the exact stored values. Registry +// shape alone is necessary but not sufficient; this proves the accessors +// read the right field (the binding). + +#include +#include +#include + +#include + +#include + +#include + +namespace fb = flatbuffers; +namespace umd = hipdnn_flatbuffers_sdk::umd; +namespace data = hipdnn_flatbuffers_sdk::data_objects; + +namespace +{ + +const umd::InputTensorBinding* findInputTensor(const umd::OpSchemaEntry* e, std::string_view name) +{ + for(std::size_t i = 0; i < e->inputTensorCount; ++i) + { + if(e->inputTensors[i].name == name) + { + return &e->inputTensors[i]; + } + } + return nullptr; +} + +const umd::OutputTensorBinding* findOutputTensor(const umd::OpSchemaEntry* e, std::string_view name) +{ + for(std::size_t i = 0; i < e->outputTensorCount; ++i) + { + if(e->outputTensors[i].name == name) + { + return &e->outputTensors[i]; + } + } + return nullptr; +} + +const umd::AttrBinding* findAttr(const umd::OpSchemaEntry* e, std::string_view name) +{ + for(std::size_t i = 0; i < e->attributeCount; ++i) + { + if(e->attributes[i].name == name) + { + return &e->attributes[i]; + } + } + return nullptr; +} + +const umd::OpSchemaEntry* sdpaEntry() +{ + return umd::lookupOpByName("sdpa_fwd"); +} + +// A minimal SDPA attribute table with known values; optionals left unset stay +// absent. Owned by the returned builder; read via GetRoot on its buffer. +fb::FlatBufferBuilder buildSdpa() +{ + fb::FlatBufferBuilder b; + data::SdpaAttributesBuilder sb(b); + sb.add_q_tensor_uid(101); + sb.add_k_tensor_uid(102); + sb.add_v_tensor_uid(103); + sb.add_o_tensor_uid(201); + sb.add_dropout_probability(0.25f); // optional present + sb.add_alibi_mask(true); // required bool + // attn_mask / page_table_* / generate_stats left unset -> absent + b.Finish(sb.Finish()); + return b; +} + +} // namespace + +TEST(TestUmdOpSchemaRegistry, SdpaEntryResolvesByNameAndType) +{ + const auto* byName = umd::lookupOpByName("sdpa_fwd"); // the umd_opcode shorthand + const auto* byType = umd::lookupOpByType(data::NodeAttributes::SdpaAttributes); + ASSERT_NE(byName, nullptr); + ASSERT_NE(byType, nullptr); + EXPECT_EQ(byName, byType); + EXPECT_EQ(byName->opcode, "sdpa_fwd"); + EXPECT_EQ(byName->tableName, "SdpaAttributes"); + EXPECT_EQ(umd::lookupOpByName("SdpaAttributes"), nullptr); // table name is not the key + EXPECT_EQ(umd::lookupOpByName("NoSuchOp"), nullptr); +} + +TEST(TestUmdOpSchemaRegistry, SdpaRequiredOperandsAndResult) +{ + const auto* e = sdpaEntry(); + ASSERT_NE(e, nullptr); + for(const std::string_view name : {"q", "k", "v"}) + { + const auto* op = findInputTensor(e, name); + ASSERT_NE(op, nullptr) << name; + EXPECT_FALSE(op->optional) << name; + } + const auto* o = findOutputTensor(e, "o"); + ASSERT_NE(o, nullptr); + EXPECT_FALSE(o->optional); +} + +TEST(TestUmdOpSchemaRegistry, SdpaOptionalOperandsClassifiedOptional) +{ + const auto* e = sdpaEntry(); + ASSERT_NE(e, nullptr); + for(const std::string_view name : + {"attn_mask", "page_table_k", "page_table_v", "scale", "seed"}) + { + const auto* op = findInputTensor(e, name); + ASSERT_NE(op, nullptr) << name; + EXPECT_TRUE(op->optional) << name; + } + // Optional results too. + for(const std::string_view name : {"stats", "amax_s", "amax_o"}) + { + const auto* r = findOutputTensor(e, name); + ASSERT_NE(r, nullptr) << name; + EXPECT_TRUE(r->optional) << name; + } +} + +// Optionality parity with the schema `= null` defaults / generated Optional. +TEST(TestUmdOpSchemaRegistry, SdpaScalarAttributeOptionalityParity) +{ + const auto* e = sdpaEntry(); + ASSERT_NE(e, nullptr); + + // `= null` scalars are optional. + for(const std::string_view name : {"generate_stats", + "dropout_probability", + "attn_scale_value", + "left_bound", + "right_bound", + "max_seq_len_kv"}) + { + const auto* a = findAttr(e, name); + ASSERT_NE(a, nullptr) << name; + EXPECT_TRUE(a->optional) << name; + } + // Defaulted (non-null) scalars/enums are required. + for(const std::string_view name : {"alibi_mask", + "padding_mask", + "causal_mask", + "causal_mask_bottom_right", + "diagonal_alignment"}) + { + const auto* a = findAttr(e, name); + ASSERT_NE(a, nullptr) << name; + EXPECT_FALSE(a->optional) << name; + } +} + +TEST(TestUmdOpSchemaRegistry, SdpaScalarAttributeTypes) +{ + const auto* e = sdpaEntry(); + ASSERT_NE(e, nullptr); + EXPECT_EQ(findAttr(e, "alibi_mask")->type, umd::AttrType::BOOL); + EXPECT_EQ(findAttr(e, "dropout_probability")->type, umd::AttrType::FLOAT); + EXPECT_EQ(findAttr(e, "left_bound")->type, umd::AttrType::INT); + EXPECT_EQ(findAttr(e, "diagonal_alignment")->type, umd::AttrType::DTYPE); +} + +TEST(TestUmdOpSchemaRegistry, AccessorRoundTripReadsLiveValues) +{ + const auto* e = sdpaEntry(); + ASSERT_NE(e, nullptr); + auto b = buildSdpa(); + const auto* attrs = fb::GetRoot(b.GetBufferPointer()); + const void* a = attrs; + + std::int64_t uid = 0; + ASSERT_TRUE(findInputTensor(e, "q")->read(a, uid)); + EXPECT_EQ(uid, 101); + ASSERT_TRUE(findInputTensor(e, "k")->read(a, uid)); + EXPECT_EQ(uid, 102); + ASSERT_TRUE(findInputTensor(e, "v")->read(a, uid)); + EXPECT_EQ(uid, 103); + ASSERT_TRUE(findOutputTensor(e, "o")->read(a, uid)); + EXPECT_EQ(uid, 201); +} + +TEST(TestUmdOpSchemaRegistry, AbsentOptionalOperandReadsFalse) +{ + const auto* e = sdpaEntry(); + ASSERT_NE(e, nullptr); + auto b = buildSdpa(); + const auto* attrs = fb::GetRoot(b.GetBufferPointer()); + const void* a = attrs; + + std::int64_t uid = -1; + EXPECT_FALSE(findInputTensor(e, "attn_mask")->read(a, uid)); + EXPECT_FALSE(findInputTensor(e, "page_table_k")->read(a, uid)); +} + +TEST(TestUmdOpSchemaRegistry, ScalarReadersReflectPresenceAndValue) +{ + const auto* e = sdpaEntry(); + ASSERT_NE(e, nullptr); + auto b = buildSdpa(); + const auto* attrs = fb::GetRoot(b.GetBufferPointer()); + const void* a = attrs; + + const umd::ScalarValue alibi = findAttr(e, "alibi_mask")->read(a); + EXPECT_EQ(alibi.type, umd::AttrType::BOOL); + EXPECT_TRUE(alibi.present); + EXPECT_TRUE(alibi.b); + + const umd::ScalarValue drop = findAttr(e, "dropout_probability")->read(a); + EXPECT_EQ(drop.type, umd::AttrType::FLOAT); + EXPECT_TRUE(drop.present); + EXPECT_DOUBLE_EQ(drop.f, 0.25); + + const umd::ScalarValue stats = findAttr(e, "generate_stats")->read(a); + EXPECT_EQ(stats.type, umd::AttrType::BOOL); + EXPECT_FALSE(stats.present); // optional, unset + + const umd::ScalarValue align = findAttr(e, "diagonal_alignment")->read(a); + EXPECT_EQ(align.type, umd::AttrType::DTYPE); + EXPECT_TRUE(align.present); + ASSERT_NE(align.dtype, nullptr); + EXPECT_STREQ(align.dtype, "TOP_LEFT"); +} diff --git a/projects/hipdnn/flatbuffers_sdk/tests/umd/fixtures/bad_nonlong.fbs b/projects/hipdnn/flatbuffers_sdk/tests/umd/fixtures/bad_nonlong.fbs new file mode 100644 index 000000000000..47c0a3f07b6a --- /dev/null +++ b/projects/hipdnn/flatbuffers_sdk/tests/umd/fixtures/bad_nonlong.fbs @@ -0,0 +1,22 @@ +// RFC 0018 B.3 negative fixture: a umd_input_tensor on a non-`long` field must +// fail registry generation (fail closed), not emit a wrong binding. Consumed by +// the umd_registry_gen_rejects_bad_schema CTest, which expects a non-zero exit. +namespace hipdnn_flatbuffers_sdk.data_objects; + +attribute "umd_input_tensor"; +attribute "umd_output_tensor"; +attribute "umd_name"; + +table BadAttributes { + x: int (umd_input_tensor, umd_name: "X"); // violation: input tensor UID must be `long` +} + +union NodeAttributes { + BadAttributes +} + +table Node { + attributes: NodeAttributes; +} + +root_type Node; diff --git a/projects/hipdnn/flatbuffers_sdk/tests/utilities/TestJson.cpp b/projects/hipdnn/flatbuffers_sdk/tests/utilities/TestJson.cpp index 0ebb7f91ab84..68340cb4b4ff 100644 --- a/projects/hipdnn/flatbuffers_sdk/tests/utilities/TestJson.cpp +++ b/projects/hipdnn/flatbuffers_sdk/tests/utilities/TestJson.cpp @@ -6,15 +6,19 @@ #include #include +#include #include #include #include #include +#include #include #include #include #include #include +#include +#include using namespace hipdnn_flatbuffers_sdk::data_objects; @@ -40,6 +44,61 @@ void toJsonAndBackTestSuite(const hipdnn_flatbuffers_sdk::data_objects::Graph* g } // namespace +TEST(TestJson, GraphUuidRoundTripsAsCanonicalString) +{ + auto builder = hipdnn_test_sdk::utilities::createEmptyValidGraph(); + auto graph = hipdnn_flatbuffers_sdk::data_objects::GetGraph(builder.GetBufferPointer()); + const auto id + = hipdnn_flatbuffers_sdk::utilities::parseUuid("01234567-89AB-4DEF-8123-456789ABCDEF"); + const auto uuid = hipdnn_flatbuffers_sdk::utilities::toFlatbufferUuid(id); + + auto graphT = std::unique_ptr(graph->UnPack()); + graphT->id = std::make_unique(uuid); + flatbuffers::FlatBufferBuilder identifiedBuilder; + identifiedBuilder.Finish(Graph::Pack(identifiedBuilder, graphT.get())); + const auto* identifiedGraph = GetGraph(identifiedBuilder.GetBufferPointer()); + + const nlohmann::json graphJson = *identifiedGraph; + ASSERT_TRUE(graphJson.contains("id")); + EXPECT_EQ(graphJson.at("id"), "01234567-89ab-4def-8123-456789abcdef"); + + flatbuffers::FlatBufferBuilder roundTripBuilder; + roundTripBuilder.Finish(hipdnn_flatbuffers_sdk::json::to(roundTripBuilder, graphJson)); + const auto* roundTripped = GetGraph(roundTripBuilder.GetBufferPointer()); + ASSERT_NE(roundTripped->id(), nullptr); + EXPECT_EQ(hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*roundTripped->id()), id); +} + +TEST(TestJson, LegacyGraphOmitsUuidAndStillRoundTrips) +{ + auto builder = hipdnn_test_sdk::utilities::createEmptyValidGraph(); + const auto* graph = GetGraph(builder.GetBufferPointer()); + const nlohmann::json graphJson = *graph; + EXPECT_FALSE(graphJson.contains("id")); + + flatbuffers::FlatBufferBuilder roundTripBuilder; + roundTripBuilder.Finish(hipdnn_flatbuffers_sdk::json::to(roundTripBuilder, graphJson)); + EXPECT_EQ(GetGraph(roundTripBuilder.GetBufferPointer())->id(), nullptr); +} + +TEST(TestJson, RejectsInvalidGraphUuid) +{ + auto builder = hipdnn_test_sdk::utilities::createEmptyValidGraph(); + const auto* graph = GetGraph(builder.GetBufferPointer()); + auto graphJson = nlohmann::json(*graph); + + // A non-string id fails the type check; a malformed string fails parsing. + graphJson["id"] = 42; + flatbuffers::FlatBufferBuilder nonStringBuilder; + EXPECT_THROW((void)hipdnn_flatbuffers_sdk::json::to(nonStringBuilder, graphJson), + std::runtime_error); + + graphJson["id"] = "not-a-uuid"; + flatbuffers::FlatBufferBuilder malformedBuilder; + EXPECT_THROW((void)hipdnn_flatbuffers_sdk::json::to(malformedBuilder, graphJson), + std::invalid_argument); +} + TEST(TestJson, GraphToJsonAndBack) { auto nodeAttributeValues = EnumValuesNodeAttributes(); diff --git a/projects/hipdnn/flatbuffers_sdk/tests/utilities/TestUuid.cpp b/projects/hipdnn/flatbuffers_sdk/tests/utilities/TestUuid.cpp new file mode 100644 index 000000000000..e4e3ce0c4e69 --- /dev/null +++ b/projects/hipdnn/flatbuffers_sdk/tests/utilities/TestUuid.cpp @@ -0,0 +1,168 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include + +#include +#include +#include +#include + +using hipdnn_flatbuffers_sdk::utilities::formatUuid; +using hipdnn_flatbuffers_sdk::utilities::isUuidV4; +using hipdnn_flatbuffers_sdk::utilities::parseUuid; +using hipdnn_flatbuffers_sdk::utilities::toFlatbufferUuid; +using hipdnn_flatbuffers_sdk::utilities::toUuidBytes; +using hipdnn_flatbuffers_sdk::utilities::UuidBytes; + +namespace +{ +constexpr UuidBytes ALL_ZERO_BYTES{}; +constexpr UuidBytes ALL_ONES_BYTES{ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; +} // namespace + +TEST(TestUuid, FormatsAndParsesCanonicalText) +{ + const auto bytes = parseUuid("01234567-89AB-4DEF-8123-456789ABCDEF"); + EXPECT_EQ(formatUuid(bytes), "01234567-89ab-4def-8123-456789abcdef"); + EXPECT_TRUE(isUuidV4(bytes)); + + const auto flatbufferUuid = toFlatbufferUuid(bytes); + EXPECT_EQ(toUuidBytes(flatbufferUuid), bytes); +} + +TEST(TestUuid, FormatUuidLowercasesHexDigits) +{ + const UuidBytes bytes{0xAB, + 0xCD, + 0xEF, + 0x01, + 0x23, + 0x45, + 0x67, + 0x89, + 0xAB, + 0xCD, + 0xEF, + 0x01, + 0x23, + 0x45, + 0x67, + 0x89}; + EXPECT_EQ(formatUuid(bytes), "abcdef01-2345-6789-abcd-ef0123456789"); +} + +TEST(TestUuid, FormatUuidPlacesDashesAtCanonicalPositions) +{ + const std::string text = formatUuid(ALL_ZERO_BYTES); + ASSERT_EQ(text.size(), 36U); + EXPECT_EQ(text[8], '-'); + EXPECT_EQ(text[13], '-'); + EXPECT_EQ(text[18], '-'); + EXPECT_EQ(text[23], '-'); + EXPECT_EQ(text, "00000000-0000-0000-0000-000000000000"); +} + +TEST(TestUuid, FormatUuidHandlesAllOnesBoundary) +{ + EXPECT_EQ(formatUuid(ALL_ONES_BYTES), "ffffffff-ffff-ffff-ffff-ffffffffffff"); +} + +TEST(TestUuid, ParseUuidIsCaseInsensitiveAndRoundTrips) +{ + EXPECT_EQ(parseUuid("ABCDEF01-2345-6789-ABCD-EF0123456789"), + parseUuid("abcdef01-2345-6789-abcd-ef0123456789")); +} + +TEST(TestUuid, ParseUuidRejectsWrongLength) +{ + EXPECT_THROW((void)parseUuid("01234567-89ab-4def-8123-456789abcde"), std::invalid_argument); + EXPECT_THROW((void)parseUuid("01234567-89ab-4def-8123-456789abcdef0"), std::invalid_argument); + EXPECT_THROW((void)parseUuid(""), std::invalid_argument); +} + +TEST(TestUuid, ParseUuidRejectsMisplacedDashes) +{ + // Length 36, but the dash sits one column right of its canonical slot each time, + // shifting a hex digit into it. + EXPECT_THROW((void)parseUuid("01234567_89ab-4def-8123-456789abcdef"), std::invalid_argument); + EXPECT_THROW((void)parseUuid("01234567-89ab_4def-8123-456789abcdef"), std::invalid_argument); + EXPECT_THROW((void)parseUuid("01234567-89ab-4def_8123-456789abcdef"), std::invalid_argument); + EXPECT_THROW((void)parseUuid("01234567-89ab-4def-8123_456789abcdef"), std::invalid_argument); +} + +TEST(TestUuid, ParseUuidRejectsNonHexCharacters) +{ + EXPECT_THROW((void)parseUuid("g1234567-89ab-4def-8123-456789abcdef"), std::invalid_argument); + EXPECT_THROW((void)parseUuid("01234567-89ab-4def-8123-456789abcdeZ"), std::invalid_argument); + EXPECT_THROW((void)parseUuid("0123456 -89ab-4def-8123-456789abcdef"), std::invalid_argument); +} + +TEST(TestUuid, ParseUuidAcceptsAllHexDigitRanges) +{ + // Exercises every branch of parseHexDigit: '0'-'9', 'a'-'f', 'A'-'F'. + const auto bytes = parseUuid("01234567-89ab-cdEF-ABcd-ef0123456789"); + EXPECT_EQ(formatUuid(bytes), "01234567-89ab-cdef-abcd-ef0123456789"); +} + +TEST(TestUuid, ParsesOpaque128BitValuesRegardlessOfVersionOrVariant) +{ + for(const auto& id : {"01234567-89ab-3def-8123-456789abcdef", + "01234567-89ab-4def-4123-456789abcdef", + "00000000-0000-0000-0000-000000000000", + "ffffffff-ffff-ffff-ffff-ffffffffffff"}) + { + EXPECT_EQ(formatUuid(parseUuid(id)), id); + } +} + +TEST(TestUuid, IsUuidV4ChecksVersionNibbleIndependentlyOfVariant) +{ + // Version nibble (byte 6 high bits) must be 0100; variant (byte 8 high bits) checked + // separately, so wrong version with a valid-looking variant still fails. + EXPECT_TRUE(isUuidV4(parseUuid("01234567-89ab-4000-8123-456789abcdef"))); + EXPECT_FALSE(isUuidV4(parseUuid("01234567-89ab-3000-8123-456789abcdef"))); + EXPECT_FALSE(isUuidV4(parseUuid("01234567-89ab-5000-8123-456789abcdef"))); +} + +TEST(TestUuid, IsUuidV4ChecksVariantNibbleIndependentlyOfVersion) +{ + // Variant nibble (byte 8 high bits) must be 10xx (0x80-0xbf); version held at v4. + EXPECT_TRUE(isUuidV4(parseUuid("01234567-89ab-4000-8123-456789abcdef"))); + EXPECT_TRUE(isUuidV4(parseUuid("01234567-89ab-4000-bfff-456789abcdef"))); + EXPECT_FALSE(isUuidV4(parseUuid("01234567-89ab-4000-0123-456789abcdef"))); + EXPECT_FALSE(isUuidV4(parseUuid("01234567-89ab-4000-c123-456789abcdef"))); +} + +TEST(TestUuid, IsUuidV4RejectsNilAndMaxUuids) +{ + EXPECT_FALSE(isUuidV4(ALL_ZERO_BYTES)); + EXPECT_FALSE(isUuidV4(ALL_ONES_BYTES)); +} + +TEST(TestUuid, ToFlatbufferUuidRoundTripsThroughEqualityOperator) +{ + const auto bytes = parseUuid("01234567-89ab-4def-8123-456789abcdef"); + const auto lhs = toFlatbufferUuid(bytes); + const auto rhs = toFlatbufferUuid(bytes); + EXPECT_EQ(lhs, rhs); + EXPECT_EQ(toUuidBytes(lhs), bytes); +} + +TEST(TestUuid, ToFlatbufferUuidPreservesByteOrder) +{ + UuidBytes bytes{}; + for(size_t i = 0; i < bytes.size(); ++i) + { + bytes[i] = static_cast(i); + } + EXPECT_EQ(toUuidBytes(toFlatbufferUuid(bytes)), bytes); +} + +TEST(TestUuid, DistinctUuidsCompareUnequal) +{ + const auto a = toFlatbufferUuid(parseUuid("01234567-89ab-4def-8123-456789abcdef")); + const auto b = toFlatbufferUuid(parseUuid("00000000-0000-0000-0000-000000000000")); + EXPECT_NE(a, b); +} diff --git a/projects/hipdnn/flatbuffers_sdk/tools/umd_registry_gen/umd_registry_gen.cpp b/projects/hipdnn/flatbuffers_sdk/tools/umd_registry_gen/umd_registry_gen.cpp new file mode 100644 index 000000000000..0f78b4e9d847 --- /dev/null +++ b/projects/hipdnn/flatbuffers_sdk/tools/umd_registry_gen/umd_registry_gen.cpp @@ -0,0 +1,483 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +// +// umd_registry_gen -- build-time generator for the RFC 0018 op-schema registry. +// +// Reads the FlatBuffers binary reflection schema (graph.bfbs), enumerates the +// NodeAttributes union members (opcode -> attribute table), classifies each +// field per RFC 0018 Appendix B.3, and emits a header-only, strongly-typed +// registry (op_schema_registry_generated.hpp) whose readers call the generated +// FlatBuffers accessors directly -- no runtime reflection (RFC 0018 B.4). +// +// Field classification (B.3): +// * umd_input_tensor + umd_name -> input-tensor edge (field MUST be a `long` UID) +// * umd_output_tensor + umd_name -> output-tensor edge (field MUST be a `long` UID) +// * neither flag, scalar -> scalar attribute, bind-named by field name +// * neither flag, non-scalar-> skipped (a vector/table/union is not a UMD +// scalar; the RFC's attribute tables are UID + +// scalar, so this only affects ops outside the +// UMD scalar namespace) +// Optionality is derived from the `= null` default (reflection `optional()`). +// +// Build errors (fail closed, non-zero exit): both flags on one field; umd_name +// without a flag; a flag on a non-integer field; a duplicate umd_name within an +// op; a binding name colliding with a reserved root (graph/kernel/device). + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flatbuffers/reflection.h" +#include "flatbuffers/reflection_generated.h" + +namespace +{ + +[[noreturn]] void fail(const std::string& msg) +{ + std::fprintf(stderr, "umd_registry_gen: error: %s\n", msg.c_str()); + std::exit(1); +} + +// Reports a per-field schema error as `table.field: msg` and exits. Builds the +// message by appending so these cold error paths avoid chained-`+` temporaries. +[[noreturn]] void + failField(const std::string& table, const std::string& field, std::string_view msg) +{ + std::string s = table; + s += '.'; + s += field; + s += ": "; + s += msg; + fail(s); +} + +std::string readFile(const std::string& path) +{ + std::ifstream f(path, std::ios::binary); + if(!f) + { + fail("cannot open input schema: " + path); + } + return {std::istreambuf_iterator(f), std::istreambuf_iterator()}; +} + +// Short (unqualified) name: text after the last '.'. +std::string shortName(const std::string& qualified) +{ + auto pos = qualified.rfind('.'); + return pos == std::string::npos ? qualified : qualified.substr(pos + 1); +} + +const reflection::KeyValue* attr(const reflection::Field* f, const char* key) +{ + return (f->attributes() != nullptr) ? f->attributes()->LookupByKey(key) : nullptr; +} + +// Table-level attribute value (e.g. `umd_opcode`), or empty when absent. +std::string objAttr(const reflection::Object* o, const char* key) +{ + if(o->attributes() == nullptr) + { + return {}; + } + const auto* kv = o->attributes()->LookupByKey(key); + if(kv == nullptr) + { + return {}; + } + return (kv->value() != nullptr) ? kv->value()->str() : std::string(); +} + +bool isIntegerBase(reflection::BaseType t) +{ + return t == reflection::Byte || t == reflection::UByte || t == reflection::Short + || t == reflection::UShort || t == reflection::Int || t == reflection::UInt + || t == reflection::Long || t == reflection::ULong; +} + +bool isScalarBase(reflection::BaseType t) +{ + return t == reflection::Bool || t == reflection::Float || t == reflection::Double + || isIntegerBase(t); +} + +// C++ keywords flatc suffixes with '_' when emitting an accessor name. +const std::set& cppKeywords() +{ + static const std::set s_keywords + = {"alignas", "alignof", "and", "asm", "auto", "bool", "break", + "case", "catch", "char", "class", "const", "continue", "default", + "delete", "do", "double", "else", "enum", "explicit", "export", + "extern", "false", "float", "for", "friend", "goto", "if", + "inline", "int", "long", "mutable", "namespace", "new", "operator", + "private", "protected", "public", "register", "return", "short", "signed", + "sizeof", "static", "struct", "switch", "template", "this", "throw", + "true", "try", "typedef", "typename", "union", "unsigned", "using", + "virtual", "void", "volatile", "while", "and_eq", "bitand", "bitor", + "compl", "not", "not_eq", "or", "or_eq", "xor", "xor_eq"}; + return s_keywords; +} + +std::string accessorName(const std::string& fieldName) +{ + return (cppKeywords().count(fieldName) != 0u) ? fieldName + "_" : fieldName; +} + +bool isReservedRoot(const std::string& n) +{ + return n == "graph" || n == "kernel" || n == "device"; +} + +constexpr const char* NS = "::hipdnn_flatbuffers_sdk::data_objects"; + +struct Emitted +{ + std::string inputTensors; // InputTensorBinding initializers + std::string outputTensors; // OutputTensorBinding initializers + std::string attributes; // AttrBinding initializers + std::size_t inputTensorCount = 0; + std::size_t outputTensorCount = 0; + std::size_t attributeCount = 0; +}; + +// Emit the reader lambda + binding line for one input/output tensor UID field. +std::string emitUidBinding(const std::string& table, + const std::string& name, + const std::string& acc, + bool optional) +{ + std::ostringstream o; + o << " {\"" << name << "\", " << (optional ? "true" : "false") << ", " + << "+[](const void* a, std::int64_t& out) -> bool { "; + if(optional) + { + o << "auto v = static_cast(a)->" << acc + << "(); if(!v) return false; out = *v; return true;"; + } + else + { + o << "out = static_cast(a)->" << acc + << "(); return true;"; + } + o << " }},\n"; + return o.str(); +} + +// Emit the reader lambda + binding line for one scalar attribute field. +std::string emitAttrBinding(const reflection::Schema* schema, + const std::string& table, + const reflection::Field* fld) +{ + const std::string name = fld->name()->str(); + const std::string acc = accessorName(name); + const auto base = fld->type()->base_type(); + const bool optional = fld->optional(); + const int enumIdx = fld->type()->index(); + const bool isEnum = enumIdx >= 0 && isIntegerBase(base); + + std::string attrType; + std::string body; + if(isEnum) + { + const std::string enumShort = shortName(schema->enums()->Get(enumIdx)->name()->str()); + attrType = "AttrType::DTYPE"; + std::ostringstream b; + b << "ScalarValue s; s.type = AttrType::DTYPE; "; + if(optional) + { + b << "auto v = static_cast(a)->" << acc + << "(); s.present = v.has_value(); if(v) s.dtype = " << NS << "::EnumName" + << enumShort << "(*v);"; + } + else + { + b << "s.present = true; s.dtype = " << NS << "::EnumName" << enumShort + << "(static_cast(a)->" << acc << "());"; + } + b << " return s;"; + body = b.str(); + } + else + { + std::string member; + if(base == reflection::Bool) + { + attrType = "AttrType::BOOL"; + member = "b"; + } + else if(base == reflection::Float || base == reflection::Double) + { + attrType = "AttrType::FLOAT"; + member = "f"; + } + else + { + attrType = "AttrType::INT"; + member = "i"; + } + std::string cast; + if(member == "i") + { + cast = "std::int64_t"; + } + else if(member == "f") + { + cast = "double"; + } + const std::string readExpr + = "static_cast(a)->" + acc + "()"; + std::ostringstream b; + b << "ScalarValue s; s.type = " << attrType << "; "; + if(optional) + { + b << "auto v = " << readExpr << "; s.present = v.has_value(); if(v) s." << member + << " = " << (cast.empty() ? "*v" : "static_cast<" + cast + ">(*v)") << ";"; + } + else + { + b << "s.present = true; s." << member << " = " + << (cast.empty() ? readExpr : "static_cast<" + cast + ">(" + readExpr + ")") << ";"; + } + b << " return s;"; + body = b.str(); + } + + std::ostringstream o; + o << " {\"" << name << "\", " << (optional ? "true" : "false") << ", " << attrType + << ", +[](const void* a) -> ScalarValue { " << body << " }},\n"; + return o.str(); +} + +Emitted classifyTable(const reflection::Schema* schema, const reflection::Object* obj) +{ + const std::string table = shortName(obj->name()->str()); + Emitted out; + std::set seenNames; + + for(const auto* fld : *obj->fields()) + { + const std::string name = fld->name()->str(); + const auto* inputTensorFlag = attr(fld, "umd_input_tensor"); + const auto* outputTensorFlag = attr(fld, "umd_output_tensor"); + const auto* nameAttr = attr(fld, "umd_name"); + const auto base = fld->type()->base_type(); + + if((inputTensorFlag != nullptr) && (outputTensorFlag != nullptr)) + { + failField(table, name, "has both umd_input_tensor and umd_output_tensor"); + } + + const bool isEdge = (inputTensorFlag != nullptr) || (outputTensorFlag != nullptr); + if(isEdge) + { + if(nameAttr == nullptr) + { + failField(table, name, "umd_input_tensor/umd_output_tensor requires umd_name"); + } + const std::string bindName + = (nameAttr->value() != nullptr) ? nameAttr->value()->str() : ""; + if(bindName.empty()) + { + failField(table, name, "umd_name must be non-empty"); + } + if(base != reflection::Long) + { + failField( + table, name, "umd_input_tensor/umd_output_tensor field must be `long` (a UID)"); + } + if(isReservedRoot(bindName)) + { + std::string msg = "umd_name '"; + msg += bindName; + msg += "' collides with a reserved root (graph/kernel/device)"; + failField(table, name, msg); + } + if(!seenNames.insert(bindName).second) + { + std::string msg = table; + msg += ": duplicate umd_name '"; + msg += bindName; + msg += '\''; + fail(msg); + } + + const std::string acc = accessorName(name); + const bool optl = fld->optional(); + if(inputTensorFlag != nullptr) + { + out.inputTensors += emitUidBinding(table, bindName, acc, optl); + ++out.inputTensorCount; + } + else + { + out.outputTensors += emitUidBinding(table, bindName, acc, optl); + ++out.outputTensorCount; + } + continue; + } + + if(nameAttr != nullptr) + { + failField(table, name, "umd_name without umd_input_tensor/umd_output_tensor"); + } + + // Unannotated: a scalar attribute (if a scalar base type). Non-scalar + // fields (vector/table/union/string) are not UMD scalars -- skip. + if(isScalarBase(base)) + { + out.attributes += emitAttrBinding(schema, table, fld); + ++out.attributeCount; + } + } + return out; +} + +} // namespace + +int main(int argc, char** argv) +{ + if(argc != 3) + { + fail("usage: umd_registry_gen "); + } + const std::string bfbsPath = argv[1]; + const std::string outPath = argv[2]; + + const std::string buf = readFile(bfbsPath); + const auto* schema = reflection::GetSchema(buf.data()); + if(schema == nullptr) + { + fail("input is not a valid reflection schema: " + bfbsPath); + } + + const reflection::Enum* nodeAttrs = nullptr; + for(const auto* e : *schema->enums()) + { + if(e->is_union() && shortName(e->name()->str()) == "NodeAttributes") + { + nodeAttrs = e; + } + } + if(nodeAttrs == nullptr) + { + fail("NodeAttributes union not found in schema"); + } + + std::ostringstream body; + std::ostringstream entries; + std::size_t opCount = 0; + std::set seenOpcodes; + + for(const auto* ev : *nodeAttrs->values()) + { + if(ev->union_type() == nullptr) + { + continue; + } + const int objIdx = ev->union_type()->index(); + if(objIdx < 0) + { + continue; // NONE + } + const reflection::Object* obj = schema->objects()->Get(objIdx); + const std::string table = shortName(obj->name()->str()); + const std::string member = ev->name()->str(); // union value name == table type name + + // Opcode key: the table's `umd_opcode` shorthand (RFC 0018), else the + // table type name. Must be unique across ops. + std::string opcode = objAttr(obj, "umd_opcode"); + if(opcode.empty()) + { + opcode = member; + } + if(!seenOpcodes.insert(opcode).second) + { + fail("duplicate umd_opcode '" + opcode + "'"); + } + + const Emitted em = classifyTable(schema, obj); + + const std::string tag = "op" + std::to_string(opCount); + std::string inputTensorsRef = "nullptr"; + std::string outputTensorsRef = "nullptr"; + std::string attrsRef = "nullptr"; + const bool anyTable = (em.inputTensorCount != 0u) || (em.outputTensorCount != 0u) + || (em.attributeCount != 0u); + if(anyTable) + { + body << "// " << table << "\nnamespace " << tag << " {\n"; + } + if(em.inputTensorCount != 0u) + { + body << "inline const InputTensorBinding inputTensors[] = {\n" + << em.inputTensors << "};\n"; + inputTensorsRef = tag + "::inputTensors"; + } + if(em.outputTensorCount != 0u) + { + body << "inline const OutputTensorBinding outputTensors[] = {\n" + << em.outputTensors << "};\n"; + outputTensorsRef = tag + "::outputTensors"; + } + if(em.attributeCount != 0u) + { + body << "inline const AttrBinding attributes[] = {\n" << em.attributes << "};\n"; + attrsRef = tag + "::attributes"; + } + if(anyTable) + { + body << "} // namespace " << tag << "\n\n"; + } + + entries << " {\"" << opcode << "\", \"" << member << "\", " << NS + << "::NodeAttributes::" << member << ", " << inputTensorsRef << ", " + << em.inputTensorCount << "u, " << outputTensorsRef << ", " << em.outputTensorCount + << "u, " << attrsRef << ", " << em.attributeCount << "u},\n"; + ++opCount; + } + + std::ostringstream hdr; + hdr << "// AUTOMATICALLY GENERATED by umd_registry_gen -- do not modify.\n" + << "// Source: FlatBuffers reflection schema (graph.bfbs). See RFC 0018 Appendix B.\n" + << "#pragma once\n\n" + << "#include \n#include \n#include \n\n" + << "#include \n" + << "#include \n\n" + << "namespace hipdnn_flatbuffers_sdk::umd\n{\nnamespace generated\n{\n\n" + << body.str() << "inline const OpSchemaEntry entries[] = {\n" + << entries.str() << "};\n\n} // namespace generated\n\n" + << "inline const OpSchemaEntry* opSchemaEntries(std::size_t& count)\n{\n" + << " count = sizeof(generated::entries) / sizeof(generated::entries[0]);\n" + << " return generated::entries;\n}\n\n" + << "inline const OpSchemaEntry* lookupOpByName(std::string_view opcode)\n{\n" + << " for(const auto& e : generated::entries)\n if(e.opcode == opcode)\n" + << " return &e;\n return nullptr;\n}\n\n" + << "inline const OpSchemaEntry* " + "lookupOpByType(::hipdnn_flatbuffers_sdk::data_objects::NodeAttributes " + "attributesType)\n{\n" + << " for(const auto& e : generated::entries)\n if(e.attributesType == " + "attributesType)\n" + << " return &e;\n return nullptr;\n}\n\n" + << "} // namespace hipdnn_flatbuffers_sdk::umd\n"; + + std::ofstream ofs(outPath, std::ios::binary); + if(!ofs) + { + fail("cannot open output for writing: " + outPath); + } + ofs << hdr.str(); + if(!ofs) + { + fail("failed writing output: " + outPath); + } + + std::fprintf( + stderr, "umd_registry_gen: emitted %zu op entries to %s\n", opCount, outPath.c_str()); + return 0; +} diff --git a/projects/hipdnn/plugin_sdk/CMakeLists.txt b/projects/hipdnn/plugin_sdk/CMakeLists.txt index aaae9b578912..9592433ccf60 100644 --- a/projects/hipdnn/plugin_sdk/CMakeLists.txt +++ b/projects/hipdnn/plugin_sdk/CMakeLists.txt @@ -43,6 +43,16 @@ export( install(TARGETS hipdnn_plugin_sdk EXPORT hipdnn_plugin_sdk_targets COMPONENT Development) # cmake-lint: disable=E1122 +# +# ingestor/ headers are WIP (RFC 0017); this glob is unconditional, so exclude them +# explicitly unless the feature is enabled. +set(HIP_DNN_PLUGIN_SDK_INSTALL_EXCLUDE_PATTERNS "") +if(NOT HIPDNN_ENABLE_KERNEL_INGESTOR) + # Matches the directory itself (not "ingestor/*"): under FILES_MATCHING, excluding + # only the contents still creates the empty directory in the install tree. + list(APPEND HIP_DNN_PLUGIN_SDK_INSTALL_EXCLUDE_PATTERNS PATTERN "ingestor" EXCLUDE) +endif() + install( DIRECTORY ${HIP_DNN_PLUGIN_SDK_INCLUDE_DIR}/ DESTINATION ${HIP_DNN_PLUGIN_SDK_INSTALL_INCLUDE_DIR} @@ -52,6 +62,7 @@ install( PATTERN "*.hpp" PATTERN "*.h" PATTERN "*.h.in" EXCLUDE + ${HIP_DNN_PLUGIN_SDK_INSTALL_EXCLUDE_PATTERNS} ) # Install generated version header diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/Catalog.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/Catalog.hpp new file mode 100644 index 000000000000..2b0b2a3d9811 --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/Catalog.hpp @@ -0,0 +1,37 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include + +#include +#include + +namespace hipdnn_plugin_sdk::ingestor +{ + +/** + * @brief An engine's kernels that fit one graph on one device, plus their sort state. + * + * Built once per (graph, device) by running the matchers, then cached. An engine is + * applicable exactly when its catalog is non-empty. + * + * `isSorted` defers ranking until something asks for an order; once sorted, `entries` + * stays in heuristic-ranked order for the life of the cache entry. + */ +struct Catalog +{ + std::vector entries; + bool isSorted = false; + /// What this engine's graph-scoped matchers resolved, merged across surviving packs + /// (RFC 0017 §8.1); a pruned pack's bindings are discarded unmerged. One token name + /// binding to different values across packs is an authoring error, rejected at merge. + BoundTokens bound; +}; + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/Descriptors.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/Descriptors.hpp new file mode 100644 index 000000000000..3d6d642db912 --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/Descriptors.hpp @@ -0,0 +1,243 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include +#include + +#include + +/** + * @file Descriptors.hpp + * @brief The universal descriptor set from RFC 0017, as parsed in-memory data. + * + * Each descriptor kind maps 1:1 onto a concept hipDNN already has: + * + * | Descriptor | Describes | + * |------------|--------------------------------------------------------------| + * | KMD | the variant fields every kernel in an engine carries | + * | UHD | how to rank the kernels that fit a graph | + * | UED | the engine: its identity, its one UHD, its one KMD, its knobs | + * | UMD | one applicability check over the graph and kernel metadata | + * | UDD | how to invoke a kernel: workspace and launch | + * | UKD | one launchable kernel: a source plus its KMD values | + * | KDP | binds a matcher set, one engine, and one UDD over N kernels | + * + * Descriptors reference each other by `id` only, never by pointer or index, so a pack + * authored independently can name a matcher or an engine it does not own. + * + * These are the *parsed* form; nothing here parses, loads, or validates a file. The + * loader that produces them from disk is ALMIOPEN-2401; the expression language that + * replaces the native symbol fields below is the UMD/UDD follow-up RFC. + */ +namespace hipdnn_plugin_sdk::ingestor +{ + +/// Stable, globally unique id every descriptor carries; descriptors reference each +/// other only by this id. A 128-bit UUID, matching hipDNN's graph identity (RFC 0017 §4). +using DescriptorId = hipdnn_flatbuffers_sdk::utilities::UuidBytes; + +/// @brief A descriptor id in its canonical text form, for diagnostics. +inline std::string toString(const DescriptorId& id) +{ + return hipdnn_flatbuffers_sdk::utilities::formatUuid(id); +} + +/// Hash for keying maps on a descriptor id (std::array has no std::hash). +struct DescriptorIdHash +{ + size_t operator()(const DescriptorId& id) const noexcept + { + // FNV-1a fold over the UUID bytes; not persisted or exposed outside this process. + size_t hash = 1469598103934665603ULL; + for(const uint8_t byte : id) + { + hash ^= static_cast(byte); + hash *= 1099511628211ULL; + } + return hash; + } +}; + +/// Value type for a KMD field or a bound `$graph.*` fact (MatchContext.hpp's +/// BoundTokens). `bool` is distinct from `int64_t` so a flag doesn't compare as 1; +/// `std::vector` covers list-valued facts like `stride_order`. +using MetadataValue = std::variant>; + +/// The type a KMD field or bound graph fact holds. Ordered to match MetadataValue's +/// alternatives. +enum class MetadataType +{ + BOOL, + INT, + FLOAT, + STRING, + INT_LIST, +}; + +/// @brief The type of a value, for checking one against its field's declaration. +inline MetadataType metadataTypeOf(const MetadataValue& value) +{ + return static_cast(value.index()); +} + +/// A kernel's complete metadata tuple: every KMD field mapped to this kernel's value. +/// Must be unique per kernel within an engine (RFC 0017 §4); ordered so equal tuples +/// compare equal regardless of insertion order. +using MetadataValues = std::map; + +/// One field a kernel may vary along, as declared by an engine's KMD. +struct MetadataField +{ + std::string name; + MetadataType type = MetadataType::INT; + /// Value used when a kernel omits this field; nullopt means the field is mandatory. + std::optional defaultValue; +}; + +/// KMD: the metadata schema, one per engine, shared by every kernel it owns. The +/// field set is the engine's kernel key; matchers read fields as `$kernel.`. +struct MetadataSchema +{ + DescriptorId id; + std::string name; + std::vector fields; +}; + +/// Which adapter builds an engine's IKernelHeuristic from a UHD's `payload` (RFC 0017 +/// §9.1). +enum class HeuristicKind +{ + /// `payload` is a NativeRegistry score symbol (IKernelHeuristic.hpp's + /// makeKernelHeuristic()). Only kind with an adapter today. + NATIVE, + /// Trained model artifact plus feature signature (UHD follow-up RFC). No adapter yet. + MODEL, +}; + +/// UHD: the kernel-selection model for one engine — chooses which kernel within an +/// engine to run, given the kernels that fit the graph. +struct HeuristicDescriptor +{ + DescriptorId id; + std::string name; + HeuristicKind kind = HeuristicKind::NATIVE; + /// `kind`'s payload: a NativeRegistry score symbol when kind == NATIVE. + std::string payload; +}; + +/// UED: the engine itself, carrying no logic of its own. `name` hashes into hipDNN's +/// engine-id space (EngineNames.hpp); must be globally unique, e.g. "rocke:SDPA". +struct EngineDescriptor +{ + DescriptorId id; + std::string name; + DescriptorId heuristicId; + DescriptorId metadataSchemaId; + /// KMD field names this engine exposes to the caller. Unmatched name is a load error. + std::vector knobs; + /// `hipdnnBackendBehaviorNote_t` values (BehaviorNote.h), reported via EngineDetails. + /// Stored as int32 so an unrecognized newer note value isn't truncated. + std::vector behaviorNotes; +}; + +/// Which inputs a matcher reads; determines how often it runs and what its failure +/// prunes (RFC 0017 §5). Provisional until the criteria language lands, when this +/// becomes derived from whether the expression references `$kernel.*`. +enum class MatchScope +{ + /// Runs once per (graph, device); failure disqualifies every kernel in every pack + /// that lists it. + GRAPH, + /// Also reads `$kernel.*`; re-evaluated per candidate kernel, disqualifying only + /// that kernel. + KERNEL, +}; + +/// UMD: one applicability check, shared by id across packs. +struct MatchDescriptor +{ + DescriptorId id; + std::string name; + MatchScope scope; + /// Resolved through NativeRegistry; a data-driven form (structural node pattern plus + /// declarative criteria) is the UMD follow-up RFC. + std::string matchSymbol; +}; + +/// UDD: how to invoke a kernel — the dispatch ABI, shared by every kernel in a pack. +/// A kernel with a different argument list or launch-formula shape belongs in +/// another pack with its own UDD (RFC 0017 §6). +struct DispatchDescriptor +{ + DescriptorId id; + std::string name; + /// Resolved through NativeRegistry to a handler supplying workspace and launch. A + /// data-driven form (symbolic grid/block/shared-memory/workspace/args) is the UDD + /// follow-up RFC. + std::string dispatchSymbol; +}; + +/// Which adapter loads and prepares one kernel's code, given its source's payload (RFC +/// 0017 §9.1's adapter dispatch point). +enum class KernelSourceKind +{ + /// A named source file plus an entry point, compiled at plan-build time. The only + /// kind this POC implements. + EMBEDDED_SOURCE, + /// A prebuilt kpack library plus the symbol to resolve inside it. No adapter yet. + KPACK_SYMBOL, + /// A standalone `.hsaco` code-object file. No adapter yet. + HSACO_FILE, + /// A rocke builder name plus its build values. No adapter yet. + ROCKE_BUILDER, +}; + +/// UKD's source: where a kernel's code comes from, a tagged union over RFC 0017 §7's +/// source kinds. Only `EMBEDDED_SOURCE` is implemented. +struct KernelSource +{ + KernelSourceKind kind = KernelSourceKind::EMBEDDED_SOURCE; + /// `EMBEDDED_SOURCE`: source file name. Unused by other kinds. + std::string sourceFile; + /// `EMBEDDED_SOURCE`: entry point within `sourceFile`. Unused by other kinds. + std::string entryPoint; +}; + +/// UKD: one launchable kernel — a source plus concrete values for its engine KMD's +/// fields. Matchers, engine, and dispatch are inherited from its pack. +struct KernelDescriptor +{ + DescriptorId id; + std::string name; + KernelSource source; + /// This kernel's values for the KMD's fields; omitted fields take the KMD default. + /// The completed tuple is this kernel's catalog key, unique engine-wide. + MetadataValues metadata; + /// Tie-break when the heuristic is not decisive. Higher wins. + int64_t priority = 0; +}; + +/// KDP: one pack binding a matcher set, one engine, and one dispatch descriptor over a +/// vector of child kernels. Everything except the child kernels is shared by id. +struct KernelDescriptorPack +{ + DescriptorId id; + std::string name; + /// A child kernel applies only when every one of these matchers passes. + std::vector matcherIds; + DescriptorId engineId; + DescriptorId dispatchId; + std::vector kernels; +}; + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/GenericEngine.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/GenericEngine.hpp new file mode 100644 index 000000000000..d7f37eeb525e --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/GenericEngine.hpp @@ -0,0 +1,163 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hipdnn_plugin_sdk::ingestor +{ + +/** + * @brief One hipDNN engine, defined entirely by a UED and the packs naming it. + * + * Satisfies hipDNN's IEngine contract using descriptor data. + * + * The engine's id is its UED name hashed into hipDNN's engine-id space, registered at + * construction, so an id collision surfaces as a failure to create the engine. + * + * Holds exactly one plan builder: a catalog entry is a candidate, not a builder. + */ +template +class GenericEngine : public IEngine +{ +public: + using IGraph = hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph; + using IEngineConfig = hipdnn_flatbuffers_sdk::flatbuffer_utilities::IEngineConfig; + + /** + * @param engine The UED this engine is; its name is this engine's identity + * and its knob list is what it exposes. + * @param stateManager The descriptor state this engine selects over, already + * validated. + * @param deviceResolver Answers which device each call is for. Held by reference; + * owned by the provider, which must keep it alive for the engine's lifetime. + * + * @throws std::invalid_argument if a knob names no field in the engine's metadata + * schema (RFC 0017 §4). + */ + GenericEngine(EngineDescriptor engine, + std::unique_ptr> stateManager, + const IDeviceResolver& deviceResolver) + : _engine(std::move(engine)) + , _stateManager(std::move(stateManager)) + , _id(hipdnn_data_sdk::utilities::engineNameToId(_engine.name)) + , _planBuilder(_engine, *_stateManager, deviceResolver) + { + const auto& fields = _stateManager->metadataSchema().fields; + for(const auto& knob : _engine.knobs) + { + const auto declared + = std::any_of(fields.begin(), fields.end(), [&knob](const MetadataField& field) { + return field.name == knob; + }); + if(!declared) + { + throw std::invalid_argument("engine '" + _engine.name + "' exposes knob '" + knob + + "', which its metadata schema does not declare"); + } + } + } + + /// Not relocatable: _planBuilder holds references into _engine and *_stateManager. + GenericEngine(const GenericEngine&) = delete; + GenericEngine& operator=(const GenericEngine&) = delete; + GenericEngine(GenericEngine&&) = delete; + GenericEngine& operator=(GenericEngine&&) = delete; + + /// The descriptor this engine was built from, for diagnostics. + const EngineDescriptor& descriptor() const + { + return _engine; + } + + int64_t id() const override + { + return _id; + } + + bool isApplicable(THandle& handle, const IGraph& opGraph) const override + { + return _planBuilder.isApplicable(handle, opGraph); + } + + /** + * @brief Reports this engine's knobs for @p opGraph. + * + * The returned buffer is detached and handed to the caller's handle to own. + */ + void getDetails(THandle& handle, + const IGraph& opGraph, + hipdnnPluginConstData_t& detailsOut) const override + { + flatbuffers::FlatBufferBuilder builder; + + std::vector> knobOffsets; + for(const auto& knob : _planBuilder.getCustomKnobs(handle, opGraph)) + { + knobOffsets.push_back(hipdnn_flatbuffers_sdk::data_objects::Knob::Pack(builder, &knob)); + } + + auto knobs = builder.CreateVector(knobOffsets); + // The UED's notes reach the caller the same way every other engine's do. + auto behaviorNotes = builder.CreateVector(_engine.behaviorNotes); + auto engineDetails = hipdnn_flatbuffers_sdk::data_objects::CreateEngineDetails( + builder, _id, knobs, behaviorNotes); + builder.Finish(engineDetails); + + auto detachedBuffer = std::make_unique(builder.Release()); + detailsOut.ptr = detachedBuffer->data(); + detailsOut.size = detachedBuffer->size(); + handle.storeEngineDetailsDetachedBuffer(detailsOut.ptr, std::move(detachedBuffer)); + } + + size_t getMaxWorkspaceSize(const THandle& handle, + const IGraph& opGraph, + const IEngineConfig& engineConfig) const override + { + TSettings executionSettings; + _planBuilder.initializeExecutionSettings(handle, opGraph, engineConfig, executionSettings); + return _planBuilder.getMaxWorkspaceSize(handle, opGraph, executionSettings); + } + + void initializeExecutionContext(const THandle& handle, + const IGraph& opGraph, + const IEngineConfig& engineConfig, + TContext& executionContext) const override + { + TSettings executionSettings; + _planBuilder.initializeExecutionSettings(handle, opGraph, engineConfig, executionSettings); + executionContext.setExecutionSettings(executionSettings); + _planBuilder.buildPlan(handle, opGraph, engineConfig, executionContext); + } + +private: + EngineDescriptor _engine; + /// Owned outright: held by pointer so the plan builder can bind a reference to it + /// in the member initializer list, which needs a stable address. + std::unique_ptr> _stateManager; + int64_t _id; + GenericPlanBuilder _planBuilder; +}; + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/GenericPlan.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/GenericPlan.hpp new file mode 100644 index 000000000000..1bf376023885 --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/GenericPlan.hpp @@ -0,0 +1,94 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace hipdnn_plugin_sdk::ingestor +{ + +/** + * @brief An executable plan for one descriptor-selected kernel. + * + * Holds the selected kernel, its dispatch handler, and launch details resolved from + * the graph at construction; execute() re-matches nothing. + * + * Immutable after construction: the same plan may execute concurrently from multiple + * threads with different device buffers. + */ +template +class GenericPlan : public IPlan +{ +public: + /** + * @param dispatcher The selected kernel and its resolved dispatch handler. + * @param context Bound graph and device state, read during construction only; + * not retained. + * @param bound What matching resolved about the graph. + */ + GenericPlan(KernelDispatcher dispatcher, + const MatchContext& context, + const BoundTokens& bound) + : _dispatcher(std::move(dispatcher)) + , _workspaceBytes(_dispatcher.handler->workspaceBytes(context, bound, _dispatcher.kernel)) + , _prepared(_dispatcher.handler->prepare(context, bound, _dispatcher.kernel)) + { + if(_prepared == nullptr) + { + throw HipdnnPluginException(HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR, + "dispatch handler for kernel '" + + toString(_dispatcher.kernel.kernelId) + + "' prepared no launch"); + } + } + + // NOLINTNEXTLINE(portability-template-virtual-member-function) + size_t getWorkspaceSize(const THandle& /*handle*/) const override + { + return _workspaceBytes; + } + + // NOLINTNEXTLINE(portability-template-virtual-member-function) + void execute(const THandle& handle, + const hipdnnPluginDeviceBuffer_t* deviceBuffers, + uint32_t numDeviceBuffers, + void* workspace = nullptr) const override + { + if(_workspaceBytes > 0 && workspace == nullptr) + { + throw HipdnnPluginException(HIPDNN_PLUGIN_STATUS_INVALID_VALUE, + "kernel '" + toString(_dispatcher.kernel.kernelId) + + "' requires " + std::to_string(_workspaceBytes) + + " workspace bytes but none was provided"); + } + + _dispatcher.handler->launch(handle, *_prepared, deviceBuffers, numDeviceBuffers, workspace); + } + + /// The kernel this plan launches (used by resolved-plan diagnostics, RFC 0017 §10). + const KernelDefinition& kernel() const + { + return _dispatcher.kernel; + } + +private: + KernelDispatcher _dispatcher; + size_t _workspaceBytes; + std::unique_ptr _prepared; +}; + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/GenericPlanBuilder.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/GenericPlanBuilder.hpp new file mode 100644 index 000000000000..d72963e6d36a --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/GenericPlanBuilder.hpp @@ -0,0 +1,340 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace hipdnn_plugin_sdk::ingestor +{ + +/// A caller's requested value for each knob it explicitly set, keyed by knob (= KMD +/// field) name. Only integer-valued fields are expressible as knobs today. +/// +/// A `TSettings` used with GenericPlanBuilder must carry one of these named +/// `ingestorKnobFilter`, populated by initializeExecutionSettings() and read by +/// getMaxWorkspaceSize() and buildPlan(). +using KnobFilter = std::map; + +/** + * @brief The one plan builder a descriptor-backed engine has. + * + * One builder regardless of catalog size: a catalog entry is a candidate, and this + * builds a plan for whichever candidate selection chose. + * + * @tparam TSettings Must carry a `KnobFilter ingestorKnobFilter` member, populated by + * initializeExecutionSettings() and read by getMaxWorkspaceSize() and + * buildPlan(). + */ +template +class GenericPlanBuilder : public IPlanBuilder +{ +public: + using IGraph = hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph; + using IEngineConfig = hipdnn_flatbuffers_sdk::flatbuffer_utilities::IEngineConfig; + + /// @param engine The UED whose knobs this builder reports. Held by reference; + /// owned by the engine, which outlives its builder. + /// @param stateManager The descriptor state to select over. Shared, not owned. + /// @param deviceResolver Answers which device each call is for. Held by reference; + /// owned by the engine, which outlives its builder. + GenericPlanBuilder(const EngineDescriptor& engine, + const KernelIngestorStateManager& stateManager, + const IDeviceResolver& deviceResolver) + : _engine(engine) + , _stateManager(stateManager) + , _deviceResolver(deviceResolver) + { + } + + /// Applicable exactly when some kernel survived matching; does not rank. + bool isApplicable(const THandle& handle, const IGraph& opGraph) const override + { + return !_stateManager.unsortedDefinitions(contextFor(handle, opGraph)).empty(); + } + + /** + * @brief The largest scratch any surviving kernel needs. + * + * The buffer is reused rather than partitioned: kernels launch one at a time on + * one stream, so a candidate's scratch is live only while it runs. + * + * @throws HipdnnPluginException(HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR) if the catalog + * is empty (see throwNoApplicableKernel()). + * @throws HipdnnPluginException(HIPDNN_PLUGIN_STATUS_INVALID_VALUE) if knob + * filtering empties a non-empty catalog (see throwUnsatisfiableKnobFilter()). + */ + size_t getMaxWorkspaceSize(const THandle& handle, + const IGraph& opGraph, + const TSettings& executionSettings) const override + { + const auto context = contextFor(handle, opGraph); + // One lookup for the entries and the bound state together. + const auto catalog = _stateManager.unsortedCatalog(context); + if(catalog.entries.empty()) + { + throwNoApplicableKernel(); + } + + // Knob settings request a catalog subset, same contract as buildPlan(). + const auto filtered + = applyKnobFilter(catalog.entries, executionSettings.ingestorKnobFilter); + if(filtered.empty()) + { + // Same failure buildPlan() reports. + throwUnsatisfiableKnobFilter(executionSettings.ingestorKnobFilter, + catalog.entries.size()); + } + + size_t maxBytes = 0; + // Unsorted deliberately: order-independent, so no ranking heuristic needed. + for(const auto& kernel : filtered) + { + const auto dispatcher = _stateManager.getDispatchDetails(kernel); + maxBytes = std::max(maxBytes, + dispatcher.handler->workspaceBytes(context, catalog.bound, kernel)); + } + return maxBytes; + } + + /** + * @brief Reads the caller's knob settings from @p engineConfig into TSettings. + * + * Populates `executionSettings.ingestorKnobFilter`, read the same way by + * getMaxWorkspaceSize() and buildPlan(). + */ + void initializeExecutionSettings(const THandle& /*handle*/, + const IGraph& /*opGraph*/, + const IEngineConfig& engineConfig, + TSettings& executionSettings) const override + { + executionSettings.ingestorKnobFilter = readKnobFilter(engineConfig); + } + + /** + * @brief Builds a plan for the top-ranked kernel satisfying every knob the caller + * set. + * + * @throws HipdnnPluginException(HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR) if the catalog + * is empty before knob filtering (see throwNoApplicableKernel()). + * @throws HipdnnPluginException(HIPDNN_PLUGIN_STATUS_INVALID_VALUE) if knob + * filtering empties a non-empty catalog (see throwUnsatisfiableKnobFilter()). + */ + void buildPlan(const THandle& handle, + const IGraph& opGraph, + const IEngineConfig& engineConfig, + TContext& executionContext) const override + { + const auto context = contextFor(handle, opGraph); + // One lookup for both the order and the bound state. + const auto catalog = _stateManager.sortedCatalog(context); + if(catalog.entries.empty()) + { + throwNoApplicableKernel(); + } + + // Filtering after ranking commutes with filter-then-rank (see tparam doc). + const auto filter = readKnobFilter(engineConfig); + const auto filtered = applyKnobFilter(catalog.entries, filter); + if(filtered.empty()) + { + throwUnsatisfiableKnobFilter(filter, catalog.entries.size()); + } + + // Logs the selected kernel (RFC 0017 §10) for debugging without a debugger. + HIPDNN_PLUGIN_LOG_INFO("ingestor: engine '" << _engine.name << "' selected kernel " + << toString(filtered.front().kernelId) + << " from " << filtered.size() + << " candidate(s) (" << catalog.entries.size() + << " before knob filtering)"); + + executionContext.setPlan(std::make_unique>( + _stateManager.getDispatchDetails(filtered.front()), context, catalog.bound)); + } + + /** + * @brief One knob per KMD field the engine exposes. + * + * A knob's legal values are what the catalog implements for this graph; its + * default is the top-ranked kernel's value. + */ + std::vector + getCustomKnobs(const THandle& handle, const IGraph& opGraph) const override + { + using namespace hipdnn_flatbuffers_sdk::data_objects; + + const auto ranked = _stateManager.sortedDefinitions(contextFor(handle, opGraph)); + std::vector knobs; + if(ranked.empty()) + { + return knobs; + } + + for(const auto& knobName : _engine.knobs) + { + const auto values = KernelIngestorStateManager::knobValues(ranked, knobName); + + std::vector choices; + choices.reserve(values.size()); + for(const auto& value : values) + { + // Only integer-valued fields are expressible as knobs today. + if(const auto* intValue = std::get_if(&value)) + { + choices.push_back(*intValue); + } + } + if(choices.empty()) + { + continue; + } + + KnobT knob; + knob.knob_id = knobName; + knob.description + = "Kernel metadata field '" + knobName + "' of engine '" + _engine.name + "'"; + + IntValueT defaultValue; + // First entry is the top-ranked kernel's value. + defaultValue.value = choices.front(); + knob.default_value.Set(defaultValue); + + IntConstraintT constraint; + constraint.min_value = *std::min_element(choices.begin(), choices.end()); + constraint.max_value = *std::max_element(choices.begin(), choices.end()); + constraint.step = 1; + constraint.valid_values = std::move(choices); + knob.constraint.Set(constraint); + + knobs.push_back(std::move(knob)); + } + + return knobs; + } + +private: + /// Binds the graph and the device this call is for. + MatchContext contextFor(const THandle& handle, const IGraph& opGraph) const + { + const auto deviceId = _deviceResolver.deviceId(handle); + return MatchContext{opGraph, deviceId, _deviceResolver.deviceProperties(deviceId)}; + } + + /// Reads @p engineConfig's settings for this engine's own knobs into a KnobFilter. + KnobFilter readKnobFilter(const IEngineConfig& engineConfig) const + { + using namespace hipdnn_flatbuffers_sdk::data_objects; + + KnobFilter filter; + if(!engineConfig.isValid()) + { + return filter; + } + + for(const auto& knobName : _engine.knobs) + { + if(!engineConfig.hasKnobSetting(knobName)) + { + continue; + } + + const auto& setting = engineConfig.getKnobSettingByName(knobName); + if(setting.valueType() != KnobValue::IntValue) + { + // Only integer-valued fields are expressible as knobs today. + throw HipdnnPluginException(HIPDNN_PLUGIN_STATUS_INVALID_VALUE, + "engine '" + _engine.name + "' knob '" + knobName + + "' must be set to an integer value"); + } + filter[knobName] = setting.valueAs().value(); + } + return filter; + } + + /// The catalog entries matching every SET knob in @p filter. + std::vector applyKnobFilter(const std::vector& catalog, + const KnobFilter& filter) const + { + if(filter.empty()) + { + return catalog; + } + + std::vector filtered; + filtered.reserve(catalog.size()); + for(const auto& kernel : catalog) + { + const bool matchesEverySetKnob + = std::all_of(filter.begin(), filter.end(), [&kernel](const auto& setting) { + const auto value = kernel.tryGetMetadata(setting.first); + const auto* intValue + = value.has_value() ? std::get_if(&*value) : nullptr; + return intValue != nullptr && *intValue == setting.second; + }); + if(matchesEverySetKnob) + { + filtered.push_back(kernel); + } + } + return filtered; + } + + /// Reports an empty catalog; shared by getMaxWorkspaceSize() and buildPlan(). + [[noreturn]] void throwNoApplicableKernel() const + { + throw HipdnnPluginException(HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR, + "engine '" + _engine.name + + "' accepted this graph but has no applicable kernel"); + } + + /** + * @brief Reports a knob combination no kernel in the catalog implements. + * + * Names every set knob and value, and how many kernels survived matching before + * knob filtering. + */ + [[noreturn]] void throwUnsatisfiableKnobFilter(const KnobFilter& filter, + size_t survivorsBeforeFilter) const + { + std::string settingsText; + for(const auto& [knobName, value] : filter) + { + if(!settingsText.empty()) + { + settingsText += ", "; + } + settingsText += knobName + "=" + std::to_string(value); + } + + throw HipdnnPluginException( + HIPDNN_PLUGIN_STATUS_INVALID_VALUE, + "engine '" + _engine.name + "' has no kernel satisfying the requested knob setting(s) " + + settingsText + " (" + std::to_string(survivorsBeforeFilter) + + " kernel(s) matched the graph before knob filtering)"); + } + + const EngineDescriptor& _engine; + /// Held by reference; owned by the engine, which outlives this builder. + const KernelIngestorStateManager& _stateManager; + const IDeviceResolver& _deviceResolver; +}; + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/IDeviceResolver.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/IDeviceResolver.hpp new file mode 100644 index 000000000000..41595e23c29a --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/IDeviceResolver.hpp @@ -0,0 +1,42 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include + +namespace hipdnn_plugin_sdk::ingestor +{ + +/** + * @brief Answers which device a call is for, from the handle that carries it. + * + * Resolved per call, not at engine construction: an engine is shared across every + * handle in the process, and a handle can rebind to a different device between calls. + */ +template +class IDeviceResolver +{ +public: + virtual ~IDeviceResolver() = default; + + /// @brief The device @p handle currently targets. + // NOLINTNEXTLINE(portability-template-virtual-member-function) + virtual DeviceId deviceId(const THandle& handle) const = 0; + + /** + * @brief Properties of @p deviceId, for the `$device.*` expression namespace. + * + * Returns a reference valid for the resolver's lifetime; a MatchContext binds it + * rather than copying. Implementations should cache. + */ + // NOLINTNEXTLINE(portability-template-virtual-member-function) + virtual const hipDeviceProp_t& deviceProperties(DeviceId deviceId) const = 0; +}; + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/IKernelDispatchHandler.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/IKernelDispatchHandler.hpp new file mode 100644 index 000000000000..69b52584c57c --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/IKernelDispatchHandler.hpp @@ -0,0 +1,94 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include + +#include +#include +#include + +namespace hipdnn_plugin_sdk::ingestor +{ + +/** + * @brief Launch state derived from the graph, resolved once at plan build. + * + * A provider derives from this to hold what its kernels need (argument uids, launch + * geometry, code object, argument buffer). Must not reference the MatchContext or + * BoundTokens it was built from (RFC 0017 §8.5). + */ +class PreparedDispatch +{ +public: + virtual ~PreparedDispatch() = default; +}; + +/** + * @brief Native escape hatch for a UDD: sizes, prepares, and launches a kernel. + * + * Templated on THandle to match IPlanBuilder and IPlan. Also answers workspace size + * (RFC 0017 §6), since that query arrives before a kernel is chosen. + */ +template +class IKernelDispatchHandler +{ +public: + virtual ~IKernelDispatchHandler() = default; + + /** + * @brief Global scratch this kernel requires, in bytes. + * + * Answered per kernel before selection; must not depend on which other kernels + * are present (the catalog is never passed in). + * + * @param bound Graph state matching already resolved (RFC 0017 §8.1: not re-derived). + */ + // NOLINTNEXTLINE(portability-template-virtual-member-function) + virtual size_t workspaceBytes(const MatchContext& context, + const BoundTokens& bound, + const KernelDefinition& kernel) const + = 0; + + /** + * @brief Resolves everything @p kernel's launch needs from the bound token state. + * + * Called once at plan build, while @p context is still valid. The returned object + * is owned by the plan and MUST NOT reference @p context or @p bound. + */ + // NOLINTNEXTLINE(portability-template-virtual-member-function) + virtual std::unique_ptr prepare(const MatchContext& context, + const BoundTokens& bound, + const KernelDefinition& kernel) const + = 0; + + /** + * @brief Launches a prepared kernel against the caller's device buffers. + * + * @param handle Supplies the stream the launch is ordered on. + * @param prepared The object this handler returned from prepare(). + * @param deviceBuffers uid/pointer pairs; resolve via + * hipdnn_plugin_sdk::findDeviceBuffer. + * @param numDeviceBuffers Length of @p deviceBuffers. + * @param workspace The plan's scratch, or nullptr when workspaceBytes() was 0. + * + * @note May run concurrently across threads; must not mutate @p prepared or the + * handler. + */ + // NOLINTNEXTLINE(portability-template-virtual-member-function) + virtual void launch(const THandle& handle, + const PreparedDispatch& prepared, + const hipdnnPluginDeviceBuffer_t* deviceBuffers, + uint32_t numDeviceBuffers, + void* workspace) const + = 0; +}; + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/IKernelHeuristic.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/IKernelHeuristic.hpp new file mode 100644 index 000000000000..64f4f08245ec --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/IKernelHeuristic.hpp @@ -0,0 +1,131 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace hipdnn_plugin_sdk::ingestor +{ + +/** + * @brief Chooses which kernel within an engine to run, given the kernels that fit. + * + * One level below hipDNN's engine-selection heuristic, which picks the engine. + * + * An implementation supplies only `score()`: it ranks one kernel at a time and never + * sees the catalog, so filtering and ranking commute (RFC 0017 §9.2). `rank()` is + * non-virtual; a selector over the whole candidate set is the heuristic follow-up RFC. + */ +class IKernelHeuristic +{ +public: + virtual ~IKernelHeuristic() = default; + + /// @brief Scores one kernel for one problem. Higher wins. + virtual double score(const KernelDefinition& kernel, const MatchContext& context) const = 0; + + /** + * @brief Orders @p catalog best-first. + * + * Scores each entry once, then sorts descending, breaking ties on the kernel's + * `priority`, then its descriptor id as bytes (stable across runs, load orders, + * and machines). + */ + std::vector rank(const Catalog& catalog, const MatchContext& context) const + { + std::vector> scored; + scored.reserve(catalog.entries.size()); + for(const auto& entry : catalog.entries) + { + scored.emplace_back(score(entry, context), &entry); + } + + std::stable_sort(scored.begin(), scored.end(), [](const auto& lhs, const auto& rhs) { + if(lhs.first != rhs.first) + { + return lhs.first > rhs.first; + } + if(lhs.second->priority != rhs.second->priority) + { + return lhs.second->priority > rhs.second->priority; + } + return lhs.second->kernelId < rhs.second->kernelId; + }); + + std::vector ranked; + ranked.reserve(scored.size()); + for(const auto& [_, entry] : scored) + { + ranked.push_back(*entry); + } + return ranked; + } +}; + +/** + * @brief An IKernelHeuristic whose score() is a native function resolved by symbol. + * + * The UHD escape hatch: the descriptor names a symbol, this resolves it on first + * score(). Construction must stay cheap (RFC 0017 §8.1, §3). A data-driven form (UHD + * follow-up RFC) would load a model artifact instead. + */ +class NativeKernelHeuristic : public IKernelHeuristic +{ +public: + /// @param scoreSymbol Resolved from the registry on first use, not here. + explicit NativeKernelHeuristic(std::string scoreSymbol) + : _scoreSymbol(std::move(scoreSymbol)) + { + } + + double score(const KernelDefinition& kernel, const MatchContext& context) const override + { + std::call_once(_resolved, [this]() { _scoreFn = ScoreRegistry::resolve(_scoreSymbol); }); + return _scoreFn(kernel, context); + } + +private: + std::string _scoreSymbol; + /// Mutable: resolution is an implementation detail of a logically-const score(), + /// which may be called from several threads at once. + mutable std::once_flag _resolved; + mutable ScoreFn _scoreFn = nullptr; +}; + +/** + * @brief Builds the IKernelHeuristic a UHD names, keyed on its kind. + * + * @throws std::invalid_argument if @p descriptor names a kind with no adapter yet + * (HeuristicKind::MODEL). Fails at descriptor-assembly time, not at the + * first rank(). + */ +inline std::shared_ptr makeKernelHeuristic(const HeuristicDescriptor& descriptor) +{ + switch(descriptor.kind) + { + case HeuristicKind::NATIVE: + return std::make_shared(descriptor.payload); + default: + throw std::invalid_argument("heuristic '" + toString(descriptor.id) + + "' names a kind with no adapter yet"); + } +} + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/KernelDefinition.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/KernelDefinition.hpp new file mode 100644 index 000000000000..156c818f3a25 --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/KernelDefinition.hpp @@ -0,0 +1,112 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include + +#include + +namespace hipdnn_plugin_sdk::ingestor +{ + +/** + * @brief One catalog entry: a kernel that passed every matcher for a graph. + * + * Copied out of KernelIngestorStateManager rather than referenced, so a caller holds a + * stable snapshot while the source cache is concurrently evicted or refilled. + * + * Carries the kernel's completed metadata tuple (its catalog key), enough source + * detail to load it, and the ids needed to find its dispatch descriptor. + */ +struct KernelDefinition +{ + DescriptorId kernelId; + /// The pack this kernel came from; owns the matchers and UDD that apply to it. + DescriptorId packId; + /// Dispatch descriptor id, denormalized from the pack so a caller holding only a + /// KernelDefinition can reach it without walking the pack list. + DescriptorId dispatchId; + /// Where this kernel's code comes from and how to load it. + KernelSource source; + /// Every KMD field, with defaults filled in for fields the UKD omitted. + MetadataValues metadata; + int64_t priority = 0; + + /// @brief The value of a KMD field, or nullopt if the kernel has no such field. + std::optional tryGetMetadata(const std::string& field) const + { + auto it = metadata.find(field); + if(it == metadata.end()) + { + return std::nullopt; + } + return it->second; + } + + /// @brief The integer value of a KMD field. + /// @throws std::out_of_range if absent, std::invalid_argument if a different + /// alternative is held. + int64_t getIntMetadata(const std::string& field) const + { + auto it = metadata.find(field); + if(it == metadata.end()) + { + throw std::out_of_range("kernel '" + toString(kernelId) + "' has no metadata field '" + + field + "'"); + } + const auto* value = std::get_if(&it->second); + if(value == nullptr) + { + throw std::invalid_argument("metadata field '" + field + "' of kernel '" + + toString(kernelId) + "' is not an integer"); + } + return *value; + } + + /// @brief The string value of a KMD field. Throws on the same terms as getIntMetadata. + const std::string& getStringMetadata(const std::string& field) const + { + auto it = metadata.find(field); + if(it == metadata.end()) + { + throw std::out_of_range("kernel '" + toString(kernelId) + "' has no metadata field '" + + field + "'"); + } + const auto* value = std::get_if(&it->second); + if(value == nullptr) + { + throw std::invalid_argument("metadata field '" + field + "' of kernel '" + + toString(kernelId) + "' is not a string"); + } + return *value; + } + + /// @brief The int-list value of a KMD field (e.g. `stride_order`). + const std::vector& getIntListMetadata(const std::string& field) const + { + auto it = metadata.find(field); + if(it == metadata.end()) + { + throw std::out_of_range("kernel '" + toString(kernelId) + "' has no metadata field '" + + field + "'"); + } + const auto* value = std::get_if>(&it->second); + if(value == nullptr) + { + throw std::invalid_argument("metadata field '" + field + "' of kernel '" + + toString(kernelId) + "' is not an int list"); + } + return *value; + } +}; + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/KernelIngestorStateManager.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/KernelIngestorStateManager.hpp new file mode 100644 index 000000000000..0f26c5c25f88 --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/KernelIngestorStateManager.hpp @@ -0,0 +1,516 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hipdnn_plugin_sdk::ingestor +{ + +/// What a caller needs to size and launch one selected kernel. Copied out, so +/// holding one does not pin the state manager's internals. +template +struct KernelDispatcher +{ + KernelDefinition kernel; + /// Non-owning; must outlive any plan built from this dispatcher (see + /// NativeRegistry's DispatchRegistry). + const IKernelDispatchHandler* handler = nullptr; +}; + +/** + * @brief The engine's view of its own kernels: which apply to a graph, in what order, + * and how to launch one. + * + * Holds the descriptor state one engine selects over (its KMD and UHD, and the packs + * naming it) and answers hipDNN's four host calls: + * + * | Host call | Answered by | + * |------------------------------|------------------------------------------------| + * | isApplicable | unsortedDefinitions() is non-empty | + * | getDetails (knobs) | sortedDefinitions() — value sets and defaults | + * | getMaxWorkspaceSize | getDispatchDetails() per survivor, then max | + * | initializeExecutionContext | sortedDefinitions().front(), getDispatchDetails() | + * + * Ranking is cached alongside the catalog, keyed on (graph, device) rather than the + * handle. + * + * Thread-safe: the cache is internally synchronized; matcher and scorer calls run + * outside the lock. + */ +template +class KernelIngestorStateManager +{ +public: + /// How many (graph, device) catalogs to retain. Entries hold ids and metadata + /// values, not kernels or graphs; eviction costs a rematch, never a wrong answer. + static constexpr size_t DEFAULT_CATALOG_CACHE_CAPACITY = 256; + + /** + * @param schema The engine's KMD. Supplies the defaults completing each kernel's + * metadata tuple, and the field types a value is checked against. + * @param matchers Every UMD any of @p packs references, by id. + * @param dispatches Every UDD any of @p packs references, by id. + * @param packs The KDPs naming this engine. + * @param heuristic That engine's UHD, resolved to a scorer. + * + * @throws std::invalid_argument if a pack names an unknown matcher or dispatch + * descriptor, or if two kernels share a metadata tuple (the catalog key + * must be unique). + * + * Eagerly validates every pack and kernel at plugin load (validateAndIndexPacks()); + * kernel source compilation and heuristic model loading stay lazy until a graph + * needs them. + */ + KernelIngestorStateManager(MetadataSchema schema, + std::vector matchers, + std::vector dispatches, + std::vector packs, + std::shared_ptr heuristic, + size_t catalogCacheCapacity = DEFAULT_CATALOG_CACHE_CAPACITY) + : _schema(std::move(schema)) + , _packs(std::move(packs)) + , _heuristic(std::move(heuristic)) + , _catalogCache(catalogCacheCapacity) + { + if(_heuristic == nullptr) + { + throw std::invalid_argument("kernel ingestor requires a heuristic"); + } + + for(auto& matcher : matchers) + { + const auto id = matcher.id; + if(const auto [it, inserted] = _matchers.emplace(id, std::move(matcher)); !inserted) + { + // A pack naming this id would otherwise silently run whichever + // matcher loaded first. + throw std::invalid_argument("duplicate match descriptor id '" + toString(id) + + "' collides with '" + it->second.name + "'"); + } + } + for(auto& dispatch : dispatches) + { + const auto id = dispatch.id; + if(const auto [it, inserted] = _dispatches.emplace(id, std::move(dispatch)); !inserted) + { + throw std::invalid_argument("duplicate dispatch descriptor id '" + toString(id) + + "' collides with '" + it->second.name + "'"); + } + } + + validateAndIndexPacks(); + } + + const MetadataSchema& metadataSchema() const + { + return _schema; + } + + /** + * @brief Every kernel that applies to the graph and device @p context names, in no + * particular order. + * + * Runs pack matchers in pruning order (graph-scoped, then kernel-scoped) and + * caches the result. A graph with no identity is matched fresh every call. + */ + std::vector unsortedDefinitions(const MatchContext& context) const + { + return catalogFor(context).entries; + } + + /** + * @brief The unranked catalog and the state matching bound, from one lookup. + * + * Returned together so a caller needing both does not match twice for a graph + * with no identity to cache under (RFC 0017 §8.1). `bound` is empty when no + * graph-scoped matcher bound anything. + */ + Catalog unsortedCatalog(const MatchContext& context) const + { + return catalogFor(context); + } + + /** + * @brief Every kernel that applies to the graph and device @p context names, best + * first. + * + * Uses the cached order if present; otherwise ranks and caches it, building the + * catalog first if needed. + */ + std::vector sortedDefinitions(const MatchContext& context) const + { + return sortedCatalog(context).entries; + } + + /** + * @brief The ranked catalog and the state matching bound, from one lookup. + * + * Avoids matching twice for a graph with no identity to cache under. + */ + Catalog sortedCatalog(const MatchContext& context) const + { + Catalog catalog = catalogFor(context); + if(catalog.isSorted) + { + return catalog; + } + + catalog.entries = _heuristic->rank(catalog, context); + catalog.isSorted = true; + + if(const auto key = cacheKey(context); key.has_value()) + { + _catalogCache.put(*key, catalog); + } + + return catalog; + } + + /** + * @brief Resolves how to size and launch @p kernel. + * @throws std::runtime_error if the kernel's dispatch descriptor is unknown or + * names a symbol the provider does not ship; fails hard rather than + * declining silently. + */ + KernelDispatcher getDispatchDetails(const KernelDefinition& kernel) const + { + auto it = _dispatches.find(kernel.dispatchId); + if(it == _dispatches.end()) + { + throw std::runtime_error("kernel '" + toString(kernel.kernelId) + + "' names unknown dispatch descriptor '" + + toString(kernel.dispatchId) + "'"); + } + + return {kernel, DispatchRegistry::resolve(it->second.dispatchSymbol)}; + } + + /// @brief The distinct values @p field takes across @p kernels, in ranked-first order. + /// + /// Reflects what the catalog implements, not the KMD field's theoretical range. + static std::vector knobValues(const std::vector& kernels, + const std::string& field) + { + std::vector values; + for(const auto& kernel : kernels) + { + const auto value = kernel.tryGetMetadata(field); + if(!value.has_value()) + { + continue; + } + if(std::find(values.begin(), values.end(), *value) == values.end()) + { + values.push_back(*value); + } + } + return values; + } + +private: + void validateAndIndexPacks() + { + std::vector seenKeys; + + for(const auto& pack : _packs) + { + for(const auto& matcherId : pack.matcherIds) + { + if(_matchers.find(matcherId) == _matchers.end()) + { + throw std::invalid_argument("pack '" + toString(pack.id) + + "' names unknown matcher '" + toString(matcherId) + + "'"); + } + } + if(_dispatches.find(pack.dispatchId) == _dispatches.end()) + { + throw std::invalid_argument("pack '" + toString(pack.id) + + "' names unknown dispatch descriptor '" + + toString(pack.dispatchId) + "'"); + } + + for(const auto& kernel : pack.kernels) + { + auto key = completeMetadata(kernel); + if(std::find(seenKeys.begin(), seenKeys.end(), key) != seenKeys.end()) + { + throw std::invalid_argument( + "kernel '" + toString(kernel.id) + + "' duplicates the metadata tuple of another kernel under schema '" + + _schema.name + "'; the tuple is the catalog key and must be unique"); + } + seenKeys.push_back(std::move(key)); + } + } + } + + /// A kernel's declared metadata values, with the KMD's defaults filled in for + /// omitted fields. This completed tuple, not the descriptor id, is the catalog key. + MetadataValues completeMetadata(const KernelDescriptor& kernel) const + { + MetadataValues complete; + + for(const auto& field : _schema.fields) + { + auto it = kernel.metadata.find(field.name); + + if(it == kernel.metadata.end()) + { + // A field with no default must be set by every kernel; omission is + // an authoring error. + if(!field.defaultValue.has_value()) + { + throw std::invalid_argument("kernel '" + toString(kernel.id) + + "' omits metadata field '" + field.name + + "', which declares no default"); + } + complete.emplace(field.name, *field.defaultValue); + continue; + } + + // Caught here rather than surfacing as a bad_variant_access far from the + // descriptor that caused it. + if(metadataTypeOf(it->second) != field.type) + { + throw std::invalid_argument("kernel '" + toString(kernel.id) + + "' supplies metadata field '" + field.name + + "' with a value of the wrong type"); + } + complete.emplace(field.name, it->second); + } + + // Rejects fields the schema doesn't declare (usually a misspelled name); + // otherwise silently ignored while joining the key. + for(const auto& [name, value] : kernel.metadata) + { + if(complete.find(name) == complete.end()) + { + throw std::invalid_argument("kernel '" + toString(kernel.id) + + "' supplies metadata field '" + name + + "', which its engine's metadata schema does " + "not declare"); + } + } + + return complete; + } + + /// Device comes from the context, not a separate argument, to avoid caching one + /// device's catalog under another device's key. + std::optional cacheKey(const MatchContext& context) const + { + const auto graphId = tryGetGraphId(context.graph); + if(!graphId.has_value()) + { + return std::nullopt; + } + return CatalogKey{*graphId, context.deviceId}; + } + + Catalog catalogFor(const MatchContext& context) const + { + const auto key = cacheKey(context); + if(key.has_value()) + { + if(auto cached = _catalogCache.get(*key); cached.has_value()) + { + HIPDNN_PLUGIN_LOG_TRACE("ingestor: catalog cache hit for device " + << context.deviceId); + return *cached; + } + } + else + { + // A graph with no identity is re-matched every call rather than cached. + HIPDNN_PLUGIN_LOG_TRACE( + "ingestor: graph carries no identity, so its catalog cannot be cached"); + } + + Catalog catalog = buildCatalog(context); + + if(key.has_value()) + { + _catalogCache.put(*key, catalog); + } + + return catalog; + } + + /// One graph-scoped matcher's verdict for one (graph, device), keyed by matcher id. + /// Memoizes per matcher so a pack merges only the matchers it lists. + struct GraphMatcherVerdict + { + bool passed = false; + BoundTokens bound; + }; + using GraphMatcherMemo + = std::unordered_map; + + /** + * @brief Runs every pack's matchers over @p context, cheapest and broadest first. + * + * Graph-scoped matchers run once per (graph, device) regardless of how many packs + * list them; a failure disqualifies the pack before kernel-scoped matching runs. + * A pruned pack's graph-scoped bindings are discarded, never merged. + */ + Catalog buildCatalog(const MatchContext& context) const + { + Catalog catalog; + GraphMatcherMemo graphVerdicts; + + for(const auto& pack : _packs) + { + // Merged into catalog.bound below only if the pack survives. + BoundTokens packBound; + if(!graphLevelMatchersPass(pack, context, graphVerdicts, packBound)) + { + // RFC 0017 §10: surfaces the pack decline reason for operators. + HIPDNN_PLUGIN_LOG_INFO("ingestor: pack " << toString(pack.id) + << " declined at a graph-scoped matcher"); + continue; + } + + mergeBound(catalog.bound, packBound, pack.id); + + size_t admitted = 0; + for(const auto& kernel : pack.kernels) + { + KernelDefinition definition{kernel.id, + pack.id, + pack.dispatchId, + kernel.source, + completeMetadata(kernel), + kernel.priority}; + + if(kernelLevelMatchersPass(pack, context, definition)) + { + catalog.entries.push_back(std::move(definition)); + ++admitted; + } + } + + HIPDNN_PLUGIN_LOG_INFO("ingestor: pack " << toString(pack.id) << " admitted " + << admitted << " of " << pack.kernels.size() + << " kernel(s) after kernel-scoped matching"); + } + + // Logged unconditionally so an empty catalog is distinguishable from a + // missing engine id. + HIPDNN_PLUGIN_LOG_INFO("ingestor: catalog for device " + << context.deviceId << " holds " << catalog.entries.size() + << " kernel(s) from " << _packs.size() << " pack(s)"); + return catalog; + } + + /// Folds @p packBound into @p bound. Throws if two packs bind the same token to + /// different values. + static void + mergeBound(BoundTokens& bound, const BoundTokens& packBound, const DescriptorId& packId) + { + for(const auto& [token, value] : packBound) + { + const auto [it, inserted] = bound.emplace(token, value); + if(!inserted && !(it->second == value)) + { + throw std::runtime_error( + "pack '" + toString(packId) + "' binds token '" + token + + "' to a value that disagrees with another pack's binding of the " + "same token; one token name must mean one thing across an engine"); + } + } + } + + /** + * @param packBound Accumulates this pack's graph-scoped bindings; merged by the + * caller only if this returns true. + */ + bool graphLevelMatchersPass(const KernelDescriptorPack& pack, + const MatchContext& context, + GraphMatcherMemo& graphVerdicts, + BoundTokens& packBound) const + { + for(const auto& matcherId : pack.matcherIds) + { + const auto& matcher = _matchers.at(matcherId); + if(matcher.scope != MatchScope::GRAPH) + { + continue; + } + + auto memo = graphVerdicts.find(matcherId); + if(memo == graphVerdicts.end()) + { + // Evaluated once per matcher; later packs reuse the memoized verdict. + GraphMatcherVerdict verdict; + verdict.passed + = GraphMatcherRegistry::resolve(matcher.matchSymbol)(context, verdict.bound); + memo = graphVerdicts.emplace(matcherId, std::move(verdict)).first; + } + + if(!memo->second.passed) + { + return false; + } + + // Merges this matcher's bindings into the pack's scoped view. + packBound.insert(memo->second.bound.begin(), memo->second.bound.end()); + } + return true; + } + + bool kernelLevelMatchersPass(const KernelDescriptorPack& pack, + const MatchContext& context, + const KernelDefinition& kernel) const + { + for(const auto& matcherId : pack.matcherIds) + { + const auto& matcher = _matchers.at(matcherId); + if(matcher.scope != MatchScope::KERNEL) + { + continue; + } + if(!KernelMatcherRegistry::resolve(matcher.matchSymbol)(context, kernel)) + { + return false; + } + } + return true; + } + + MetadataSchema _schema; + std::unordered_map _matchers; + std::unordered_map _dispatches; + std::vector _packs; + std::shared_ptr _heuristic; + /// Mutable because the query methods are logically const; the cache is internally + /// synchronized. + mutable LruCache _catalogCache; +}; + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/LruCache.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/LruCache.hpp new file mode 100644 index 000000000000..a5115c6aa950 --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/LruCache.hpp @@ -0,0 +1,108 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include +#include +#include + +namespace hipdnn_plugin_sdk::ingestor +{ + +/** + * @brief A bounded, thread-safe least-recently-used cache. + * + * Sized by entry count rather than bytes: entries hold descriptor ids and bound field + * values, not kernels or graphs. + * + * Eviction costs a rematch, never a wrong answer. + * + * @tparam Key Must be hashable by @p Hash and equality-comparable. + * @tparam Value Copied in and out; callers hold snapshots, not references into the cache. + * @tparam Hash Defaults to std::hash; composite keys need an explicit hash. + */ +template > +class LruCache +{ +public: + /// @throws std::invalid_argument if @p capacity is zero. + explicit LruCache(size_t capacity) + : _capacity(capacity) + { + if(capacity == 0) + { + throw std::invalid_argument("LruCache capacity must be non-zero"); + } + } + + /// @brief Looks up @p key, marking it most-recently-used on a hit. + /// @return A copy of the cached value, or nullopt on a miss. + std::optional get(const Key& key) + { + const std::lock_guard lock(_mutex); + + auto it = _index.find(key); + if(it == _index.end()) + { + return std::nullopt; + } + + _order.splice(_order.begin(), _order, it->second); + return it->second->second; + } + + /// @brief Inserts or overwrites @p key, marking it most-recently-used and evicting the + /// least-recently-used entry if that pushes the cache over capacity. + void put(const Key& key, Value value) + { + const std::lock_guard lock(_mutex); + + auto it = _index.find(key); + if(it != _index.end()) + { + it->second->second = std::move(value); + _order.splice(_order.begin(), _order, it->second); + return; + } + + _order.emplace_front(key, std::move(value)); + _index[key] = _order.begin(); + + if(_index.size() > _capacity) + { + _index.erase(_order.back().first); + _order.pop_back(); + } + } + + size_t size() const + { + const std::lock_guard lock(_mutex); + return _index.size(); + } + + size_t capacity() const + { + return _capacity; + } + +private: + using Entry = std::pair; + + mutable std::mutex _mutex; + size_t _capacity; + /// Most-recently-used first; a list keeps splice-to-front from invalidating _index iterators. + std::list _order; + std::unordered_map::iterator, Hash> _index; +}; + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/MatchContext.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/MatchContext.hpp new file mode 100644 index 000000000000..66be4cc1eb0b --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/MatchContext.hpp @@ -0,0 +1,122 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace hipdnn_plugin_sdk::ingestor +{ + +/// The device a catalog was built for: a plain HIP device ordinal (hipGetDevice). +using DeviceId = int; + +/// Device id meaning "no resolvable device". Negative so it never aliases a real ordinal; +/// matchers decline on it rather than matching against another device's facts. +inline constexpr DeviceId NO_DEVICE = -1; + +/// A finalized graph's stable identity, preserved across serialization round trips. +using GraphId = hipdnn_flatbuffers_sdk::utilities::UuidBytes; + +/// The catalog cache key. Excludes the handle: a handle's lifetime is unrelated to a plan's +/// validity, so keying on it would be wrong. +/// +/// RFC 0017 §8.1 also keys this on engine id and a descriptor-inventory generation (§8.6); +/// both are constant today so they are omitted. +struct CatalogKey +{ + GraphId graphId; + DeviceId deviceId; + + bool operator==(const CatalogKey& other) const noexcept + { + return graphId == other.graphId && deviceId == other.deviceId; + } +}; + +/// A user-defined key has no std::hash, so the cache is given this explicitly. +struct CatalogKeyHash +{ + size_t operator()(const CatalogKey& key) const noexcept + { + // UUID v4 bytes are already well-distributed; this folds them with the device ordinal. + size_t hash = 1469598103934665603ULL; + for(const uint8_t byte : key.graphId) + { + hash ^= static_cast(byte); + hash *= 1099511628211ULL; + } + hash ^= static_cast(key.deviceId) + 0x9e3779b9ULL + (hash << 6U) + (hash >> 2U); + return hash; + } +}; + +/// Token name to MetadataValue map of what matching resolved for one graph. +using BoundTokens = std::unordered_map; + +/// Returns nullopt when absent or bound to a non-integer value, so type confusion never reads +/// as a missing token. +inline std::optional tryGetBoundInt(const BoundTokens& bound, std::string_view token) +{ + const auto it = bound.find(std::string(token)); + if(it == bound.end()) + { + return std::nullopt; + } + const auto* value = std::get_if(&it->second); + if(value == nullptr) + { + return std::nullopt; + } + return *value; +} + +/** + * @brief Bound token state a matcher, scorer, or dispatch formula reads (`$graph.*`, + * `$device.*`); `$kernel.*` arrives separately as a KernelDefinition per candidate. + * + * Holds references, not copies: built on the stack for one matching pass and must not + * outlive the graph it names. + */ +struct MatchContext +{ + const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& graph; + DeviceId deviceId; + const hipDeviceProp_t& deviceProperties; +}; + +/// @brief The graph's stable identity, or nullopt when absent (legacy/unfinalized graphs) or +/// non-v4 (uniqueness is otherwise unguaranteed). Callers must treat both as "cannot cache". +inline std::optional + tryGetGraphId(const hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph& graph) +{ + const auto* id = graph.getGraph().id(); + if(id == nullptr) + { + return std::nullopt; + } + const auto bytes = hipdnn_flatbuffers_sdk::utilities::toUuidBytes(*id); + if(!hipdnn_flatbuffers_sdk::utilities::isUuidV4(bytes)) + { + return std::nullopt; + } + return bytes; +} + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/NativeRegistry.hpp b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/NativeRegistry.hpp new file mode 100644 index 000000000000..ea4a1f2acdbf --- /dev/null +++ b/projects/hipdnn/plugin_sdk/include/hipdnn_plugin_sdk/ingestor/NativeRegistry.hpp @@ -0,0 +1,121 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include + +#include +#include +#include + +/** + * @file NativeRegistry.hpp + * @brief Symbol name to native function, the escape hatch descriptors resolve through. + * + * A descriptor names a custom predicate or plan by symbol rather than carrying inline + * code (RFC 0017); the provider resolves it here. Lookup fails closed. + */ +namespace hipdnn_plugin_sdk::ingestor +{ + +/// Graph-scoped applicability check: reads only graph/device facts, so it runs once per +/// (graph, device); failure prunes every kernel in the pack. On success, writes resolved +/// values into @p bound (RFC 0017 §8.5). Must be thread-safe. +using GraphMatcherFn = bool (*)(const MatchContext&, BoundTokens& bound); + +/// Kernel-scoped applicability check: also reads the candidate's metadata, so it runs +/// once per surviving kernel and disqualifies that kernel alone. Must be thread-safe. +using KernelMatcherFn = bool (*)(const MatchContext&, const KernelDefinition&); + +/// Scores one kernel for one problem. Higher is better. Never handed the catalog (see +/// IKernelHeuristic). Must be thread-safe. +using ScoreFn = double (*)(const KernelDefinition&, const MatchContext&); + +/** + * @brief The provider's registry of native implementations, keyed by symbol name. + * + * One instance per registered type per loaded image: requires `CXX_VISIBILITY_PRESET + * hidden` and `--exclude-libs=ALL` (`src/CMakeLists.txt`) so two loaded copies do not + * share a registry. + * + * Thread-safe: registration and lookup are mutex-guarded. + * + * @tparam T The registered callable or interface pointer type. + */ +template +class NativeRegistry +{ +public: + /** + * @brief Registers @p implementation under @p symbol. + * @throws std::runtime_error if @p symbol is already registered (which wins would + * depend on static-init order). + */ + static void registerSymbol(const std::string& symbol, T implementation) + { + auto& self = instance(); + const std::lock_guard lock(self._mutex); + + auto [it, inserted] = self._symbols.emplace(symbol, implementation); + if(!inserted) + { + throw std::runtime_error("duplicate native ingestor symbol registration: " + symbol); + } + } + + /** + * @brief Resolves @p symbol. + * @throws std::runtime_error if no implementation is registered under it. + */ + static T resolve(const std::string& symbol) + { + auto& self = instance(); + const std::lock_guard lock(self._mutex); + + auto it = self._symbols.find(symbol); + if(it == self._symbols.end()) + { + throw std::runtime_error("unresolved native ingestor symbol: " + symbol); + } + return it->second; + } + + /// @brief Removes @p symbol if present. Used by tests to swap in a temporary + /// implementation and restore the original; production code never unregisters. + static void unregisterSymbol(const std::string& symbol) + { + auto& self = instance(); + const std::lock_guard lock(self._mutex); + self._symbols.erase(symbol); + } + +private: + static NativeRegistry& instance() + { + // Function-local static: avoids depending on unspecified namespace-scope init order. + static NativeRegistry s_instance; + return s_instance; + } + + std::mutex _mutex; + std::unordered_map _symbols; +}; + +using GraphMatcherRegistry = NativeRegistry; +using KernelMatcherRegistry = NativeRegistry; +using ScoreRegistry = NativeRegistry; + +/// Non-owning: the provider owns each handler and must keep it alive for as long as +/// any plan built from it can execute. +template +using DispatchRegistry = NativeRegistry*>; + +} // namespace hipdnn_plugin_sdk::ingestor + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/CMakeLists.txt b/projects/hipdnn/plugin_sdk/tests/CMakeLists.txt index 3c09159bc76d..499e439a2c18 100644 --- a/projects/hipdnn/plugin_sdk/tests/CMakeLists.txt +++ b/projects/hipdnn/plugin_sdk/tests/CMakeLists.txt @@ -26,6 +26,17 @@ add_executable(hipdnn_plugin_sdk_tests TestVersion.cpp TestPluginDeviceBuffers.cpp TestRuntimePassByValue.cpp + $<$:ingestor/TestLruCache.cpp> + $<$:ingestor/TestNativeRegistry.cpp> + $<$:ingestor/TestKernelHeuristic.cpp> + $<$:ingestor/TestMatchContext.cpp> + $<$:ingestor/TestDescriptors.cpp> + $<$:ingestor/TestKernelDefinition.cpp> + $<$:ingestor/TestKernelIngestorStateManager.cpp> + $<$:ingestor/TestGenericPlan.cpp> + $<$:ingestor/TestGenericEngine.cpp> + $<$:ingestor/TestGenericPlanBuilder.cpp> + $<$:TestKernelIngestorConcurrency.cpp> ) target_compile_options(hipdnn_plugin_sdk_tests PRIVATE ${HIPDNN_WARNING_COMPILE_OPTIONS} ${HIPDNN_NO_RTTI_OPTIONS}) diff --git a/projects/hipdnn/plugin_sdk/tests/TestKernelIngestorConcurrency.cpp b/projects/hipdnn/plugin_sdk/tests/TestKernelIngestorConcurrency.cpp new file mode 100644 index 000000000000..e9580281763f --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/TestKernelIngestorConcurrency.cpp @@ -0,0 +1,221 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include "ingestor/KernelIngestorTestFixtures.hpp" + +/** + * @file TestKernelIngestorConcurrency.cpp + * @brief Concurrency tests for the ingestor's shared per-process catalog cache: internal + * consistency and thread/order independence under concurrent access. + */ +namespace +{ + +using namespace hipdnn_plugin_sdk::ingestor; +using namespace hipdnn_plugin_sdk::ingestor::testing; + +constexpr int THREAD_COUNT = 8; +constexpr int ITERATIONS_PER_THREAD = 200; + +/// Runs @p body on threadCount threads, releasing them together so they truly overlap. +template +void runConcurrently(int threadCount, Body body) +{ + std::atomic go{false}; + std::vector threads; + threads.reserve(static_cast(threadCount)); + + for(int i = 0; i < threadCount; ++i) + { + threads.emplace_back([&go, &body, i]() { + while(!go.load(std::memory_order_acquire)) + { + std::this_thread::yield(); + } + body(i); + }); + } + + go.store(true, std::memory_order_release); + for(auto& thread : threads) + { + thread.join(); + } +} + +// --------------------------------------------------------------------------- +// LruCache +// --------------------------------------------------------------------------- + +TEST(TestIngestorCacheConcurrency, SurvivesConcurrentReadsAndWrites) +{ + // Capacity far below the key range: eviction runs constantly, so readers race + // evicting writers on the same entries. + LruCache cache(16); + + runConcurrently(THREAD_COUNT, [&cache](int thread) { + for(int i = 0; i < ITERATIONS_PER_THREAD; ++i) + { + const int key = (thread * ITERATIONS_PER_THREAD + i) % 128; + cache.put(key, key * 2); + + // Hit must match; a miss is legal (raced with eviction). + if(const auto found = cache.get(key); found.has_value()) + { + EXPECT_EQ(*found, key * 2); + } + } + }); + + EXPECT_LE(cache.size(), cache.capacity()); +} + +TEST(TestIngestorCacheConcurrency, NeverExceedsCapacityUnderContention) +{ + LruCache cache(4); + + runConcurrently(THREAD_COUNT, [&cache](int thread) { + for(int i = 0; i < ITERATIONS_PER_THREAD; ++i) + { + cache.put(thread * ITERATIONS_PER_THREAD + i, i); + } + }); + + // Concurrent insertion must not grow the cache past capacity. + EXPECT_LE(cache.size(), cache.capacity()); +} + +// --------------------------------------------------------------------------- +// KernelIngestorStateManager +// --------------------------------------------------------------------------- + +TEST(TestIngestorStateManagerConcurrency, ServesOneGraphFromManyThreadsConsistently) +{ + const ScopedTestSymbols symbols; + const auto manager = makeTestStateManager(); + const TestGraph graph(makeGraphId(0x21)); + const auto properties = testDeviceProperties(); + + // Same graph and device from every thread: shared-cache read contention. + runConcurrently(THREAD_COUNT, [&](int) { + for(int i = 0; i < ITERATIONS_PER_THREAD; ++i) + { + const MatchContext context{graph, 0, properties}; + const auto definitions = manager->unsortedDefinitions(context); + ASSERT_EQ(definitions.size(), 2U); + } + }); +} + +TEST(TestIngestorStateManagerConcurrency, RanksConsistentlyFromManyThreads) +{ + const ScopedTestSymbols symbols; + const auto manager = makeTestStateManager(); + const TestGraph graph(makeGraphId(0x22)); + const auto properties = testDeviceProperties(); + + // Ranking writes the sorted catalog back: readers and the writer race here. + runConcurrently(THREAD_COUNT, [&](int) { + for(int i = 0; i < ITERATIONS_PER_THREAD; ++i) + { + const MatchContext context{graph, 0, properties}; + const auto ranked = manager->sortedDefinitions(context); + ASSERT_EQ(ranked.size(), 2U); + EXPECT_EQ(ranked.front().getIntMetadata(std::string(BLOCK_SIZE)), 256); + } + }); +} + +TEST(TestIngestorStateManagerConcurrency, KeepsPerDeviceCatalogsDistinct) +{ + const ScopedTestSymbols symbols; + const auto manager = makeTestStateManager(); + const TestGraph graph(makeGraphId(0x23)); + const auto properties = testDeviceProperties(); + + // Same graph, distinct devices concurrently: catalog is keyed on both and must not + // cross-serve. + runConcurrently(THREAD_COUNT, [&](int thread) { + const DeviceId deviceId = thread % 4; + for(int i = 0; i < ITERATIONS_PER_THREAD; ++i) + { + const MatchContext context{graph, deviceId, properties}; + const auto definitions = manager->unsortedDefinitions(context); + ASSERT_EQ(definitions.size(), 2U); + } + }); +} + +TEST(TestIngestorStateManagerConcurrency, ServesUncacheableGraphsConcurrently) +{ + const ScopedTestSymbols symbols; + const auto manager = makeTestStateManager(); + // No graph identity: every call rematches, exercising concurrent matcher execution + // instead of the cache. + const TestGraph graph; + const auto properties = testDeviceProperties(); + + runConcurrently(THREAD_COUNT, [&](int) { + for(int i = 0; i < ITERATIONS_PER_THREAD; ++i) + { + const MatchContext context{graph, 0, properties}; + ASSERT_EQ(manager->unsortedDefinitions(context).size(), 2U); + } + }); +} + +TEST(TestIngestorStateManagerConcurrency, EvictsUnderConcurrentDistinctGraphs) +{ + const ScopedTestSymbols symbols; + // Capacity 2 against many distinct graphs: constant eviction races readers/writers. + const auto manager = makeTestStateManager(2); + const auto properties = testDeviceProperties(); + + runConcurrently(THREAD_COUNT, [&](int thread) { + for(int i = 0; i < ITERATIONS_PER_THREAD; ++i) + { + const TestGraph graph(makeGraphId(static_cast((thread * 7 + i) % 32))); + const MatchContext context{graph, 0, properties}; + + // Eviction only costs a rematch, never a wrong answer. + ASSERT_EQ(manager->unsortedDefinitions(context).size(), 2U); + } + }); +} + +// --------------------------------------------------------------------------- +// NativeRegistry +// --------------------------------------------------------------------------- + +TEST(TestIngestorRegistryConcurrency, ResolvesFromManyThreads) +{ + const ScopedTestSymbols symbols; + + // Exercises concurrent resolve() from the applicability path. + runConcurrently(THREAD_COUNT, [](int) { + for(int i = 0; i < ITERATIONS_PER_THREAD; ++i) + { + EXPECT_NE(GraphMatcherRegistry::resolve(std::string(GRAPH_MATCH_SYMBOL)), nullptr); + EXPECT_NE(ScoreRegistry::resolve(std::string(SCORE_SYMBOL)), nullptr); + } + }); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/IngestorMocks.hpp b/projects/hipdnn/plugin_sdk/tests/ingestor/IngestorMocks.hpp new file mode 100644 index 000000000000..5fc39d3e8182 --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/IngestorMocks.hpp @@ -0,0 +1,68 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include + +#include + +#include +#include +#include + +#include "KernelIngestorTestFixtures.hpp" + +/** + * @file IngestorMocks.hpp + * @brief gmock doubles for the ingestor's two pure interfaces. + * + * IKernelDispatchHandler and IDeviceResolver are exercised only through the generic code + * that consumes them (GenericPlan/GenericPlanBuilder for the dispatch handler, + * GenericPlanBuilder::contextFor for the device resolver); these mocks back those tests. + */ +namespace hipdnn_plugin_sdk::ingestor::testing +{ + +/// Mocks the native dispatch escape hatch, so GenericPlan/GenericPlanBuilder tests can +/// assert what a plan asked of it without a real kernel launch. +class MockKernelDispatchHandler : public IKernelDispatchHandler +{ +public: + MOCK_METHOD(size_t, + workspaceBytes, + (const MatchContext& context, + const BoundTokens& bound, + const KernelDefinition& kernel), + (const, override)); + MOCK_METHOD(std::unique_ptr, + prepare, + (const MatchContext& context, + const BoundTokens& bound, + const KernelDefinition& kernel), + (const, override)); + MOCK_METHOD(void, + launch, + (const StubHandle& handle, + const PreparedDispatch& prepared, + const hipdnnPluginDeviceBuffer_t* deviceBuffers, + uint32_t numDeviceBuffers, + void* workspace), + (const, override)); +}; + +/// Mocks device resolution so GenericPlanBuilder::contextFor() can be shown resolving +/// per-handle device facts into the MatchContext it builds, per call rather than once at +/// construction. +class MockDeviceResolver : public IDeviceResolver +{ +public: + MOCK_METHOD(DeviceId, deviceId, (const StubHandle& handle), (const, override)); + MOCK_METHOD(const hipDeviceProp_t&, deviceProperties, (DeviceId deviceId), (const, override)); +}; + +} // namespace hipdnn_plugin_sdk::ingestor::testing + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/KernelIngestorTestFixtures.hpp b/projects/hipdnn/plugin_sdk/tests/ingestor/KernelIngestorTestFixtures.hpp new file mode 100644 index 000000000000..4690607d430e --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/KernelIngestorTestFixtures.hpp @@ -0,0 +1,654 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @file KernelIngestorTestFixtures.hpp + * @brief Shared fixtures for the ingestor's SDK-level tests under `plugin_sdk/tests/ingestor/`. + * + * Everything here is `inline`: this header is included by several translation units in + * the same test binary. + * + * Two catalog shapes recur: + * - `int`-handle (`TestHandle`/`StateManager`/`makeStateManager()`): two FLOAT kernels + * differing in block size plus one HALF kernel a kernel-scoped matcher prunes, wired + * to counting matchers. + * - `int`-handle via `makeTestStateManager()`: same catalog, wired to fixed + * (non-counting) matchers, for tests needing only a stable answer. + * - `StubHandle` (`StubSettings`/`StubContext`/`makeStubStateManager()`): minimal + * stand-ins for the provider types `GenericEngine`/`GenericPlanBuilder` are + * parameterized on. + */ +namespace hipdnn_plugin_sdk::ingestor::testing +{ + +constexpr const char* BLOCK_SIZE = "block_size"; +constexpr const char* DTYPE = "dtype"; +constexpr const char* GRAPH_MATCH_SYMBOL = "hipdnn.kernel_ingestor.test.graph_match"; +constexpr const char* KERNEL_MATCH_SYMBOL = "hipdnn.kernel_ingestor.test.kernel_match"; +constexpr const char* SCORE_SYMBOL = "hipdnn.kernel_ingestor.test.score"; + +/// Minimal IGraph exposing only identity; every other member throws so a test depending +/// on graph contents fails loudly. +class TestGraph : public hipdnn_flatbuffers_sdk::flatbuffer_utilities::IGraph +{ +public: + /// @param graphId Identity to carry, or nullopt for a legacy/unfinalized graph. + explicit TestGraph(std::optional graphId = std::nullopt) + { + flatbuffers::Offset graph; + if(graphId.has_value()) + { + const auto uuid = hipdnn_flatbuffers_sdk::utilities::toFlatbufferUuid(*graphId); + auto name = _builder.CreateString("test_graph"); + hipdnn_flatbuffers_sdk::data_objects::GraphBuilder graphBuilder(_builder); + graphBuilder.add_name(name); + graphBuilder.add_id(&uuid); + graph = graphBuilder.Finish(); + } + else + { + graph = hipdnn_flatbuffers_sdk::data_objects::CreateGraphDirect(_builder, "test_graph"); + } + _builder.Finish(graph); + } + + const hipdnn_flatbuffers_sdk::data_objects::Graph& getGraph() const override + { + return *flatbuffers::GetRoot( + _builder.GetBufferPointer()); + } + + bool isValid() const override + { + return true; + } + + uint32_t nodeCount() const override + { + return 0; + } + + bool hasOnlySupportedAttributes( + std::set /*supported*/) const override + { + return true; + } + + const hipdnn_flatbuffers_sdk::data_objects::Node& getNode(uint32_t /*index*/) const override + { + throw std::logic_error("TestGraph carries no nodes"); + } + + const hipdnn_flatbuffers_sdk::flatbuffer_utilities::INodeWrapper& + getNodeWrapper(uint32_t /*index*/) const override + { + throw std::logic_error("TestGraph carries no nodes"); + } + + const std::vector>& + nodeWrappers() const override + { + throw std::logic_error("TestGraph carries no nodes"); + } + + const std::unordered_map& + getTensorMap() const override + { + return _tensors; + } + +private: + flatbuffers::FlatBufferBuilder _builder; + std::unordered_map + _tensors; +}; + +/// Test graph id distinct per seed, shaped as a valid v4 UUID. +inline GraphId makeGraphId(uint8_t seed) +{ + GraphId id{}; + id.fill(seed); + id[6] = static_cast((id[6] & 0x0fU) | 0x40U); + id[8] = static_cast((id[8] & 0x3fU) | 0x80U); + return id; +} + +/// Distinct-per-seed id shaped to fail the v4 check, for tests needing a non-nil +/// "no identity" id. +inline GraphId makeNonV4GraphId(uint8_t seed) +{ + GraphId id{}; + id.fill(seed); + id[6] = static_cast(id[6] & 0x0fU); + return id; +} + +/// The nil UUID: all-zero bytes. +inline GraphId makeNilGraphId() +{ + return GraphId{}; +} + +inline hipDeviceProp_t testDeviceProperties() +{ + hipDeviceProp_t properties{}; + properties.warpSize = 64; + return properties; +} + +inline bool acceptAnyGraph(const MatchContext& /*context*/, BoundTokens& /*bound*/) +{ + return true; +} + +/// Accepts only FLOAT kernels; a HALF kernel in the pack is pruned. +inline bool acceptFloatKernels(const MatchContext& /*context*/, const KernelDefinition& kernel) +{ + return kernel.getStringMetadata(DTYPE) == "FLOAT"; +} + +/// Bigger block size scores higher, giving ranking a defined winner. +inline double scoreByBlockSize(const KernelDefinition& kernel, const MatchContext& /*context*/) +{ + return static_cast(kernel.getIntMetadata(BLOCK_SIZE)); +} + +/// RAII: registers this fixture's symbols for the object's lifetime, so tests sharing +/// the process-wide registry stay independent. +class ScopedTestSymbols +{ +public: + ScopedTestSymbols() + { + GraphMatcherRegistry::registerSymbol(GRAPH_MATCH_SYMBOL, &acceptAnyGraph); + KernelMatcherRegistry::registerSymbol(KERNEL_MATCH_SYMBOL, &acceptFloatKernels); + ScoreRegistry::registerSymbol(SCORE_SYMBOL, &scoreByBlockSize); + } + + ~ScopedTestSymbols() + { + GraphMatcherRegistry::unregisterSymbol(GRAPH_MATCH_SYMBOL); + KernelMatcherRegistry::unregisterSymbol(KERNEL_MATCH_SYMBOL); + ScoreRegistry::unregisterSymbol(SCORE_SYMBOL); + } + + ScopedTestSymbols(const ScopedTestSymbols&) = delete; + ScopedTestSymbols& operator=(const ScopedTestSymbols&) = delete; +}; + +/// Device resolver reporting one fixed device; no multi-device behavior is exercised. +class TestDeviceResolver : public IDeviceResolver +{ +public: + DeviceId deviceId(const int& /*handle*/) const override + { + return 0; + } + + const hipDeviceProp_t& deviceProperties(DeviceId /*deviceId*/) const override + { + return _properties; + } + +private: + hipDeviceProp_t _properties = testDeviceProperties(); +}; + +/// Descriptor id built from a short seed, keeping the real 128-bit type readable. +inline DescriptorId testId(uint8_t seed) +{ + DescriptorId id{}; + id.fill(seed); + return id; +} + +inline const DescriptorId ENGINE_ID = testId(0xE0); +inline const DescriptorId SCHEMA_ID = testId(0xE1); +inline const DescriptorId HEURISTIC_ID = testId(0xE2); +inline const DescriptorId GRAPH_MATCHER_ID = testId(0xE3); +inline const DescriptorId KERNEL_MATCHER_ID = testId(0xE4); +inline const DescriptorId DISPATCH_ID = testId(0xE5); +inline const DescriptorId PACK_ID = testId(0xE6); + +inline KernelDescriptor makeTestKernel(const DescriptorId& id, + const std::string& name, + int64_t blockSize, + const std::string& dtype) +{ + KernelDescriptor kernel; + kernel.id = id; + kernel.name = name; + kernel.source.sourceFile = "Test.cpp"; + kernel.source.entryPoint = "TestKernel"; + kernel.metadata = {{BLOCK_SIZE, MetadataValue{blockSize}}, {DTYPE, MetadataValue{dtype}}}; + return kernel; +} + +/// Two FLOAT kernels differing in block size, plus a HALF kernel the kernel-scoped +/// matcher prunes: two survivors with a defined ranking. +inline std::unique_ptr> + makeTestStateManager(size_t cacheCapacity + = KernelIngestorStateManager::DEFAULT_CATALOG_CACHE_CAPACITY) +{ + MetadataSchema schema; + schema.id = SCHEMA_ID; + schema.name = "test schema"; + // block_size has a default; dtype is mandatory, so every kernel below sets it. + schema.fields = {{BLOCK_SIZE, MetadataType::INT, MetadataValue{int64_t{64}}}, + {DTYPE, MetadataType::STRING, std::nullopt}}; + + std::vector matchers{ + {GRAPH_MATCHER_ID, "graph scoped", MatchScope::GRAPH, GRAPH_MATCH_SYMBOL}, + {KERNEL_MATCHER_ID, "kernel scoped", MatchScope::KERNEL, KERNEL_MATCH_SYMBOL}}; + std::vector dispatches{ + {DISPATCH_ID, "test dispatch", "hipdnn.kernel_ingestor.test.dispatch"}}; + + KernelDescriptorPack pack; + pack.id = PACK_ID; + pack.name = "test pack"; + pack.matcherIds = {GRAPH_MATCHER_ID, KERNEL_MATCHER_ID}; + pack.engineId = ENGINE_ID; + pack.dispatchId = DISPATCH_ID; + pack.kernels = {makeTestKernel(testId(0x64), "kernel_64_float", 64, "FLOAT"), + makeTestKernel(testId(0x65), "kernel_256_float", 256, "FLOAT"), + makeTestKernel(testId(0x66), "kernel_64_half", 64, "HALF")}; + + return std::make_unique>( + std::move(schema), + std::move(matchers), + std::move(dispatches), + std::vector{std::move(pack)}, + std::make_shared(SCORE_SYMBOL), + cacheCapacity); +} + +// Counting matchers: `TestHandle = int` catalog instrumented to count matcher-scope +// calls. Shared by TestKernelIngestorStateManager.cpp and TestGenericPlanBuilder.cpp. + +/// Counts matcher calls: graph-scoped runs once per (graph, device); kernel-scoped runs +/// once per surviving kernel. +struct MatcherCounters +{ + int graphCalls = 0; + int kernelCalls = 0; + + void reset() + { + graphCalls = 0; + kernelCalls = 0; + } +}; + +inline MatcherCounters& counters() +{ + static MatcherCounters s_counters; + return s_counters; +} + +/// Value acceptGraph binds, so a test can verify it reaches dispatch intact. +constexpr int64_t BOUND_TOKEN_VALUE = 4242; + +inline bool acceptGraph(const MatchContext& /*context*/, BoundTokens& bound) +{ + ++counters().graphCalls; + // Stands in for the tensor uids/dimensions a real matcher would resolve. + bound["test.bound_token"] = BOUND_TOKEN_VALUE; + return true; +} + +inline bool rejectGraph(const MatchContext& /*context*/, BoundTokens& /*bound*/) +{ + ++counters().graphCalls; + return false; +} + +inline bool countingFloatKernels(const MatchContext& context, const KernelDefinition& kernel) +{ + ++counters().kernelCalls; + return acceptFloatKernels(context, kernel); +} + +/// Every kernel scores the same, so ranking falls through to the tie-break. +constexpr const char* CONSTANT_SCORE_SYMBOL = "hipdnn.kernel_ingestor.test.constant_score"; + +inline double scoreConstant(const KernelDefinition& /*kernel*/, const MatchContext& /*context*/) +{ + return 1.0; +} + +/// RAII: registers the constant scorer. Must outlive the ranking call since the +/// heuristic resolves its symbol lazily. +class ScopedConstantScore +{ +public: + ScopedConstantScore() + { + ScoreRegistry::registerSymbol(CONSTANT_SCORE_SYMBOL, &scoreConstant); + } + + ~ScopedConstantScore() + { + ScoreRegistry::unregisterSymbol(CONSTANT_SCORE_SYMBOL); + } + + ScopedConstantScore(const ScopedConstantScore&) = delete; + ScopedConstantScore& operator=(const ScopedConstantScore&) = delete; +}; + +inline MetadataSchema makeSchema() +{ + return {SCHEMA_ID, + "test schema", + {{BLOCK_SIZE, MetadataType::INT, MetadataValue{int64_t{64}}}, + {DTYPE, MetadataType::STRING, std::nullopt}}}; +} + +inline KernelDescriptor makeKernel(const DescriptorId& id, + const std::string& name, + int64_t blockSize, + const std::string& dtype, + int64_t priority = 0) +{ + auto kernel = makeTestKernel(id, name, blockSize, dtype); + kernel.priority = priority; + return kernel; +} + +/// The pack shape a real engine ships, wired to the counting matchers above. +inline KernelDescriptorPack makePack(const std::vector& matcherIds) +{ + KernelDescriptorPack pack; + pack.id = PACK_ID; + pack.name = "test pack"; + pack.matcherIds = matcherIds; + pack.engineId = ENGINE_ID; + pack.dispatchId = DISPATCH_ID; + pack.kernels = {makeKernel(testId(0x64), "kernel_64_float", 64, "FLOAT"), + makeKernel(testId(0x65), "kernel_256_float", 256, "FLOAT"), + makeKernel(testId(0x66), "kernel_64_half", 64, "HALF")}; + return pack; +} + +inline std::vector makeTestMatchers() +{ + return {{GRAPH_MATCHER_ID, "graph scoped", MatchScope::GRAPH, "test.graph"}, + {KERNEL_MATCHER_ID, "kernel scoped", MatchScope::KERNEL, "test.kernel"}}; +} + +inline std::vector makeTestDispatches() +{ + return {{DISPATCH_ID, "test dispatch", "test.dispatch"}}; +} + +/// A catalog entry built directly by ranking and plan tests. +inline KernelDefinition + makeDefinition(const DescriptorId& id, int64_t blockSize, int64_t priority = 0) +{ + return {id, + PACK_ID, + DISPATCH_ID, + KernelSource{KernelSourceKind::EMBEDDED_SOURCE, "Test.cpp", "TestKernel"}, + {{BLOCK_SIZE, MetadataValue{blockSize}}}, + priority}; +} + +/// RAII: registers matchers under caller-supplied symbol names without disturbing the +/// shared fixture's registrations. +class ScopedSymbols +{ +public: + ScopedSymbols(std::string graphSymbol, + GraphMatcherFn graphFn, + std::string kernelSymbol, + KernelMatcherFn kernelFn) + : _graphSymbol(std::move(graphSymbol)) + , _kernelSymbol(std::move(kernelSymbol)) + { + GraphMatcherRegistry::registerSymbol(_graphSymbol, graphFn); + KernelMatcherRegistry::registerSymbol(_kernelSymbol, kernelFn); + // Heuristic resolves its symbol lazily, not at construction; ranking tests need + // it registered for the duration too. + ScoreRegistry::registerSymbol(SCORE_SYMBOL, &scoreByBlockSize); + counters().reset(); + } + + ~ScopedSymbols() + { + GraphMatcherRegistry::unregisterSymbol(_graphSymbol); + KernelMatcherRegistry::unregisterSymbol(_kernelSymbol); + ScoreRegistry::unregisterSymbol(SCORE_SYMBOL); + } + + ScopedSymbols(const ScopedSymbols&) = delete; + ScopedSymbols& operator=(const ScopedSymbols&) = delete; + +private: + std::string _graphSymbol; + std::string _kernelSymbol; +}; + +using TestHandle = int; +using StateManager = KernelIngestorStateManager; + +inline std::unique_ptr + makeStateManager(const std::string& scoreSymbol = SCORE_SYMBOL, + size_t cacheCapacity = StateManager::DEFAULT_CATALOG_CACHE_CAPACITY) +{ + std::vector matchers{ + {GRAPH_MATCHER_ID, "graph scoped", MatchScope::GRAPH, "test.graph"}, + {KERNEL_MATCHER_ID, "kernel scoped", MatchScope::KERNEL, "test.kernel"}}; + std::vector dispatches{{DISPATCH_ID, "test dispatch", "test.dispatch"}}; + + return std::make_unique( + makeSchema(), + std::move(matchers), + std::move(dispatches), + std::vector{makePack({GRAPH_MATCHER_ID, KERNEL_MATCHER_ID})}, + std::make_shared(scoreSymbol), + cacheCapacity); +} + +/// Registers @p handler under @p symbol in DispatchRegistry for the object's +/// lifetime. +template +class ScopedDispatchRegistration +{ +public: + ScopedDispatchRegistration(std::string symbol, const IKernelDispatchHandler& handler) + : _symbol(std::move(symbol)) + { + DispatchRegistry::registerSymbol(_symbol, &handler); + } + + ~ScopedDispatchRegistration() + { + DispatchRegistry::unregisterSymbol(_symbol); + } + + ScopedDispatchRegistration(const ScopedDispatchRegistration&) = delete; + ScopedDispatchRegistration& operator=(const ScopedDispatchRegistration&) = delete; + +private: + std::string _symbol; +}; + +// StubHandle: minimal stand-ins for the types GenericEngine/GenericPlanBuilder are +// parameterized on. Only members those templates touch are present, so a template +// depending on more fails to compile here. + +struct StubHandle +{ + void storeEngineDetailsDetachedBuffer(const void* /*ptr*/, + std::unique_ptr buffer) + { + _buffers.push_back(std::move(buffer)); + } + +private: + std::vector> _buffers; +}; + +/// Every TSettings GenericPlanBuilder is instantiated over must carry this field; +/// initializeExecutionSettings() populates it, getMaxWorkspaceSize() reads it back. +struct StubSettings +{ + KnobFilter ingestorKnobFilter; +}; + +struct StubContext +{ + void setExecutionSettings(const StubSettings& /*settings*/) {} + + void setPlan(std::unique_ptr> plan) + { + _plan = std::move(plan); + } + +private: + std::unique_ptr> _plan; +}; + +/// Device resolver over StubHandle reporting one fixed device; use IngestorMocks.hpp's +/// MockDeviceResolver for per-handle resolution. +class StubDeviceResolver : public IDeviceResolver +{ +public: + DeviceId deviceId(const StubHandle& /*handle*/) const override + { + return 0; + } + + const hipDeviceProp_t& deviceProperties(DeviceId /*deviceId*/) const override + { + return _properties; + } + +private: + hipDeviceProp_t _properties = testDeviceProperties(); +}; + +/// State manager over StubHandle: one FLOAT kernel behind one graph-scoped matcher. +inline std::unique_ptr> makeStubStateManager() +{ + MetadataSchema schema; + schema.id = SCHEMA_ID; + schema.name = "test schema"; + schema.fields = {{BLOCK_SIZE, MetadataType::INT, MetadataValue{int64_t{64}}}, + {DTYPE, MetadataType::STRING, std::nullopt}}; + + KernelDescriptorPack pack; + pack.id = PACK_ID; + pack.name = "test pack"; + pack.matcherIds = {GRAPH_MATCHER_ID}; + pack.engineId = ENGINE_ID; + pack.dispatchId = DISPATCH_ID; + pack.kernels = {makeTestKernel(testId(0x64), "kernel_64_float", 64, "FLOAT")}; + + return std::make_unique>( + std::move(schema), + std::vector{ + {GRAPH_MATCHER_ID, "graph scoped", MatchScope::GRAPH, GRAPH_MATCH_SYMBOL}}, + std::vector{ + {DISPATCH_ID, "test dispatch", "hipdnn.kernel_ingestor.test.dispatch"}}, + std::vector{std::move(pack)}, + std::make_shared(SCORE_SYMBOL)); +} + +inline EngineDescriptor makeEngineWithKnobs(std::vector knobs) +{ + EngineDescriptor engine; + engine.id = ENGINE_ID; + engine.name = "test:engine"; + engine.heuristicId = HEURISTIC_ID; + engine.metadataSchemaId = SCHEMA_ID; + engine.knobs = std::move(knobs); + return engine; +} + +/// IEngineConfig setting `knobName` to an int `value`, built as a real flatbuffer (the +/// same parsing path a real caller's setAttribute() eventually produces). +inline hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineConfigWrapper makeIntKnobEngineConfig( + flatbuffers::FlatBufferBuilder& builder, const std::string& knobName, int64_t value) +{ + using namespace hipdnn_flatbuffers_sdk::data_objects; + + std::vector> knobSettings; + knobSettings.push_back(CreateKnobSettingDirect( + builder, knobName.c_str(), KnobValue::IntValue, CreateIntValue(builder, value).Union())); + auto knobsVector = builder.CreateVector(knobSettings); + builder.Finish(CreateEngineConfig(builder, ENGINE_ID.front(), knobsVector)); + + return hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineConfigWrapper( + builder.GetBufferPointer(), builder.GetSize()); +} + +/// FloatValue knob setting, to reach readKnobFilter()'s type-rejection branch (other +/// builders here produce IntValue). +inline hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineConfigWrapper makeFloatKnobEngineConfig( + flatbuffers::FlatBufferBuilder& builder, const std::string& knobName, double value) +{ + using namespace hipdnn_flatbuffers_sdk::data_objects; + + std::vector> knobSettings; + knobSettings.push_back(CreateKnobSettingDirect(builder, + knobName.c_str(), + KnobValue::FloatValue, + CreateFloatValue(builder, value).Union())); + auto knobsVector = builder.CreateVector(knobSettings); + builder.Finish(CreateEngineConfig(builder, ENGINE_ID.front(), knobsVector)); + + return hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineConfigWrapper( + builder.GetBufferPointer(), builder.GetSize()); +} + +/// StringValue counterpart of makeFloatKnobEngineConfig(). +inline hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineConfigWrapper makeStringKnobEngineConfig( + flatbuffers::FlatBufferBuilder& builder, const std::string& knobName, const std::string& value) +{ + using namespace hipdnn_flatbuffers_sdk::data_objects; + + std::vector> knobSettings; + knobSettings.push_back( + CreateKnobSettingDirect(builder, + knobName.c_str(), + KnobValue::StringValue, + CreateStringValueDirect(builder, value.c_str()).Union())); + auto knobsVector = builder.CreateVector(knobSettings); + builder.Finish(CreateEngineConfig(builder, ENGINE_ID.front(), knobsVector)); + + return hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineConfigWrapper( + builder.GetBufferPointer(), builder.GetSize()); +} + +} // namespace hipdnn_plugin_sdk::ingestor::testing + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/TestDescriptors.cpp b/projects/hipdnn/plugin_sdk/tests/ingestor/TestDescriptors.cpp new file mode 100644 index 000000000000..66a741caaa39 --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/TestDescriptors.cpp @@ -0,0 +1,128 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include + +#include + +#include + +#include "KernelIngestorTestFixtures.hpp" + +/** + * @file TestDescriptors.cpp + * @brief Tests for Descriptors.hpp: id formatting/hashing, the MetadataValue/MetadataType + * pairing, and the KernelSource tagged union. + */ +namespace +{ + +using namespace hipdnn_plugin_sdk::ingestor; +using namespace hipdnn_plugin_sdk::ingestor::testing; + +// --------------------------------------------------------------------------- +// toString(DescriptorId) / DescriptorIdHash +// --------------------------------------------------------------------------- + +TEST(TestIngestorDescriptors, ToStringFormatsAsCanonicalUuidText) +{ + const auto id = testId(0xAB); + + const auto text = toString(id); + + // Canonical UUID text form: 32 hex digits plus 4 hyphens, 36 characters. + EXPECT_EQ(text.size(), 36U); + EXPECT_EQ(text.find("ab"), 0U); +} + +TEST(TestIngestorDescriptors, DescriptorIdHashIsConsistentForEqualIds) +{ + const DescriptorIdHash hash; + const auto first = testId(0x11); + const auto second = testId(0x11); + + EXPECT_EQ(hash(first), hash(second)); +} + +TEST(TestIngestorDescriptors, DescriptorIdHashDistinguishesDifferentIds) +{ + const DescriptorIdHash hash; + + EXPECT_NE(hash(testId(0x11)), hash(testId(0x22))); +} + +// --------------------------------------------------------------------------- +// metadataTypeOf: MetadataType must track MetadataValue's variant order exactly. +// --------------------------------------------------------------------------- + +struct MetadataTypeCase +{ + std::string name; + MetadataValue value; + MetadataType expectedType; +}; + +class TestIngestorDescriptorsMetadataTypeOf : public ::testing::TestWithParam +{ +}; + +TEST_P(TestIngestorDescriptorsMetadataTypeOf, ReportsTheVariantsAlternativeType) +{ + EXPECT_EQ(metadataTypeOf(GetParam().value), GetParam().expectedType); +} + +INSTANTIATE_TEST_SUITE_P( + EveryAlternative, + TestIngestorDescriptorsMetadataTypeOf, + ::testing::Values( + MetadataTypeCase{"Bool", MetadataValue{true}, MetadataType::BOOL}, + MetadataTypeCase{"Int", MetadataValue{int64_t{42}}, MetadataType::INT}, + MetadataTypeCase{"Float", MetadataValue{1.5}, MetadataType::FLOAT}, + MetadataTypeCase{"String", MetadataValue{std::string{"FLOAT"}}, MetadataType::STRING}, + MetadataTypeCase{ + "IntList", MetadataValue{std::vector{1, 2, 3}}, MetadataType::INT_LIST}), + [](const ::testing::TestParamInfo& info) { return info.param.name; }); + +// --------------------------------------------------------------------------- +// KernelSourceKind / KernelSource: tagged union over RFC 0017 §7's source kinds. +// --------------------------------------------------------------------------- + +TEST(TestIngestorDescriptors, KernelSourceDefaultsToEmbeddedSource) +{ + // Only kind this POC implements. + const KernelSource source{}; + + EXPECT_EQ(source.kind, KernelSourceKind::EMBEDDED_SOURCE); + EXPECT_TRUE(source.sourceFile.empty()); + EXPECT_TRUE(source.entryPoint.empty()); +} + +TEST(TestIngestorDescriptors, KernelSourceCarriesEmbeddedSourceFileAndEntryPoint) +{ + KernelSource source; + source.kind = KernelSourceKind::EMBEDDED_SOURCE; + source.sourceFile = "PointwiseAdd.cpp"; + source.entryPoint = "pointwise_add_kernel"; + + EXPECT_EQ(source.sourceFile, "PointwiseAdd.cpp"); + EXPECT_EQ(source.entryPoint, "pointwise_add_kernel"); +} + +// --------------------------------------------------------------------------- +// HeuristicDescriptor: HeuristicKind as data; dispatch is tested in TestKernelHeuristic.cpp. +// --------------------------------------------------------------------------- + +TEST(TestIngestorDescriptors, HeuristicDescriptorDefaultsToNativeKind) +{ + const HeuristicDescriptor descriptor{}; + + EXPECT_EQ(descriptor.kind, HeuristicKind::NATIVE); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/TestGenericEngine.cpp b/projects/hipdnn/plugin_sdk/tests/ingestor/TestGenericEngine.cpp new file mode 100644 index 000000000000..5b6949cf7e20 --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/TestGenericEngine.cpp @@ -0,0 +1,190 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#include "KernelIngestorTestFixtures.hpp" + +/** + * @file TestGenericEngine.cpp + * @brief Unit tests for GenericEngine.hpp: knob validation at construction and + * IEngine overrides delegating to the plan builder and state manager. + */ +namespace +{ + +using namespace hipdnn_plugin_sdk::ingestor; +using namespace hipdnn_plugin_sdk::ingestor::testing; + +using StubEngine = GenericEngine; + +// GenericEngine holds its plan builder by reference to its own members, so it is +// non-movable and non-copyable. +static_assert(!std::is_move_constructible_v); +static_assert(!std::is_move_assignable_v); +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); + +// --------------------------------------------------------------------------- +// Construction: knob validation against the metadata schema. +// --------------------------------------------------------------------------- + +TEST(TestIngestorGenericEngine, AcceptsAKnobNamingADeclaredMetadataField) +{ + const ScopedTestSymbols symbols; + const StubDeviceResolver resolver; + + EXPECT_NO_THROW( + (StubEngine(makeEngineWithKnobs({BLOCK_SIZE}), makeStubStateManager(), resolver))); +} + +TEST(TestIngestorGenericEngine, RejectsAKnobNamingNoMetadataField) +{ + const ScopedTestSymbols symbols; + const StubDeviceResolver resolver; + + // A knob naming no metadata field would otherwise be silently dropped. + EXPECT_THROW( + (StubEngine(makeEngineWithKnobs({"no_such_field"}), makeStubStateManager(), resolver)), + std::invalid_argument); +} + +// --------------------------------------------------------------------------- +// IEngine overrides: each delegates to the plan builder or the state manager. +// --------------------------------------------------------------------------- + +TEST(TestIngestorGenericEngine, IdHashesTheUedNameIntoHipdnnsEngineIdSpace) +{ + const ScopedTestSymbols symbols; + const StubDeviceResolver resolver; + const StubEngine engine(makeEngineWithKnobs({BLOCK_SIZE}), makeStubStateManager(), resolver); + + // Pins that construction produces a stable, non-zero id. + EXPECT_NE(engine.id(), 0); +} + +TEST(TestIngestorGenericEngine, IsApplicableTrueWhenTheStateManagerHasASurvivingKernel) +{ + const ScopedTestSymbols symbols; + const StubDeviceResolver resolver; + const StubEngine engine(makeEngineWithKnobs({BLOCK_SIZE}), makeStubStateManager(), resolver); + + StubHandle handle; + const TestGraph graph(makeGraphId(0x60)); + + EXPECT_TRUE(engine.isApplicable(handle, graph)); +} + +TEST(TestIngestorGenericEngine, IsApplicableFalseWhenNoMatcherAccepts) +{ + // A distinct symbol avoids colliding with ScopedTestSymbols' matcher running + // concurrently in another test. + constexpr const char* REJECT_SYMBOL = "hipdnn.kernel_ingestor.test.generic_engine.reject"; + GraphMatcherRegistry::registerSymbol(REJECT_SYMBOL, &rejectGraph); + ScoreRegistry::registerSymbol(SCORE_SYMBOL, &scoreByBlockSize); + + MetadataSchema schema; + schema.id = SCHEMA_ID; + schema.name = "test schema"; + schema.fields = {{BLOCK_SIZE, MetadataType::INT, MetadataValue{int64_t{64}}}, + {DTYPE, MetadataType::STRING, std::nullopt}}; + + KernelDescriptorPack pack; + pack.id = PACK_ID; + pack.name = "test pack"; + pack.matcherIds = {GRAPH_MATCHER_ID}; + pack.engineId = ENGINE_ID; + pack.dispatchId = DISPATCH_ID; + pack.kernels = {makeTestKernel(testId(0x64), "kernel_64_float", 64, "FLOAT")}; + + auto stateManager = std::make_unique>( + std::move(schema), + std::vector{ + {GRAPH_MATCHER_ID, "graph scoped", MatchScope::GRAPH, REJECT_SYMBOL}}, + std::vector{ + {DISPATCH_ID, "test dispatch", "hipdnn.kernel_ingestor.test.dispatch"}}, + std::vector{std::move(pack)}, + std::make_shared(SCORE_SYMBOL)); + + const StubDeviceResolver resolver; + const StubEngine engine(makeEngineWithKnobs({BLOCK_SIZE}), std::move(stateManager), resolver); + + StubHandle handle; + const TestGraph graph(makeGraphId(0x61)); + + EXPECT_FALSE(engine.isApplicable(handle, graph)); + + GraphMatcherRegistry::unregisterSymbol(REJECT_SYMBOL); + ScoreRegistry::unregisterSymbol(SCORE_SYMBOL); +} + +TEST(TestIngestorGenericEngine, GetDetailsReportsTheEnginesKnobs) +{ + const ScopedTestSymbols symbols; + const StubDeviceResolver resolver; + const StubEngine engine(makeEngineWithKnobs({BLOCK_SIZE}), makeStubStateManager(), resolver); + + StubHandle handle; + const TestGraph graph(makeGraphId(0x62)); + hipdnnPluginConstData_t details{}; + + engine.getDetails(handle, graph, details); + + ASSERT_NE(details.ptr, nullptr); + const hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineDetailsWrapper wrapper(details.ptr, + details.size); + ASSERT_TRUE(wrapper.isValid()); + EXPECT_EQ(wrapper.engineId(), engine.id()); + ASSERT_EQ(wrapper.knobCount(), 1U); + EXPECT_EQ(wrapper.getKnobByName(BLOCK_SIZE).knobId(), BLOCK_SIZE); +} + +TEST(TestIngestorGenericEngine, GetMaxWorkspaceSizeDelegatesToThePlanBuilder) +{ + // No dispatch symbol is registered, so getDispatchDetails() throws -- proving + // the call reached the plan builder rather than returning a stub zero. + const ScopedTestSymbols symbols; + const StubDeviceResolver resolver; + const StubEngine engine(makeEngineWithKnobs({BLOCK_SIZE}), makeStubStateManager(), resolver); + + const StubHandle handle; + const TestGraph graph(makeGraphId(0x63)); + const hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineConfigWrapper emptyConfig(nullptr, 0); + + EXPECT_THROW(engine.getMaxWorkspaceSize(handle, graph, emptyConfig), std::runtime_error); +} + +TEST(TestIngestorGenericEngine, InitializeExecutionContextDelegatesToThePlanBuilder) +{ + // No dispatch handler is registered, so buildPlan()'s lookup throws -- proving + // this reached the plan builder. + const ScopedTestSymbols symbols; + const StubDeviceResolver resolver; + const StubEngine engine(makeEngineWithKnobs({BLOCK_SIZE}), makeStubStateManager(), resolver); + + const StubHandle handle; + const TestGraph graph(makeGraphId(0x64)); + const hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineConfigWrapper emptyConfig(nullptr, 0); + StubContext context; + + EXPECT_THROW(engine.initializeExecutionContext(handle, graph, emptyConfig, context), + std::runtime_error); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/TestGenericPlan.cpp b/projects/hipdnn/plugin_sdk/tests/ingestor/TestGenericPlan.cpp new file mode 100644 index 000000000000..26a2dfe58d7f --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/TestGenericPlan.cpp @@ -0,0 +1,157 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "IngestorMocks.hpp" +#include "KernelIngestorTestFixtures.hpp" + +/** + * @file TestGenericPlan.cpp + * @brief Unit tests for GenericPlan.hpp: the plan every descriptor-backed engine + * produces, holding one selected kernel and its resolved dispatch handler. + */ +namespace +{ + +using namespace hipdnn_plugin_sdk::ingestor; +using namespace hipdnn_plugin_sdk::ingestor::testing; +using ::testing::_; +using ::testing::Return; + +KernelDispatcher makeDispatcher(const MockKernelDispatchHandler& handler) +{ + return {makeDefinition(testId(0x01), 64), &handler}; +} + +TEST(TestIngestorGenericPlan, ConstructorThrowsInternalErrorWhenPrepareReturnsNull) +{ + // prepare() returning null means dispatch resolved no launch for a kernel that + // applicability already accepted -- an internal inconsistency, not a caller error. + const MockKernelDispatchHandler handler; + EXPECT_CALL(handler, workspaceBytes(_, _, _)).WillOnce(Return(0)); + EXPECT_CALL(handler, prepare(_, _, _)).WillOnce(Return(nullptr)); + + const TestGraph graph; + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + const BoundTokens bound; + + try + { + const GenericPlan plan(makeDispatcher(handler), context, bound); + FAIL() << "expected HipdnnPluginException"; + } + catch(const hipdnn_plugin_sdk::HipdnnPluginException& ex) + { + EXPECT_EQ(ex.getStatus(), HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR); + } +} + +TEST(TestIngestorGenericPlan, GetWorkspaceSizeReportsWhatTheHandlerComputedAtConstruction) +{ + const MockKernelDispatchHandler handler; + EXPECT_CALL(handler, workspaceBytes(_, _, _)).WillOnce(Return(size_t{4096})); + EXPECT_CALL(handler, prepare(_, _, _)) + .WillOnce(Return(::testing::ByMove(std::make_unique()))); + + const TestGraph graph; + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + const BoundTokens bound; + + const GenericPlan plan(makeDispatcher(handler), context, bound); + const StubHandle handle; + + EXPECT_EQ(plan.getWorkspaceSize(handle), 4096U); +} + +TEST(TestIngestorGenericPlan, ExecuteThrowsInvalidValueWhenWorkspaceRequiredButNoneProvided) +{ + const MockKernelDispatchHandler handler; + EXPECT_CALL(handler, workspaceBytes(_, _, _)).WillOnce(Return(size_t{1024})); + EXPECT_CALL(handler, prepare(_, _, _)) + .WillOnce(Return(::testing::ByMove(std::make_unique()))); + // Workspace is missing, so launch() must never be reached. + EXPECT_CALL(handler, launch(_, _, _, _, _)).Times(0); + + const TestGraph graph; + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + const BoundTokens bound; + + const GenericPlan plan(makeDispatcher(handler), context, bound); + const StubHandle handle; + + try + { + plan.execute(handle, nullptr, 0, nullptr); + FAIL() << "expected HipdnnPluginException"; + } + catch(const hipdnn_plugin_sdk::HipdnnPluginException& ex) + { + EXPECT_EQ(ex.getStatus(), HIPDNN_PLUGIN_STATUS_INVALID_VALUE); + } +} + +TEST(TestIngestorGenericPlan, ExecuteForwardsDeviceBuffersAndWorkspaceToLaunchUnchanged) +{ + const MockKernelDispatchHandler handler; + EXPECT_CALL(handler, workspaceBytes(_, _, _)).WillOnce(Return(size_t{256})); + EXPECT_CALL(handler, prepare(_, _, _)) + .WillOnce(Return(::testing::ByMove(std::make_unique()))); + + const std::array buffers{ + {{/*uid=*/7, /*ptr=*/reinterpret_cast(0x1234)}}}; + int workspaceStorage = 0; + void* const workspace = &workspaceStorage; + + EXPECT_CALL(handler, launch(_, _, buffers.data(), 1U, workspace)).Times(1); + + const TestGraph graph; + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + const BoundTokens bound; + + const GenericPlan plan(makeDispatcher(handler), context, bound); + const StubHandle handle; + + plan.execute(handle, buffers.data(), 1, workspace); +} + +TEST(TestIngestorGenericPlan, KernelReturnsTheDispatchersSelectedKernel) +{ + const MockKernelDispatchHandler handler; + EXPECT_CALL(handler, workspaceBytes(_, _, _)).WillOnce(Return(0)); + EXPECT_CALL(handler, prepare(_, _, _)) + .WillOnce(Return(::testing::ByMove(std::make_unique()))); + + const TestGraph graph; + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + const BoundTokens bound; + const auto kernelId = testId(0x77); + + const GenericPlan plan({makeDefinition(kernelId, 64), &handler}, context, bound); + + EXPECT_EQ(plan.kernel().kernelId, kernelId); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/TestGenericPlanBuilder.cpp b/projects/hipdnn/plugin_sdk/tests/ingestor/TestGenericPlanBuilder.cpp new file mode 100644 index 000000000000..d6acf76699ba --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/TestGenericPlanBuilder.cpp @@ -0,0 +1,513 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "IngestorMocks.hpp" +#include "KernelIngestorTestFixtures.hpp" + +/** + * @file TestGenericPlanBuilder.cpp + * @brief Unit tests for GenericPlanBuilder.hpp: applicability, workspace sizing, plan + * building, knob filtering, and per-handle device resolution. + */ +namespace +{ + +using namespace hipdnn_plugin_sdk::ingestor; +using namespace hipdnn_plugin_sdk::ingestor::testing; +using ::testing::_; +using ::testing::Ref; +using ::testing::Return; +using ::testing::ReturnRef; + +// --------------------------------------------------------------------------- +// isApplicable / getMaxWorkspaceSize / buildPlan / getCustomKnobs, using a +// two-survivor catalog (TestHandle = int). +// --------------------------------------------------------------------------- + +/// Workspace requirement equals the kernel's block_size, so getMaxWorkspaceSize can assert +/// a real max instead of a shared constant. +class WorkspaceEqualsBlockSizeHandler : public IKernelDispatchHandler +{ +public: + size_t workspaceBytes(const MatchContext& /*context*/, + const BoundTokens& /*bound*/, + const KernelDefinition& kernel) const override + { + return static_cast(kernel.getIntMetadata(BLOCK_SIZE)); + } + + std::unique_ptr prepare(const MatchContext& /*context*/, + const BoundTokens& /*bound*/, + const KernelDefinition& /*kernel*/) const override + { + return std::make_unique(); + } + + void launch(const TestHandle& /*handle*/, + const PreparedDispatch& /*prepared*/, + const hipdnnPluginDeviceBuffer_t* /*deviceBuffers*/, + uint32_t /*numDeviceBuffers*/, + void* /*workspace*/) const override + { + } +}; + +struct KnobFilterSettings +{ + KnobFilter ingestorKnobFilter; +}; + +struct KnobFilterContext +{ + void setExecutionSettings(const KnobFilterSettings& /*settings*/) {} + + void setPlan(std::unique_ptr> plan) + { + _plan = std::move(plan); + } + + /// Safe without RTTI: setPlan() only ever receives a GenericPlan here. + const GenericPlan& plan() const + { + return static_cast&>(*_plan); + } + +private: + std::unique_ptr> _plan; +}; + +using TestPlanBuilder = GenericPlanBuilder; + +hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineConfigWrapper + makeEmptyEngineConfig(flatbuffers::FlatBufferBuilder& builder) +{ + builder.Finish( + hipdnn_flatbuffers_sdk::data_objects::CreateEngineConfig(builder, ENGINE_ID.front())); + return hipdnn_flatbuffers_sdk::flatbuffer_utilities::EngineConfigWrapper( + builder.GetBufferPointer(), builder.GetSize()); +} + +TEST(TestIngestorGenericPlanBuilder, IsApplicableTrueWhenTheCatalogHasASurvivor) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + const TestGraph graph(makeGraphId(0x90)); + + EXPECT_TRUE(builder.isApplicable(0, graph)); +} + +TEST(TestIngestorGenericPlanBuilder, IsApplicableFalseWhenTheGraphMatcherRejects) +{ + const ScopedSymbols symbols("test.graph", rejectGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + const TestGraph graph(makeGraphId(0x91)); + + EXPECT_FALSE(builder.isApplicable(0, graph)); +} + +TEST(TestIngestorGenericPlanBuilder, GetMaxWorkspaceSizeTakesTheMaxAcrossSurvivors) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const WorkspaceEqualsBlockSizeHandler handler; + const ScopedDispatchRegistration dispatch("test.dispatch", handler); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + const TestGraph graph(makeGraphId(0x92)); + // Two survivors (block_size 64, 256): expects the max, not the first entry or a sum. + EXPECT_EQ(builder.getMaxWorkspaceSize(0, graph, KnobFilterSettings{}), 256U); +} + +TEST(TestIngestorGenericPlanBuilder, BuildPlanThrowsInternalErrorOnAnEmptyRankedCatalog) +{ + // Applicability accepted a graph with no matching kernel at all (not a knob issue); + // this must carry a different status/message than the knob-filtering case below. + const ScopedSymbols symbols("test.graph", rejectGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + flatbuffers::FlatBufferBuilder fbb; + const auto engineConfig = makeEmptyEngineConfig(fbb); + const TestGraph graph(makeGraphId(0x93)); + KnobFilterContext context; + + try + { + builder.buildPlan(0, graph, engineConfig, context); + FAIL() << "expected HipdnnPluginException"; + } + catch(const hipdnn_plugin_sdk::HipdnnPluginException& ex) + { + EXPECT_EQ(ex.getStatus(), HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR); + EXPECT_EQ(ex.getMessage(), + "engine '" + engine.name + "' accepted this graph but has no applicable kernel"); + } +} + +TEST(TestIngestorGenericPlanBuilder, GetMaxWorkspaceSizeThrowsInternalErrorOnAnEmptyCatalog) +{ + // Must agree with buildPlan()'s empty-catalog case above, not silently return 0. + const ScopedSymbols symbols("test.graph", rejectGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + const TestGraph graph(makeGraphId(0x9A)); + + try + { + builder.getMaxWorkspaceSize(0, graph, KnobFilterSettings{}); + FAIL() << "expected HipdnnPluginException"; + } + catch(const hipdnn_plugin_sdk::HipdnnPluginException& ex) + { + EXPECT_EQ(ex.getStatus(), HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR); + EXPECT_EQ(ex.getMessage(), + "engine '" + engine.name + "' accepted this graph but has no applicable kernel"); + } +} + +TEST(TestIngestorGenericPlanBuilder, GetCustomKnobsReportsMinMaxStepAndRankedDefault) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + const TestGraph graph(makeGraphId(0x94)); + const auto knobs = builder.getCustomKnobs(0, graph); + + ASSERT_EQ(knobs.size(), 1U); + const auto& knob = knobs.front(); + EXPECT_EQ(knob.knob_id, BLOCK_SIZE); + ASSERT_TRUE(knob.constraint.AsIntConstraint() != nullptr); + const auto& constraint = *knob.constraint.AsIntConstraint(); + EXPECT_EQ(constraint.min_value, 64); + EXPECT_EQ(constraint.max_value, 256); + EXPECT_EQ(constraint.step, 1); + ASSERT_TRUE(knob.default_value.AsIntValue() != nullptr); + // The heuristic ranks 256 first, so leaving the knob alone must reproduce that pick. + EXPECT_EQ(knob.default_value.AsIntValue()->value, 256); +} + +TEST(TestIngestorGenericPlanBuilder, GetCustomKnobsSkipsFieldsWithNoIntegerValues) +{ + // dtype takes only string values, so it can never become an integer knob and must be + // skipped. + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE, DTYPE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + const TestGraph graph(makeGraphId(0x95)); + const auto knobs = builder.getCustomKnobs(0, graph); + + ASSERT_EQ(knobs.size(), 1U); + EXPECT_EQ(knobs.front().knob_id, BLOCK_SIZE); +} + +// --------------------------------------------------------------------------- +// Knob filtering: setting a knob must actually restrict kernel selection. +// --------------------------------------------------------------------------- + +TEST(TestIngestorGenericPlanBuilder, HonorsAnExplicitKnobSettingOverTheHeuristicDefault) +{ + // The heuristic would pick 256 by default; setting block_size=64 must override it. + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const WorkspaceEqualsBlockSizeHandler handler; + const ScopedDispatchRegistration dispatch("test.dispatch", handler); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + flatbuffers::FlatBufferBuilder fbb; + const auto engineConfig = makeIntKnobEngineConfig(fbb, BLOCK_SIZE, 64); + + const TestGraph graph(makeGraphId(30)); + KnobFilterSettings settings; + builder.initializeExecutionSettings(0, graph, engineConfig, settings); + + KnobFilterContext context; + builder.buildPlan(0, graph, engineConfig, context); + + EXPECT_EQ(context.plan().kernel().getIntMetadata(BLOCK_SIZE), 64); +} + +TEST(TestIngestorGenericPlanBuilder, UnsatisfiableKnobValueThrowsInvalidValueNamingItAndTheValue) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const WorkspaceEqualsBlockSizeHandler handler; + const ScopedDispatchRegistration dispatch("test.dispatch", handler); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + // No surviving kernel carries block_size 999: the catalog matches (two FLOAT + // kernels), but knob filtering excludes both. + flatbuffers::FlatBufferBuilder fbb; + const auto engineConfig = makeIntKnobEngineConfig(fbb, BLOCK_SIZE, 999); + + const TestGraph graph(makeGraphId(31)); + KnobFilterContext context; + + try + { + builder.buildPlan(0, graph, engineConfig, context); + FAIL() << "expected HipdnnPluginException"; + } + catch(const hipdnn_plugin_sdk::HipdnnPluginException& ex) + { + EXPECT_EQ(ex.getStatus(), HIPDNN_PLUGIN_STATUS_INVALID_VALUE); + EXPECT_NE(ex.getMessage().find(BLOCK_SIZE), std::string::npos); + EXPECT_NE(ex.getMessage().find("999"), std::string::npos); + // Survivor count distinguishes "knobs excluded everything" from "graph matched nothing". + EXPECT_NE(ex.getMessage().find("2 kernel(s) matched the graph before knob filtering"), + std::string::npos); + } +} + +TEST(TestIngestorGenericPlanBuilder, GetMaxWorkspaceSizeHonorsTheSameKnobFilterAsBuildPlan) +{ + // getMaxWorkspaceSize and buildPlan must apply the same knob filter: block_size=64 + // caps the max at 64, not the unfiltered catalog's 256. + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const WorkspaceEqualsBlockSizeHandler handler; + const ScopedDispatchRegistration dispatch("test.dispatch", handler); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + flatbuffers::FlatBufferBuilder fbb; + const auto engineConfig = makeIntKnobEngineConfig(fbb, BLOCK_SIZE, 64); + const TestGraph graph(makeGraphId(0x96)); + + KnobFilterSettings settings; + builder.initializeExecutionSettings(0, graph, engineConfig, settings); + + EXPECT_EQ(builder.getMaxWorkspaceSize(0, graph, settings), 64U); +} + +TEST(TestIngestorGenericPlanBuilder, + GetMaxWorkspaceSizeThrowsInvalidValueWhenTheFilterIsUnsatisfiable) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const WorkspaceEqualsBlockSizeHandler handler; + const ScopedDispatchRegistration dispatch("test.dispatch", handler); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + flatbuffers::FlatBufferBuilder fbb; + const auto engineConfig = makeIntKnobEngineConfig(fbb, BLOCK_SIZE, 999); + const TestGraph graph(makeGraphId(0x97)); + + KnobFilterSettings settings; + builder.initializeExecutionSettings(0, graph, engineConfig, settings); + + EXPECT_THROW(builder.getMaxWorkspaceSize(0, graph, settings), + hipdnn_plugin_sdk::HipdnnPluginException); +} + +TEST(TestIngestorGenericPlanBuilder, + EmptyCatalogBeforeFilteringStillThrowsInternalErrorWithItsOriginalMessage) +{ + const ScopedSymbols symbols("test.graph", rejectGraph, "test.kernel", countingFloatKernels); + const WorkspaceEqualsBlockSizeHandler handler; + const ScopedDispatchRegistration dispatch("test.dispatch", handler); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + flatbuffers::FlatBufferBuilder fbb; + // An empty engine config exercises the no-kernels-at-all path, not knob filtering. + const auto engineConfig = makeEmptyEngineConfig(fbb); + + const TestGraph graph(makeGraphId(32)); + KnobFilterContext context; + + try + { + builder.buildPlan(0, graph, engineConfig, context); + FAIL() << "expected HipdnnPluginException"; + } + catch(const hipdnn_plugin_sdk::HipdnnPluginException& ex) + { + EXPECT_EQ(ex.getStatus(), HIPDNN_PLUGIN_STATUS_INTERNAL_ERROR); + EXPECT_EQ(ex.getMessage(), + "engine '" + engine.name + "' accepted this graph but has no applicable kernel"); + } +} + +// --------------------------------------------------------------------------- +// readKnobFilter(): rejecting a non-integer setting for an engine-exposed knob. +// --------------------------------------------------------------------------- + +TEST(TestIngestorGenericPlanBuilder, InitializeExecutionSettingsRejectsAFloatValuedKnobSetting) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + flatbuffers::FlatBufferBuilder fbb; + const auto engineConfig = makeFloatKnobEngineConfig(fbb, BLOCK_SIZE, 64.0); + const TestGraph graph(makeGraphId(0x9B)); + KnobFilterSettings settings; + + try + { + builder.initializeExecutionSettings(0, graph, engineConfig, settings); + FAIL() << "expected HipdnnPluginException"; + } + catch(const hipdnn_plugin_sdk::HipdnnPluginException& ex) + { + EXPECT_EQ(ex.getStatus(), HIPDNN_PLUGIN_STATUS_INVALID_VALUE); + EXPECT_EQ(ex.getMessage(), + "engine '" + engine.name + "' knob '" + BLOCK_SIZE + + "' must be set to an integer value"); + } +} + +TEST(TestIngestorGenericPlanBuilder, InitializeExecutionSettingsRejectsAStringValuedKnobSetting) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const TestDeviceResolver resolver; + const TestPlanBuilder builder(engine, *manager, resolver); + + flatbuffers::FlatBufferBuilder fbb; + const auto engineConfig = makeStringKnobEngineConfig(fbb, BLOCK_SIZE, "fast"); + const TestGraph graph(makeGraphId(0x9C)); + KnobFilterSettings settings; + + try + { + builder.initializeExecutionSettings(0, graph, engineConfig, settings); + FAIL() << "expected HipdnnPluginException"; + } + catch(const hipdnn_plugin_sdk::HipdnnPluginException& ex) + { + EXPECT_EQ(ex.getStatus(), HIPDNN_PLUGIN_STATUS_INVALID_VALUE); + EXPECT_EQ(ex.getMessage(), + "engine '" + engine.name + "' knob '" + BLOCK_SIZE + + "' must be set to an integer value"); + } +} + +// --------------------------------------------------------------------------- +// contextFor(): a mocked device resolver proves per-handle device resolution folds into +// the MatchContext (uses StubHandle/StubSettings/StubContext to match IngestorMocks.hpp). +// --------------------------------------------------------------------------- + +constexpr DeviceId DEVICE_FOR_HANDLE_A = 42; +constexpr DeviceId DEVICE_FOR_HANDLE_B = 7; +constexpr const char* DEVICE_GATED_MATCH_SYMBOL + = "hipdnn.kernel_ingestor.test.generic_plan_builder.device_gated"; + +bool acceptsOnlyDeviceA(const MatchContext& context, BoundTokens& /*bound*/) +{ + return context.deviceId == DEVICE_FOR_HANDLE_A; +} + +TEST(TestIngestorGenericPlanBuilder, ContextForFoldsPerHandleDeviceResolutionIntoTheMatchContext) +{ + GraphMatcherRegistry::registerSymbol(DEVICE_GATED_MATCH_SYMBOL, &acceptsOnlyDeviceA); + ScoreRegistry::registerSymbol(SCORE_SYMBOL, &scoreByBlockSize); + + MetadataSchema schema; + schema.id = SCHEMA_ID; + schema.name = "test schema"; + schema.fields = {{BLOCK_SIZE, MetadataType::INT, MetadataValue{int64_t{64}}}, + {DTYPE, MetadataType::STRING, std::nullopt}}; + + KernelDescriptorPack pack; + pack.id = PACK_ID; + pack.name = "test pack"; + pack.matcherIds = {GRAPH_MATCHER_ID}; + pack.engineId = ENGINE_ID; + pack.dispatchId = DISPATCH_ID; + pack.kernels = {makeTestKernel(testId(0x64), "kernel_64_float", 64, "FLOAT")}; + + const KernelIngestorStateManager manager( + std::move(schema), + std::vector{ + {GRAPH_MATCHER_ID, "device gated", MatchScope::GRAPH, DEVICE_GATED_MATCH_SYMBOL}}, + std::vector{ + {DISPATCH_ID, "test dispatch", "hipdnn.kernel_ingestor.test.dispatch"}}, + std::vector{std::move(pack)}, + std::make_shared(SCORE_SYMBOL)); + + const auto engine = makeEngineWithKnobs({BLOCK_SIZE}); + const MockDeviceResolver resolver; + const auto properties = testDeviceProperties(); + const StubHandle handleA; + const StubHandle handleB; + + EXPECT_CALL(resolver, deviceId(Ref(handleA))).WillRepeatedly(Return(DEVICE_FOR_HANDLE_A)); + EXPECT_CALL(resolver, deviceId(Ref(handleB))).WillRepeatedly(Return(DEVICE_FOR_HANDLE_B)); + EXPECT_CALL(resolver, deviceProperties(_)).WillRepeatedly(ReturnRef(properties)); + + const GenericPlanBuilder builder( + engine, manager, resolver); + const TestGraph graph(makeGraphId(0x98)); + + // Same builder and graph, two handles differing only in resolved device: isApplicable + // can differ only if contextFor() asks the resolver per handle. + EXPECT_TRUE(builder.isApplicable(handleA, graph)); + EXPECT_FALSE(builder.isApplicable(handleB, graph)); + + GraphMatcherRegistry::unregisterSymbol(DEVICE_GATED_MATCH_SYMBOL); + ScoreRegistry::unregisterSymbol(SCORE_SYMBOL); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/TestKernelDefinition.cpp b/projects/hipdnn/plugin_sdk/tests/ingestor/TestKernelDefinition.cpp new file mode 100644 index 000000000000..77b609287908 --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/TestKernelDefinition.cpp @@ -0,0 +1,163 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "KernelIngestorTestFixtures.hpp" + +/** + * @file TestKernelDefinition.cpp + * @brief Tests for KernelDefinition.hpp: tryGetMetadata()'s optional contract, and the + * typed getters' happy path and throw branches. + */ +namespace +{ + +using namespace hipdnn_plugin_sdk::ingestor; +using namespace hipdnn_plugin_sdk::ingestor::testing; + +KernelDefinition makeKernelWithMetadata(MetadataValues metadata) +{ + return {testId(0x01), + PACK_ID, + DISPATCH_ID, + KernelSource{KernelSourceKind::EMBEDDED_SOURCE, "Test.cpp", "TestKernel"}, + std::move(metadata), + /*priority=*/0}; +} + +// --------------------------------------------------------------------------- +// tryGetMetadata +// --------------------------------------------------------------------------- + +TEST(TestIngestorKernelDefinition, TryGetMetadataReturnsTheValueWhenPresent) +{ + const auto kernel = makeKernelWithMetadata({{BLOCK_SIZE, MetadataValue{int64_t{64}}}}); + + const auto value = kernel.tryGetMetadata(BLOCK_SIZE); + + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(std::get(*value), 64); +} + +TEST(TestIngestorKernelDefinition, TryGetMetadataReturnsNulloptWhenAbsent) +{ + // Nullopt rather than throwing: lets callers query fields an older kernel predates. + const auto kernel = makeKernelWithMetadata({}); + + EXPECT_EQ(kernel.tryGetMetadata(BLOCK_SIZE), std::nullopt); +} + +// Typed accessor happy path. + +TEST(TestIngestorKernelDefinition, GetIntMetadataReturnsTheIntegerValue) +{ + const auto kernel = makeKernelWithMetadata({{BLOCK_SIZE, MetadataValue{int64_t{256}}}}); + + EXPECT_EQ(kernel.getIntMetadata(BLOCK_SIZE), 256); +} + +TEST(TestIngestorKernelDefinition, GetStringMetadataReturnsTheStringValue) +{ + const auto kernel = makeKernelWithMetadata({{DTYPE, MetadataValue{std::string{"FLOAT"}}}}); + + EXPECT_EQ(kernel.getStringMetadata(DTYPE), "FLOAT"); +} + +TEST(TestIngestorKernelDefinition, GetIntListMetadataReturnsTheListValue) +{ + constexpr const char* STRIDE_ORDER = "stride_order"; + const auto kernel + = makeKernelWithMetadata({{STRIDE_ORDER, MetadataValue{std::vector{3, 1, 2, 0}}}}); + + EXPECT_EQ(kernel.getIntListMetadata(STRIDE_ORDER), (std::vector{3, 1, 2, 0})); +} + +// Throw matrix: each typed getter x "field absent" (out_of_range) / "wrong alternative" +// (invalid_argument). + +struct KernelDefinitionThrowCase +{ + std::string name; + /// Builds the kernel this case exercises. + std::function makeKernel; + /// Invokes the getter under test. + std::function callGetter; + /// True for std::out_of_range (field absent); false for std::invalid_argument (wrong + /// alternative). RTTI is disabled, so distinguished via EXPECT_THROW's static type. + bool expectsOutOfRange; +}; + +class TestIngestorKernelDefinitionThrowMatrix + : public ::testing::TestWithParam +{ +}; + +TEST_P(TestIngestorKernelDefinitionThrowMatrix, ThrowsOnTheExpectedFailureMode) +{ + const auto& testCase = GetParam(); + const auto kernel = testCase.makeKernel(); + + if(testCase.expectsOutOfRange) + { + EXPECT_THROW(testCase.callGetter(kernel), std::out_of_range); + } + else + { + EXPECT_THROW(testCase.callGetter(kernel), std::invalid_argument); + } +} + +INSTANTIATE_TEST_SUITE_P( + EveryAccessorTimesEveryFailureMode, + TestIngestorKernelDefinitionThrowMatrix, + ::testing::Values( + KernelDefinitionThrowCase{ + "GetIntMetadataAbsentFieldThrowsOutOfRange", + [] { return makeKernelWithMetadata({}); }, + [](const KernelDefinition& kernel) { kernel.getIntMetadata(BLOCK_SIZE); }, + /*expectsOutOfRange=*/true}, + KernelDefinitionThrowCase{ + "GetIntMetadataWrongAlternativeThrowsInvalidArgument", + [] { return makeKernelWithMetadata({{BLOCK_SIZE, MetadataValue{std::string{"64"}}}}); }, + [](const KernelDefinition& kernel) { kernel.getIntMetadata(BLOCK_SIZE); }, + /*expectsOutOfRange=*/false}, + KernelDefinitionThrowCase{ + "GetStringMetadataAbsentFieldThrowsOutOfRange", + [] { return makeKernelWithMetadata({}); }, + [](const KernelDefinition& kernel) { kernel.getStringMetadata(DTYPE); }, + /*expectsOutOfRange=*/true}, + KernelDefinitionThrowCase{ + "GetStringMetadataWrongAlternativeThrowsInvalidArgument", + [] { return makeKernelWithMetadata({{DTYPE, MetadataValue{int64_t{1}}}}); }, + [](const KernelDefinition& kernel) { kernel.getStringMetadata(DTYPE); }, + /*expectsOutOfRange=*/false}, + KernelDefinitionThrowCase{ + "GetIntListMetadataAbsentFieldThrowsOutOfRange", + [] { return makeKernelWithMetadata({}); }, + [](const KernelDefinition& kernel) { kernel.getIntListMetadata("stride_order"); }, + /*expectsOutOfRange=*/true}, + KernelDefinitionThrowCase{ + "GetIntListMetadataWrongAlternativeThrowsInvalidArgument", + [] { return makeKernelWithMetadata({{"stride_order", MetadataValue{int64_t{1}}}}); }, + [](const KernelDefinition& kernel) { kernel.getIntListMetadata("stride_order"); }, + /*expectsOutOfRange=*/false}), + [](const ::testing::TestParamInfo& info) { + return info.param.name; + }); + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/TestKernelHeuristic.cpp b/projects/hipdnn/plugin_sdk/tests/ingestor/TestKernelHeuristic.cpp new file mode 100644 index 000000000000..fb7b6a2d690f --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/TestKernelHeuristic.cpp @@ -0,0 +1,139 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include "KernelIngestorTestFixtures.hpp" + +/** + * @file TestKernelHeuristic.cpp + * @brief Tests for IKernelHeuristic.hpp: lazy symbol resolution, ranking/tie-break + * order, and the makeKernelHeuristic() factory. + */ +namespace +{ + +using namespace hipdnn_plugin_sdk::ingestor; +using namespace hipdnn_plugin_sdk::ingestor::testing; + +TEST(TestIngestorKernelHeuristic, ResolvesItsSymbolOnFirstUseRatherThanAtConstruction) +{ + // RFC 0017 §8.1/§3: constructing against an unregistered symbol must be legal; + // only scoring may fail. An engine whose matchers reject a graph never pays for it. + const NativeKernelHeuristic heuristic("hipdnn.kernel_ingestor.test.not_yet_registered"); + + const TestGraph graph; + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + const auto kernel = makeDefinition(testId(0x01), 64); + + EXPECT_THROW(heuristic.score(kernel, context), std::runtime_error); +} + +TEST(TestIngestorKernelHeuristic, RanksHigherScoringKernelsFirst) +{ + const ScopedTestSymbols symbols; + const TestGraph graph; + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + + Catalog catalog; + const auto lowId = testId(0x01); + const auto highId = testId(0x02); + catalog.entries = {makeDefinition(lowId, 64), makeDefinition(highId, 256)}; + + const NativeKernelHeuristic heuristic(SCORE_SYMBOL); + const auto ranked = heuristic.rank(catalog, context); + + ASSERT_EQ(ranked.size(), 2U); + EXPECT_EQ(ranked.front().kernelId, highId); +} + +TEST(TestIngestorKernelHeuristic, BreaksScoreTiesOnPriority) +{ + const ScopedConstantScore constantScore; + const TestGraph graph; + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + + Catalog catalog; + const auto lowPriorityId = testId(0x01); + const auto highPriorityId = testId(0x02); + catalog.entries = {makeDefinition(lowPriorityId, 64, 1), makeDefinition(highPriorityId, 64, 5)}; + + const NativeKernelHeuristic heuristic(CONSTANT_SCORE_SYMBOL); + const auto ranked = heuristic.rank(catalog, context); + + ASSERT_EQ(ranked.size(), 2U); + EXPECT_EQ(ranked.front().kernelId, highPriorityId); +} + +TEST(TestIngestorKernelHeuristic, BreaksRemainingTiesOnKernelIdForStabilityAcrossRuns) +{ + const ScopedConstantScore constantScore; + const TestGraph graph; + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + + Catalog catalog; + // Inserted highest-id first: result must not depend on load order. + const auto lowerId = testId(0x01); + const auto higherId = testId(0x02); + catalog.entries = {makeDefinition(higherId, 64), makeDefinition(lowerId, 64)}; + + const NativeKernelHeuristic heuristic(CONSTANT_SCORE_SYMBOL); + const auto ranked = heuristic.rank(catalog, context); + + ASSERT_EQ(ranked.size(), 2U); + EXPECT_EQ(ranked.front().kernelId, lowerId); +} + +// makeKernelHeuristic(): the UHD -> IKernelHeuristic adapter factory. + +TEST(TestIngestorKernelHeuristic, MakeKernelHeuristicBuildsANativeHeuristicForNativeKind) +{ + const ScopedTestSymbols symbols; + + HeuristicDescriptor descriptor; + descriptor.id = HEURISTIC_ID; + descriptor.name = "test heuristic"; + descriptor.kind = HeuristicKind::NATIVE; + descriptor.payload = SCORE_SYMBOL; + + const auto heuristic = makeKernelHeuristic(descriptor); + + ASSERT_NE(heuristic, nullptr); + // Confirms the factory wired the payload symbol through. + const TestGraph graph; + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + EXPECT_EQ(heuristic->score(makeDefinition(testId(0x01), 128), context), 128.0); +} + +TEST(TestIngestorKernelHeuristic, MakeKernelHeuristicThrowsForAKindWithNoAdapter) +{ + // HeuristicKind::MODEL has no adapter yet (RFC 0017's UHD follow-up); fails at + // descriptor-assembly time, not at first rank(). + HeuristicDescriptor descriptor; + descriptor.id = HEURISTIC_ID; + descriptor.name = "model heuristic"; + descriptor.kind = HeuristicKind::MODEL; + descriptor.payload = "some/model/artifact.bin"; + + EXPECT_THROW(makeKernelHeuristic(descriptor), std::invalid_argument); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/TestKernelIngestorStateManager.cpp b/projects/hipdnn/plugin_sdk/tests/ingestor/TestKernelIngestorStateManager.cpp new file mode 100644 index 000000000000..fbbc8975e5bf --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/TestKernelIngestorStateManager.cpp @@ -0,0 +1,674 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#include "KernelIngestorTestFixtures.hpp" + +/** + * @file TestKernelIngestorStateManager.cpp + * @brief Unit tests for KernelIngestorStateManager: matching, pruning, caching, ranking, + * knob discovery, metadata completion, and construction-time validation. + */ +namespace +{ + +using namespace hipdnn_plugin_sdk::ingestor; +using namespace hipdnn_plugin_sdk::ingestor::testing; + +/// Binds "test.bound_token" to a value that disagrees with acceptGraph's. +inline bool bindConflictingTokenValue(const MatchContext& /*context*/, BoundTokens& bound) +{ + bound["test.bound_token"] = BOUND_TOKEN_VALUE + 1; + return true; +} + +/// Binds "test.bound_token" to acceptGraph's value, via a different matcher id. +inline bool bindAgreeingTokenValue(const MatchContext& /*context*/, BoundTokens& bound) +{ + bound["test.bound_token"] = BOUND_TOKEN_VALUE; + return true; +} + +// --------------------------------------------------------------------------- +// Matching and pruning +// --------------------------------------------------------------------------- + +TEST(TestKernelIngestorStateManager, KernelLevelMatcherPrunesTheCatalog) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const TestGraph graph(makeGraphId(1)); + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + + const auto definitions = manager->unsortedDefinitions(context); + + // Three kernels in the pack; the HALF one does not survive. + ASSERT_EQ(definitions.size(), 2U); + for(const auto& definition : definitions) + { + EXPECT_EQ(definition.getStringMetadata(DTYPE), "FLOAT"); + } +} + +TEST(TestKernelIngestorStateManager, GraphLevelMatcherFailurePrunesTheWholePack) +{ + const ScopedSymbols symbols("test.graph", rejectGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const TestGraph graph(makeGraphId(2)); + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + + EXPECT_TRUE(manager->unsortedDefinitions(context).empty()); + // A rejected graph costs one matcher call, not one per kernel. + EXPECT_EQ(counters().graphCalls, 1); + EXPECT_EQ(counters().kernelCalls, 0); +} + +TEST(TestKernelIngestorStateManager, MatchesOncePerGraphAndDevice) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const TestGraph graph(makeGraphId(3)); + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + + manager->unsortedDefinitions(context); + manager->unsortedDefinitions(context); + manager->unsortedDefinitions(context); + + EXPECT_EQ(counters().graphCalls, 1); + // One call per kernel in the pack, on the single uncached pass. + EXPECT_EQ(counters().kernelCalls, 3); +} + +TEST(TestKernelIngestorStateManager, EvaluatesASharedGraphMatcherOncePerGraphNotOncePerPack) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + + // Matchers are shared by id, so the graph-scoped check runs once, not once per pack. + // The first pack also fails a second, unshared matcher, so only the second pack + // survives, isolating binding reuse (read from the memo) from re-evaluation. + constexpr const char* UNSHARED_FAIL_SYMBOL = "test.cross_pack_binding_reuse.fail"; + GraphMatcherRegistry::registerSymbol(UNSHARED_FAIL_SYMBOL, rejectGraph); + const auto unsharedFailMatcherId = testId(0x84); + + const KernelDescriptorPack first = makePack({GRAPH_MATCHER_ID, unsharedFailMatcherId}); + KernelDescriptorPack second = makePack({GRAPH_MATCHER_ID, KERNEL_MATCHER_ID}); + second.id = testId(0x80); + // Distinct metadata tuple: uniqueness is engine-wide, not per pack. + second.kernels = {makeKernel(testId(0x81), "second_pack_kernel", 512, "FLOAT")}; + + const StateManager manager( + makeSchema(), + {{GRAPH_MATCHER_ID, "graph scoped", MatchScope::GRAPH, "test.graph"}, + {unsharedFailMatcherId, "always fails", MatchScope::GRAPH, UNSHARED_FAIL_SYMBOL}, + {KERNEL_MATCHER_ID, "kernel scoped", MatchScope::KERNEL, "test.kernel"}}, + makeTestDispatches(), + {first, second}, + std::make_shared(SCORE_SYMBOL)); + + const TestGraph graph(makeGraphId(20)); + const auto properties = testDeviceProperties(); + const auto catalog = manager.unsortedCatalog(MatchContext{graph, 0, properties}); + + // Two calls: the shared matcher once, plus the first pack's unshared failing matcher. + EXPECT_EQ(counters().graphCalls, 2); + // Only the second pack's kernel survives; the first pack was pruned before its + // kernels ran. + ASSERT_EQ(catalog.entries.size(), 1U); + EXPECT_EQ(catalog.entries.front().packId, second.id); + // The binding reaches catalog.bound via the second pack's memo read, not the + // pruned first pack that actually ran the matcher. + EXPECT_EQ(catalog.bound.count("test.bound_token"), 1U); + + GraphMatcherRegistry::unregisterSymbol(UNSHARED_FAIL_SYMBOL); +} + +TEST(TestKernelIngestorStateManager, ASharedGraphMatcherFailurePrunesEveryPackListingIt) +{ + const ScopedSymbols symbols("test.graph", rejectGraph, "test.kernel", countingFloatKernels); + + auto first = makePack({GRAPH_MATCHER_ID, KERNEL_MATCHER_ID}); + auto second = makePack({GRAPH_MATCHER_ID, KERNEL_MATCHER_ID}); + second.id = testId(0x82); + second.kernels = {makeKernel(testId(0x83), "second_pack_kernel", 512, "FLOAT")}; + + const StateManager manager(makeSchema(), + makeTestMatchers(), + makeTestDispatches(), + {first, second}, + std::make_shared(SCORE_SYMBOL)); + + const TestGraph graph(makeGraphId(21)); + const auto properties = testDeviceProperties(); + + EXPECT_TRUE(manager.unsortedDefinitions(MatchContext{graph, 0, properties}).empty()); + // One failing evaluation disqualifies both packs, and no per-kernel work runs at all. + EXPECT_EQ(counters().graphCalls, 1); + EXPECT_EQ(counters().kernelCalls, 0); +} + +TEST(TestKernelIngestorStateManager, PrunedPackBindingsAreNotVisibleToSurvivingPackKernels) +{ + // Two packs share an engine. The pruned pack's own matcher binds a token before a + // second matcher in the same pack fails and prunes it; the surviving pack lists + // neither matcher, so it must not see that binding. + constexpr const char* PASS_NO_BIND_SYMBOL = "test.pruned_bound_isolation.pass_no_bind"; + constexpr const char* LEAK_THEN_PRUNE_SYMBOL = "test.pruned_bound_isolation.leak"; + constexpr const char* ALWAYS_FAILS_SYMBOL = "test.pruned_bound_isolation.fail"; + + // acceptAnyGraph passes and binds nothing; acceptGraph passes and binds + // "test.bound_token"; rejectGraph always fails. + GraphMatcherRegistry::registerSymbol(PASS_NO_BIND_SYMBOL, &acceptAnyGraph); + GraphMatcherRegistry::registerSymbol(LEAK_THEN_PRUNE_SYMBOL, &acceptGraph); + GraphMatcherRegistry::registerSymbol(ALWAYS_FAILS_SYMBOL, &rejectGraph); + counters().reset(); + + const auto passNoBindMatcherId = testId(0xA4); + const auto leakMatcherId = testId(0xA5); + const auto failMatcherId = testId(0xA6); + + KernelDescriptorPack survivingPack; + survivingPack.id = testId(0xA0); + survivingPack.name = "surviving pack"; + survivingPack.matcherIds = {passNoBindMatcherId}; + survivingPack.engineId = ENGINE_ID; + survivingPack.dispatchId = DISPATCH_ID; + survivingPack.kernels = {makeKernel(testId(0xA1), "surviving_kernel", 64, "FLOAT")}; + + KernelDescriptorPack prunedPack; + prunedPack.id = testId(0xA2); + prunedPack.name = "pruned pack"; + // Order matters: the leaking matcher must run before the failing one prunes the + // pack (matchers run in listed order). + prunedPack.matcherIds = {leakMatcherId, failMatcherId}; + prunedPack.engineId = ENGINE_ID; + prunedPack.dispatchId = DISPATCH_ID; + prunedPack.kernels = {makeKernel(testId(0xA3), "pruned_kernel", 128, "FLOAT")}; + + const StateManager manager( + makeSchema(), + {{passNoBindMatcherId, "passes, binds nothing", MatchScope::GRAPH, PASS_NO_BIND_SYMBOL}, + {leakMatcherId, "passes, binds a token", MatchScope::GRAPH, LEAK_THEN_PRUNE_SYMBOL}, + {failMatcherId, "always fails", MatchScope::GRAPH, ALWAYS_FAILS_SYMBOL}}, + makeTestDispatches(), + {survivingPack, prunedPack}, + std::make_shared(SCORE_SYMBOL)); + + const TestGraph graph(makeGraphId(50)); + const auto properties = testDeviceProperties(); + const auto catalog = manager.unsortedCatalog(MatchContext{graph, 0, properties}); + + // Only the surviving pack's kernel enters the catalog. + ASSERT_EQ(catalog.entries.size(), 1U); + EXPECT_EQ(catalog.entries.front().packId, survivingPack.id); + + // The pruned pack's bound token must not reach the catalog the surviving pack + // reads, even though its matcher returned true before the pack was pruned. + EXPECT_EQ(catalog.bound.count("test.bound_token"), 0U); + + GraphMatcherRegistry::unregisterSymbol(PASS_NO_BIND_SYMBOL); + GraphMatcherRegistry::unregisterSymbol(LEAK_THEN_PRUNE_SYMBOL); + GraphMatcherRegistry::unregisterSymbol(ALWAYS_FAILS_SYMBOL); +} + +TEST(TestKernelIngestorStateManager, TwoPacksBindingOneTokenToDifferentValuesThrows) +{ + // Conflicting values under one token name is an authoring error; a silent merge + // would hide it. + constexpr const char* CONFLICTING_SYMBOL = "test.bound_conflict.conflicting"; + GraphMatcherRegistry::registerSymbol(CONFLICTING_SYMBOL, &bindConflictingTokenValue); + + const auto conflictingMatcherId = testId(0xB0); + const KernelDescriptorPack first = makePack({GRAPH_MATCHER_ID}); + KernelDescriptorPack second = makePack({conflictingMatcherId}); + second.id = testId(0xB1); + second.kernels = {makeKernel(testId(0xB2), "second_pack_kernel", 512, "FLOAT")}; + + const StateManager manager(makeSchema(), + {{GRAPH_MATCHER_ID, "graph scoped", MatchScope::GRAPH, "test.graph"}, + {conflictingMatcherId, + "binds a conflicting value", + MatchScope::GRAPH, + CONFLICTING_SYMBOL}}, + makeTestDispatches(), + {first, second}, + std::make_shared(SCORE_SYMBOL)); + + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const TestGraph graph(makeGraphId(51)); + const auto properties = testDeviceProperties(); + + EXPECT_THROW(manager.unsortedCatalog(MatchContext{graph, 0, properties}), std::runtime_error); + + GraphMatcherRegistry::unregisterSymbol(CONFLICTING_SYMBOL); +} + +TEST(TestKernelIngestorStateManager, TwoPacksBindingOneTokenToTheSameValueMergeCleanly) +{ + // Packs agreeing on a token's value must still merge cleanly. + constexpr const char* AGREEING_SYMBOL = "test.bound_conflict.agreeing"; + GraphMatcherRegistry::registerSymbol(AGREEING_SYMBOL, &bindAgreeingTokenValue); + + const auto agreeingMatcherId = testId(0xB3); + const KernelDescriptorPack first = makePack({GRAPH_MATCHER_ID}); + KernelDescriptorPack second = makePack({agreeingMatcherId}); + second.id = testId(0xB4); + second.kernels = {makeKernel(testId(0xB5), "second_pack_kernel", 512, "FLOAT")}; + + const StateManager manager( + makeSchema(), + {{GRAPH_MATCHER_ID, "graph scoped", MatchScope::GRAPH, "test.graph"}, + {agreeingMatcherId, "binds the same value", MatchScope::GRAPH, AGREEING_SYMBOL}}, + makeTestDispatches(), + {first, second}, + std::make_shared(SCORE_SYMBOL)); + + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const TestGraph graph(makeGraphId(52)); + const auto properties = testDeviceProperties(); + + const auto catalog = manager.unsortedCatalog(MatchContext{graph, 0, properties}); + + EXPECT_EQ(tryGetBoundInt(catalog.bound, "test.bound_token"), BOUND_TOKEN_VALUE); + // Agreement never prunes anything: all 4 kernels across both packs survive. + EXPECT_EQ(catalog.entries.size(), 4U); +} + +TEST(TestKernelIngestorStateManager, MatchesSeparatelyPerDevice) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const TestGraph graph(makeGraphId(4)); + const auto properties = testDeviceProperties(); + + manager->unsortedDefinitions(MatchContext{graph, 0, properties}); + manager->unsortedDefinitions(MatchContext{graph, 1, properties}); + + // A kernel applicable on one device need not be applicable on another. + EXPECT_EQ(counters().graphCalls, 2); +} + +TEST(TestKernelIngestorStateManager, RematchesEveryCallWhenTheGraphHasNoIdentity) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + // A graph with no identity has no key to memoize under; inventing one would alias + // unrelated graphs. + const TestGraph graph; + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + + const auto first = manager->unsortedDefinitions(context); + const auto second = manager->unsortedDefinitions(context); + + // Correct both times; only the caching is lost. + EXPECT_EQ(first.size(), 2U); + EXPECT_EQ(second.size(), 2U); + EXPECT_EQ(counters().graphCalls, 2); +} + +TEST(TestKernelIngestorStateManager, DistinctGraphsCarryingANilUuidDoNotShareACatalogEntry) +{ + // Two graphs sharing the nil id must not share a cache entry, or the second would + // get the first's catalog. + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const TestGraph first(makeNilGraphId()); + const TestGraph second(makeNilGraphId()); + const auto properties = testDeviceProperties(); + + const auto firstDefinitions = manager->unsortedDefinitions(MatchContext{first, 0, properties}); + const auto secondDefinitions + = manager->unsortedDefinitions(MatchContext{second, 0, properties}); + + EXPECT_EQ(counters().graphCalls, 2); + EXPECT_EQ(firstDefinitions.size(), 2U); + EXPECT_EQ(secondDefinitions.size(), 2U); +} + +// --------------------------------------------------------------------------- +// Bound state alongside the catalog +// --------------------------------------------------------------------------- + +TEST(TestKernelIngestorStateManager, CarriesWhatMatchingBoundThroughToDispatch) +{ + // RFC 0017 section 8.5: matching also binds the fields dispatch reads, instead of + // re-deriving them from the graph. + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const auto properties = testDeviceProperties(); + const TestGraph graph; + const MatchContext context{graph, 0, properties}; + + const auto bound = manager->unsortedCatalog(context).bound; + + ASSERT_EQ(bound.count("test.bound_token"), 1U); + // tryGetBoundInt also pins that the bound value is the integer alternative, not + // another. + EXPECT_EQ(tryGetBoundInt(bound, "test.bound_token"), BOUND_TOKEN_VALUE); +} + +TEST(TestKernelIngestorStateManager, ReadingBoundStateAfterMatchingDoesNotRematch) +{ + // Section 8.1: bound state is cached with the catalog, so recovering it never + // re-matches. The graph carries an identity, so a cache entry serves the second + // and third reads. + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const auto properties = testDeviceProperties(); + const TestGraph graph(makeGraphId(11)); + const MatchContext context{graph, 0, properties}; + + static_cast(manager->unsortedCatalog(context)); + const auto afterMatching = counters().graphCalls; + + // Asserting only graphCalls would miss a cache that emptied Catalog::bound; the + // cached reads must carry it too. + const auto secondRead = manager->unsortedCatalog(context).bound; + const auto thirdRead = manager->sortedCatalog(context).bound; + + EXPECT_EQ(counters().graphCalls, afterMatching); + EXPECT_EQ(tryGetBoundInt(secondRead, "test.bound_token"), BOUND_TOKEN_VALUE); + EXPECT_EQ(tryGetBoundInt(thirdRead, "test.bound_token"), BOUND_TOKEN_VALUE); +} + +// --------------------------------------------------------------------------- +// Caching +// --------------------------------------------------------------------------- + +TEST(TestKernelIngestorStateManager, RematchesAfterCacheEviction) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(SCORE_SYMBOL, 1); + const auto properties = testDeviceProperties(); + const TestGraph first(makeGraphId(5)); + const TestGraph second(makeGraphId(6)); + + manager->unsortedDefinitions(MatchContext{first, 0, properties}); + manager->unsortedDefinitions(MatchContext{second, 0, properties}); + manager->unsortedDefinitions(MatchContext{first, 0, properties}); + + // Capacity 1 evicts the first graph, forcing a rematch; eviction costs work, not + // correctness. + EXPECT_EQ(counters().graphCalls, 3); +} + +// --------------------------------------------------------------------------- +// Ranking reuse and knob value discovery +// --------------------------------------------------------------------------- + +TEST(TestKernelIngestorStateManager, SortedDefinitionsAreRankedBestFirst) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const TestGraph graph(makeGraphId(7)); + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + + const auto sorted = manager->sortedDefinitions(context); + + ASSERT_EQ(sorted.size(), 2U); + EXPECT_EQ(sorted.front().getIntMetadata(BLOCK_SIZE), 256); +} + +TEST(TestKernelIngestorStateManager, RankingReusesTheAlreadyMatchedCatalog) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const TestGraph graph(makeGraphId(8)); + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + + manager->unsortedDefinitions(context); + manager->sortedDefinitions(context); + manager->sortedDefinitions(context); + + // Ranking is a read of the cached catalog, never a rematch. + EXPECT_EQ(counters().graphCalls, 1); + EXPECT_EQ(counters().kernelCalls, 3); +} + +TEST(TestKernelIngestorStateManager, KnobValuesComeFromTheCatalogInRankedOrder) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + const auto manager = makeStateManager(); + const TestGraph graph(makeGraphId(9)); + const auto properties = testDeviceProperties(); + const MatchContext context{graph, 0, properties}; + + const auto values = StateManager::knobValues(manager->sortedDefinitions(context), BLOCK_SIZE); + + // The pruned HALF kernel also has block_size 64 but contributes no value; a knob + // reflects the surviving catalog, not the schema range. + ASSERT_EQ(values.size(), 2U); + EXPECT_EQ(std::get(values[0]), 256); + EXPECT_EQ(std::get(values[1]), 64); +} + +// --------------------------------------------------------------------------- +// getDispatchDetails +// --------------------------------------------------------------------------- + +TEST(TestKernelIngestorStateManager, GetDispatchDetailsThrowsOnADanglingDispatchId) +{ + // A missing dispatch after a graph was accepted is a hard error, not a silent + // decline. Built directly, bypassing pack validation, since validateAndIndexPacks() + // cannot see a definition that never went through a pack. + const auto manager = makeStateManager(); + const auto kernel = makeDefinition(testId(0x01), 64); + + EXPECT_THROW(manager->getDispatchDetails(kernel), std::runtime_error); +} + +// --------------------------------------------------------------------------- +// completeMetadata (exercised through the constructor/matching path) +// --------------------------------------------------------------------------- + +TEST(TestKernelIngestorStateManager, CompletesAnOmittedFieldFromItsSchemaDefault) +{ + const ScopedSymbols symbols("test.graph", acceptGraph, "test.kernel", countingFloatKernels); + + KernelDescriptorPack pack = makePack({GRAPH_MATCHER_ID, KERNEL_MATCHER_ID}); + // Omits block_size, which defaults, and states dtype, which does not. + KernelDescriptor sparse; + sparse.id = testId(0x70); + sparse.name = "kernel_defaults"; + sparse.metadata = {{DTYPE, MetadataValue{std::string{"FLOAT"}}}}; + pack.kernels = {sparse}; + + const StateManager manager(makeSchema(), + makeTestMatchers(), + makeTestDispatches(), + {pack}, + std::make_shared(SCORE_SYMBOL)); + + const TestGraph graph(makeGraphId(10)); + const auto properties = testDeviceProperties(); + const auto definitions = manager.unsortedDefinitions(MatchContext{graph, 0, properties}); + + ASSERT_EQ(definitions.size(), 1U); + EXPECT_EQ(definitions.front().getIntMetadata(BLOCK_SIZE), 64); + EXPECT_EQ(definitions.front().getStringMetadata(DTYPE), "FLOAT"); +} + +// --------------------------------------------------------------------------- +// Construction-time validation (TEST_P): each case must fail construction with +// std::invalid_argument. +// --------------------------------------------------------------------------- + +struct StateManagerConstructionThrowCase +{ + std::string name; + std::function()> construct; +}; + +class TestKernelIngestorStateManagerConstructionThrows + : public ::testing::TestWithParam +{ +}; + +TEST_P(TestKernelIngestorStateManagerConstructionThrows, RejectsAtConstruction) +{ + EXPECT_THROW(GetParam().construct(), std::invalid_argument); +} + +INSTANTIATE_TEST_SUITE_P( + EagerValidationChecks, + TestKernelIngestorStateManagerConstructionThrows, + ::testing::Values( + StateManagerConstructionThrowCase{ + "RejectsAKernelOmittingAFieldWithNoDefault", + [] { + // dtype has no schema default, so every kernel must state it explicitly. + KernelDescriptorPack pack = makePack({GRAPH_MATCHER_ID, KERNEL_MATCHER_ID}); + KernelDescriptor missingDtype; + missingDtype.id = testId(0x71); + missingDtype.name = "kernel_missing_dtype"; + missingDtype.metadata = {{BLOCK_SIZE, MetadataValue{int64_t{64}}}}; + pack.kernels = {missingDtype}; + return std::make_unique( + makeSchema(), + makeTestMatchers(), + makeTestDispatches(), + std::vector{pack}, + std::make_shared(SCORE_SYMBOL)); + }}, + StateManagerConstructionThrowCase{ + "RejectsAKernelSupplyingAFieldTheSchemaDoesNotDeclare", + [] { + // An undeclared field (e.g. a misspelling) silently takes its default + // while the stray value joins the catalog key unread. + KernelDescriptorPack pack = makePack({GRAPH_MATCHER_ID, KERNEL_MATCHER_ID}); + KernelDescriptor undeclared; + undeclared.id = testId(0x74); + undeclared.name = "kernel_undeclared_field"; + undeclared.metadata = {{BLOCK_SIZE, MetadataValue{int64_t{64}}}, + {DTYPE, MetadataValue{std::string{"FLOAT"}}}, + {"blocksize", MetadataValue{int64_t{128}}}}; + pack.kernels = {undeclared}; + return std::make_unique( + makeSchema(), + makeTestMatchers(), + makeTestDispatches(), + std::vector{pack}, + std::make_shared(SCORE_SYMBOL)); + }}, + StateManagerConstructionThrowCase{ + "RejectsAKernelSupplyingAFieldOfTheWrongType", + [] { + // A wrong-typed field would otherwise surface as a bad_variant_access far + // away, inside a matcher or scorer. + KernelDescriptorPack pack = makePack({GRAPH_MATCHER_ID, KERNEL_MATCHER_ID}); + KernelDescriptor wrongType; + wrongType.id = testId(0x72); + wrongType.name = "kernel_wrong_type"; + wrongType.metadata = {{BLOCK_SIZE, MetadataValue{std::string{"64"}}}, + {DTYPE, MetadataValue{std::string{"FLOAT"}}}}; + pack.kernels = {wrongType}; + return std::make_unique( + makeSchema(), + makeTestMatchers(), + makeTestDispatches(), + std::vector{pack}, + std::make_shared(SCORE_SYMBOL)); + }}, + StateManagerConstructionThrowCase{ + "RejectsAPackNamingAnUnknownMatcher", + [] { + // A dangling matcher reference is caught at construction, not when a + // graph first arrives. + return std::make_unique( + makeSchema(), + std::vector{}, + makeTestDispatches(), + std::vector{makePack({testId(0xFF)})}, + std::make_shared(SCORE_SYMBOL)); + }}, + StateManagerConstructionThrowCase{ + "RejectsAPackNamingAnUnknownDispatchDescriptor", + [] { + return std::make_unique( + makeSchema(), + makeTestMatchers(), + std::vector{}, + std::vector{makePack({GRAPH_MATCHER_ID})}, + std::make_shared(SCORE_SYMBOL)); + }}, + StateManagerConstructionThrowCase{ + "RejectsTwoKernelsSharingAMetadataTuple", + [] { + // Same completed tuple as kernel_64_float: selection would have two + // indistinguishable candidates and no basis to prefer either. + KernelDescriptorPack pack = makePack({GRAPH_MATCHER_ID}); + pack.kernels.push_back(makeKernel(testId(0x73), "kernel_duplicate", 64, "FLOAT")); + return std::make_unique( + makeSchema(), + makeTestMatchers(), + makeTestDispatches(), + std::vector{pack}, + std::make_shared(SCORE_SYMBOL)); + }}, + StateManagerConstructionThrowCase{ + "RejectsADuplicateMatchDescriptorId", + [] { + // Silent first-wins under a duplicate id would run whichever matcher + // loaded first. + std::vector matchers = makeTestMatchers(); + matchers.push_back({GRAPH_MATCHER_ID, + "a different matcher entirely", + MatchScope::KERNEL, + "test.kernel"}); + return std::make_unique( + makeSchema(), + matchers, + makeTestDispatches(), + std::vector{makePack({GRAPH_MATCHER_ID})}, + std::make_shared(SCORE_SYMBOL)); + }}, + StateManagerConstructionThrowCase{ + "RejectsADuplicateDispatchDescriptorId", + [] { + std::vector dispatches = makeTestDispatches(); + dispatches.push_back( + {DISPATCH_ID, "a different dispatch entirely", "test.dispatch.other"}); + return std::make_unique( + makeSchema(), + makeTestMatchers(), + dispatches, + std::vector{makePack({GRAPH_MATCHER_ID})}, + std::make_shared(SCORE_SYMBOL)); + }}, + StateManagerConstructionThrowCase{"RejectsAMissingHeuristic", + [] { + return std::make_unique( + makeSchema(), + std::vector{}, + std::vector{}, + std::vector{}, + nullptr); + }}), + [](const ::testing::TestParamInfo& info) { + return info.param.name; + }); + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/TestLruCache.cpp b/projects/hipdnn/plugin_sdk/tests/ingestor/TestLruCache.cpp new file mode 100644 index 000000000000..5cecd909be1f --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/TestLruCache.cpp @@ -0,0 +1,126 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include +#include +#include + +#include + +#include + +/** + * @file TestLruCache.cpp + * @brief Tests for the bounded, thread-safe LRU cache LruCache.hpp declares. + * Concurrent access is covered separately by TestKernelIngestorConcurrency.cpp. + */ +namespace +{ + +using namespace hipdnn_plugin_sdk::ingestor; + +TEST(TestIngestorLruCache, ReturnsCachedValueWithinCapacity) +{ + LruCache cache(2); + cache.put(1, "one"); + + const auto found = cache.get(1); + + ASSERT_TRUE(found.has_value()); + EXPECT_EQ(*found, "one"); +} + +TEST(TestIngestorLruCache, ReportsMissForAbsentKey) +{ + LruCache cache(2); + + EXPECT_FALSE(cache.get(42).has_value()); +} + +TEST(TestIngestorLruCache, RejectsZeroCapacity) +{ + // Zero capacity would evict every entry on insert; always a caller bug. + using IntCache = LruCache; + EXPECT_THROW(IntCache(0), std::invalid_argument); +} + +// Eviction order: depends on recency, not insertion order; get() counts as a touch. + +struct LruEvictionCase +{ + std::string name; + /// Populates the cache, leaving it one insert away from a capacity-2 overflow. + std::function&)> setup; + int expectedEvicted; + int expectedSurvivor; +}; + +class TestIngestorLruCacheEvictionOrder : public ::testing::TestWithParam +{ +}; + +TEST_P(TestIngestorLruCacheEvictionOrder, EvictsByRecencyNotInsertionOrder) +{ + const auto& testCase = GetParam(); + LruCache cache(2); + testCase.setup(cache); + + // Key 3 is always most-recently-used and must survive the crossing insert. + cache.put(3, "three"); + + EXPECT_FALSE(cache.get(testCase.expectedEvicted).has_value()); + EXPECT_TRUE(cache.get(testCase.expectedSurvivor).has_value()); + EXPECT_TRUE(cache.get(3).has_value()); + EXPECT_EQ(cache.size(), 2U); +} + +INSTANTIATE_TEST_SUITE_P(EvictionOrder, + TestIngestorLruCacheEvictionOrder, + ::testing::Values(LruEvictionCase{"PlainInsertionOrderEvictsOldest", + [](LruCache& cache) { + cache.put(1, "one"); + cache.put(2, "two"); + }, + /*expectedEvicted=*/1, + /*expectedSurvivor=*/2}, + LruEvictionCase{"ReadingAnEntryProtectsItFromEviction", + [](LruCache& cache) { + cache.put(1, "one"); + cache.put(2, "two"); + // Touching 1 makes 2 LRU. + ASSERT_TRUE( + cache.get(1).has_value()); + }, + /*expectedEvicted=*/2, + /*expectedSurvivor=*/1}, + LruEvictionCase{"OverwritingAKeyRefreshesItsRecency", + [](LruCache& cache) { + cache.put(1, "one"); + cache.put(2, "two"); + // Overwrite counts as a touch. + cache.put(1, "uno"); + }, + /*expectedEvicted=*/2, + /*expectedSurvivor=*/1}), + [](const ::testing::TestParamInfo& info) { + return info.param.name; + }); + +TEST(TestIngestorLruCache, OverwritingAKeyDoesNotGrowTheCache) +{ + LruCache cache(2); + cache.put(1, "one"); + cache.put(1, "uno"); + + const auto found = cache.get(1); + + ASSERT_TRUE(found.has_value()); + EXPECT_EQ(*found, "uno"); + EXPECT_EQ(cache.size(), 1U); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/TestMatchContext.cpp b/projects/hipdnn/plugin_sdk/tests/ingestor/TestMatchContext.cpp new file mode 100644 index 000000000000..54462f1bcf5e --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/TestMatchContext.cpp @@ -0,0 +1,127 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include + +#include + +#include + +#include "KernelIngestorTestFixtures.hpp" + +/** + * @file TestMatchContext.cpp + * @brief Unit tests for MatchContext.hpp: the catalog cache key's equality and hash, and + * tryGetGraphId()'s optional-identity contract. + */ +namespace +{ + +using namespace hipdnn_plugin_sdk::ingestor; +using namespace hipdnn_plugin_sdk::ingestor::testing; + +// --------------------------------------------------------------------------- +// CatalogKey::operator== +// --------------------------------------------------------------------------- + +TEST(TestIngestorMatchContext, CatalogKeysWithEqualFieldsCompareEqual) +{ + const CatalogKey first{makeGraphId(1), 0}; + const CatalogKey second{makeGraphId(1), 0}; + + EXPECT_TRUE(first == second); +} + +struct CatalogKeyInequalityCase +{ + std::string name; + CatalogKey key; +}; + +class TestIngestorMatchContextCatalogKeyInequality + : public ::testing::TestWithParam +{ +}; + +TEST_P(TestIngestorMatchContextCatalogKeyInequality, KeysDifferingInOneFieldCompareUnequal) +{ + const CatalogKey reference{makeGraphId(1), 0}; + + EXPECT_FALSE(reference == GetParam().key); +} + +INSTANTIATE_TEST_SUITE_P( + OneFieldAtATime, + TestIngestorMatchContextCatalogKeyInequality, + ::testing::Values(CatalogKeyInequalityCase{"DifferentGraphId", CatalogKey{makeGraphId(2), 0}}, + CatalogKeyInequalityCase{"DifferentDeviceId", CatalogKey{makeGraphId(1), 1}}), + [](const ::testing::TestParamInfo& info) { return info.param.name; }); + +// --------------------------------------------------------------------------- +// CatalogKeyHash +// --------------------------------------------------------------------------- + +TEST(TestIngestorMatchContext, CatalogKeyHashIsConsistentForEqualKeys) +{ + const CatalogKey first{makeGraphId(3), 2}; + const CatalogKey second{makeGraphId(3), 2}; + const CatalogKeyHash hash; + + // Equal keys must hash equal, or LruCache's unordered_map lookup breaks silently. + EXPECT_EQ(hash(first), hash(second)); +} + +TEST(TestIngestorMatchContext, CatalogKeyHashDistinguishesDifferentDeviceIds) +{ + const CatalogKeyHash hash; + const CatalogKey onDeviceZero{makeGraphId(4), 0}; + const CatalogKey onDeviceOne{makeGraphId(4), 1}; + + // Collisions are legal, but this hash folds in deviceId; ignoring it would regress + // lookup cost while still "working" via collision. + EXPECT_NE(hash(onDeviceZero), hash(onDeviceOne)); +} + +// --------------------------------------------------------------------------- +// tryGetGraphId +// --------------------------------------------------------------------------- + +TEST(TestIngestorMatchContext, TryGetGraphIdReturnsTheGraphsIdentity) +{ + const TestGraph graph(makeGraphId(0x42)); + + const auto id = tryGetGraphId(graph); + + ASSERT_TRUE(id.has_value()); + EXPECT_EQ(*id, makeGraphId(0x42)); +} + +TEST(TestIngestorMatchContext, TryGetGraphIdReturnsNulloptForAGraphWithNoIdentity) +{ + // No key to memoize under; callers must treat this as "cannot cache", not an error. + const TestGraph graph; + + EXPECT_EQ(tryGetGraphId(graph), std::nullopt); +} + +TEST(TestIngestorMatchContext, TryGetGraphIdReturnsNulloptForANilId) +{ + // A present-but-nil id must not read as a valid, cacheable key. + const TestGraph graph(makeNilGraphId()); + + EXPECT_EQ(tryGetGraphId(graph), std::nullopt); +} + +TEST(TestIngestorMatchContext, TryGetGraphIdReturnsNulloptForANonV4Id) +{ + // Any non-v4 id is as unable to promise per-graph uniqueness as nil. + const TestGraph graph(makeNonV4GraphId(0x55)); + + EXPECT_EQ(tryGetGraphId(graph), std::nullopt); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/plugin_sdk/tests/ingestor/TestNativeRegistry.cpp b/projects/hipdnn/plugin_sdk/tests/ingestor/TestNativeRegistry.cpp new file mode 100644 index 000000000000..59a657b8d6c3 --- /dev/null +++ b/projects/hipdnn/plugin_sdk/tests/ingestor/TestNativeRegistry.cpp @@ -0,0 +1,54 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#ifdef HIPDNN_ENABLE_KERNEL_INGESTOR + +#include + +#include + +#include +#include + +#include "KernelIngestorTestFixtures.hpp" + +/** + * @file TestNativeRegistry.cpp + * @brief Tests for the symbol-name-to-native-callable registry: registration, + * resolution, and fail-closed behavior on unresolved symbols. + */ +namespace +{ + +using namespace hipdnn_plugin_sdk::ingestor; +using namespace hipdnn_plugin_sdk::ingestor::testing; + +TEST(TestIngestorNativeRegistry, ResolvesARegisteredSymbol) +{ + GraphMatcherRegistry::registerSymbol("registry.resolves", acceptGraph); + + EXPECT_EQ(GraphMatcherRegistry::resolve("registry.resolves"), acceptGraph); + + GraphMatcherRegistry::unregisterSymbol("registry.resolves"); +} + +TEST(TestIngestorNativeRegistry, RejectsDuplicateRegistration) +{ + GraphMatcherRegistry::registerSymbol("registry.duplicate", acceptGraph); + + // Two implementations behind one name: which wins depends on static-init order. + EXPECT_THROW(GraphMatcherRegistry::registerSymbol("registry.duplicate", rejectGraph), + std::runtime_error); + + GraphMatcherRegistry::unregisterSymbol("registry.duplicate"); +} + +TEST(TestIngestorNativeRegistry, FailsClosedOnUnknownSymbol) +{ + // Unshipped symbol must error, not silently match nothing. + EXPECT_THROW(GraphMatcherRegistry::resolve("registry.never_registered"), std::runtime_error); +} + +} // namespace + +#endif // HIPDNN_ENABLE_KERNEL_INGESTOR diff --git a/projects/hipdnn/scripts/run_flatc.py b/projects/hipdnn/scripts/run_flatc.py index e65b51b412c6..2fd06544c8e1 100644 --- a/projects/hipdnn/scripts/run_flatc.py +++ b/projects/hipdnn/scripts/run_flatc.py @@ -1,6 +1,20 @@ # Copyright © Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT +"""Regenerate hipDNN FlatBuffers C++ headers from edited .fbs schemas. + +Runs `flatc --cpp` for each changed schema, writing the data-object headers +(data_objects/*_generated.h) into the committed include tree. This is the +pre-commit entry point (hook `flatc-hipdnn`). + +Scope limit: this script does NOT regenerate the UMD op-schema registry +(umd/op_schema_registry_generated.hpp). That header is derived from a compiled +C++ tool (umd_registry_gen) via the FlatBuffers reflection schema (graph.bfbs), +which the pre-commit environment cannot build. It is regenerated by the CMake +build (HIPDNN_GENERATE_SDK_HEADERS, default ON) and enforced in CI by the +"Verify UMD op-schema registry is in sync" step in hipdnn-superbuild-ci.yml. +""" + import sys import subprocess import shutil