From 182de5b0f5567e1ca91bd93aa0056e3074ac420a Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Wed, 26 Aug 2026 12:28:19 -0700 Subject: [PATCH 1/3] hex-devices: add support for lazy session allocation and cleanup dev interfaces Co-authored-by: Marco Colombo --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 141 ++++++++++++++++--------- 1 file changed, 90 insertions(+), 51 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index c1e9f919d92a..9e91b0d66696 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -376,9 +376,6 @@ struct ggml_hexagon_session { std::unordered_map> cloned_buffers; std::unordered_set sync_peers; - ggml_backend_buffer_type buffer_type = {}; - ggml_backend_buffer_type host_buffer_type = {}; - uint32_t n_threads = 0; uint32_t n_hvx = 0; uint32_t n_hmx = 0; @@ -430,14 +427,40 @@ struct ggml_hexagon_session { // ** backend buffers +struct ggml_backend_hexagon_device_context { + int dev_id; + int phys_idx; + int virt_idx; + std::string name; + ggml_backend_dev_t dev = nullptr; + size_t max_bufsize = 0; + + ggml_backend_buffer_type buffer_type = {}; + ggml_backend_buffer_type host_buffer_type = {}; + + std::unique_ptr sess; + + ggml_backend_hexagon_device_context(int dev_id, int phys_idx, int virt_idx, const std::string & name, ggml_backend_dev_t dev); + ~ggml_backend_hexagon_device_context(); + + const char * c_name() const { return name.c_str(); } + + ggml_hexagon_session * session() { + if (!sess) { + sess = std::make_unique(dev_id, dev); + } + return sess.get(); + } +}; + struct ggml_backend_hexagon_buffer_type_context { - ggml_backend_hexagon_buffer_type_context(const std::string & name, ggml_hexagon_session * sess) { - this->sess = sess; - this->name = name; + ggml_backend_hexagon_buffer_type_context(const std::string & name, ggml_backend_hexagon_device_context * dev_ctx) { + this->dev_ctx = dev_ctx; + this->name = name; } - ggml_hexagon_session * sess; - std::string name; + ggml_backend_hexagon_device_context * dev_ctx; + std::string name; }; struct ggml_hexagon_rpcmem_block { @@ -576,7 +599,8 @@ struct ggml_hexagon_shared_buffer { }; static ggml_hexagon_session * ggml_backend_hexagon_buffer_get_sess(ggml_backend_buffer_t buffer) { - return static_cast(buffer->buft->context)->sess; + auto sbuf = static_cast(buffer->context); + return sbuf->sess; } static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) { @@ -1494,24 +1518,26 @@ static const char * ggml_backend_hexagon_buffer_type_name(ggml_backend_buffer_ty static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buffer_type, size_t size) { - auto sess = static_cast(buffer_type->context)->sess; + auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; + auto sess = dev_ctx->session(); try { ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE); return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size); } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", sess->c_name(), exc.what()); + GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", dev_ctx->c_name(), exc.what()); return nullptr; } } static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer( ggml_backend_buffer_type_t buffer_type, size_t size) { - auto sess = static_cast(buffer_type->context)->sess; + auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; + auto sess = dev_ctx->session(); try { ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false, GGML_HEXAGON_FENCE_BUFFER_SIZE); return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_host_buffer_interface, sbuf, size); } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", sess->c_name(), exc.what()); + GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", dev_ctx->c_name(), exc.what()); return nullptr; } } @@ -1536,7 +1562,7 @@ static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffe static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { auto * context = static_cast(buft->context); - return context->sess->max_bufsize; + return context->dev_ctx->max_bufsize; } static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) { @@ -1567,6 +1593,22 @@ static ggml_backend_buffer_type_i ggml_backend_hexagon_host_buffer_type_interfac /* .is_host = */ ggml_backend_hexagon_host_buffer_type_is_host, }; +ggml_backend_hexagon_device_context::ggml_backend_hexagon_device_context(int dev_id, int phys_idx, int virt_idx, const std::string & name, ggml_backend_dev_t dev) + : dev_id(dev_id), phys_idx(phys_idx), virt_idx(virt_idx), name(name), dev(dev), max_bufsize(opt_mbuf) { + buffer_type.device = dev; + buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; + buffer_type.context = new ggml_backend_hexagon_buffer_type_context(name, this); + + host_buffer_type.device = dev; + host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface; + host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(name + "-HOST", this); +} + +ggml_backend_hexagon_device_context::~ggml_backend_hexagon_device_context() { + delete static_cast(buffer_type.context); + delete static_cast(host_buffer_type.context); +} + static bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) { return b->buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment; } @@ -3055,32 +3097,22 @@ void ggml_hexagon_session::release() noexcept(true) { } ggml_hexagon_session::ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false) { - buffer_type.device = dev; - host_buffer_type.device = dev; - op_batch = nullptr; op_queue = nullptr; fence_seq = ((uintptr_t)this) & 0xFFFF; try { allocate(dev_id); - - buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; - buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name, this); - - host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface; - host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name + "-HOST", this); } catch (const std::exception & exc) { release(); throw; } + + GGML_UNUSED(dev); } ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) { release(); - - delete static_cast(buffer_type.context); - delete static_cast(host_buffer_type.context); } // ** backend interface @@ -3957,11 +3989,13 @@ static void ggml_hexagon_precompute_fused_mmnx_params( } static bool ggml_hexagon_tensor_is_host(const struct ggml_hexagon_session * sess, const struct ggml_tensor * t) { - return t && t->buffer && t->buffer->buft == &sess->host_buffer_type; + return t && t->buffer && ggml_backend_buft_is_host(t->buffer->buft); + GGML_UNUSED(sess); } static bool ggml_hexagon_tensor_is_non_host(const struct ggml_hexagon_session * sess, const struct ggml_tensor * t) { - return t && t->buffer && t->buffer->buft != &sess->host_buffer_type; + return t && t->buffer && !ggml_backend_buft_is_host(t->buffer->buft); + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) { @@ -5265,7 +5299,8 @@ bool ggml_backend_is_hexagon(ggml_backend_t backend) { // device interface static ggml_backend_t ggml_backend_hexagon_device_init(ggml_backend_dev_t dev, const char * params) { - auto sess = static_cast(dev->context); + auto dev_ctx = static_cast(dev->context); + auto sess = dev_ctx->session(); return new ggml_backend{ /* .guid = */ ggml_backend_hexagon_guid(), @@ -5278,8 +5313,8 @@ static ggml_backend_t ggml_backend_hexagon_device_init(ggml_backend_dev_t dev, c } static const char * ggml_backend_hexagon_device_get_name(ggml_backend_dev_t dev) { - auto sess = static_cast(dev->context); - return sess->c_name(); + auto dev_ctx = static_cast(dev->context); + return dev_ctx->c_name(); GGML_UNUSED(dev); } @@ -5317,16 +5352,16 @@ static void ggml_backend_hexagon_device_get_props(ggml_backend_dev_t dev, struct } static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_buffer_type(ggml_backend_dev_t dev) { - auto sess = static_cast(dev->context); - return &sess->buffer_type; + auto dev_ctx = static_cast(dev->context); + return &dev_ctx->buffer_type; } static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_host_buffer_type(ggml_backend_dev_t dev) { if (!opt_hostbuf) { return NULL; } - auto sess = static_cast(dev->context); - return &sess->host_buffer_type; + auto dev_ctx = static_cast(dev->context); + return &dev_ctx->host_buffer_type; } static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -5417,7 +5452,8 @@ static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess } static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { - auto sess = static_cast(dev->context); + auto dev_ctx = static_cast(dev->context); + auto sess = dev_ctx->session(); // reject ops that match the filter if (opt_opfilter && std::regex_match(ggml_op_desc(op), *opt_opfilter)) { @@ -5487,6 +5523,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons supp = ggml_hexagon_supported_unary(sess, op); break; default: + supp = false; break; } break; @@ -5499,6 +5536,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons supp = ggml_hexagon_supported_activations(sess, op); break; default: + supp = false; break; } break; @@ -5584,17 +5622,17 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons } static bool ggml_backend_hexagon_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { - auto sess = static_cast(dev->context); + auto dev_ctx = static_cast(dev->context); // Technically we can clone hexagon buffers from any session but for some reason the output is garbled with layer-split, // tensor-split works correctly, so it needs mode debugging and investigation. For now accept only our own buffers. #if 0 bool supp = (buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment); #else - bool supp = (buft == &sess->host_buffer_type) || (buft == &sess->buffer_type); + bool supp = (buft == &dev_ctx->host_buffer_type) || (buft == &dev_ctx->buffer_type); #endif - HEX_VERBOSE("ggml-hex: %s device-supports-buft %s %s\n", sess->name.c_str(), ggml_backend_buft_name(buft), supp ? "yes" : "no"); + HEX_VERBOSE("ggml-hex: %s device-supports-buft %s %s\n", dev_ctx->c_name(), ggml_backend_buft_name(buft), supp ? "yes" : "no"); return supp; } @@ -5623,16 +5661,17 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) { GGML_LOG_INFO("ggml-hex: Hexagon Arch version v%d\n", opt_arch); - // Create devices / sessions + // Create devices for (size_t i = 0; i < opt_ndev; i++) { - devices[i].iface = ggml_backend_hexagon_device_i; - devices[i].reg = reg; - try { - devices[i].context = new ggml_hexagon_session(i, &devices[i]); - } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: failed to create device/session %zu\n", i); - devices[i].context = nullptr; - } + devices[i].iface = ggml_backend_hexagon_device_i; + devices[i].reg = reg; + devices[i].context = new ggml_backend_hexagon_device_context( + i, + opt_device_configs[i].physical_idx, + opt_device_configs[i].virtual_idx, + opt_device_configs[i].name, + &devices[i] + ); } } @@ -5640,10 +5679,10 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) { ggml_hexagon_registry::~ggml_hexagon_registry() { GGML_LOG_INFO("ggml-hex: releasing registry\n"); - // Release devices / sessions + // Release devices for (size_t i = 0; i < opt_ndev; i++) { - auto sess = static_cast(devices[i].context); - delete sess; + auto dev_ctx = static_cast(devices[i].context); + delete dev_ctx; } } From f52cd94a08469df8e6d123de1649adace58f24bc Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Wed, 26 Aug 2026 15:03:54 -0700 Subject: [PATCH 2/3] hex-devices: support for runtime discovery of available NPU cores Co-authored-by: Alexander Lu Co-authored-by: Ehsan Bateni --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 117 +++++++++++++++++++++---- ggml/src/ggml-hexagon/htp-drv.cpp | 10 +++ ggml/src/ggml-hexagon/htp-drv.h | 2 + 3 files changed, 113 insertions(+), 16 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 9e91b0d66696..ac73f9a87255 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -76,7 +76,72 @@ struct ggml_hexagon_device_config { static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS]; +struct ggml_hexagon_domain_info { + int id; + std::string name; +}; + +// Enumerate NSP domains via FASTRPC_GET_DOMAINS if supported. +// Filters to FASTRPC_NSP type only so non-compute domains are excluded. +static const std::vector & ggml_hexagon_discover_domains() { + static std::vector cached; + static bool discovered = false; + if (discovered) { + return cached; + } + discovered = true; + + system_req_payload domain_info = {}; + domain_info.id = FASTRPC_GET_DOMAINS; + domain_info.sys.domains = nullptr; + domain_info.sys.max_domains = 0; + domain_info.sys.flags = DOMAINS_LIST_FLAGS_SET_TYPE(0, FASTRPC_NSP); + + int err = remote_system_request(&domain_info); + if (err != AEE_SUCCESS) { + GGML_LOG_DEBUG("ggml-hex: FASTRPC_GET_DOMAINS query failed (0x%x), using static CDSP domains\n", (unsigned) err); + return cached; + } + + if (domain_info.sys.num_domains <= 0) { + GGML_LOG_DEBUG("ggml-hex: FASTRPC_GET_DOMAINS reported 0 domains, using static CDSP domains\n"); + return cached; + } + + std::vector domains(domain_info.sys.num_domains); + domain_info.sys.domains = domains.data(); + domain_info.sys.max_domains = (int) domains.size(); + + err = remote_system_request(&domain_info); + if (err != AEE_SUCCESS) { + GGML_LOG_WARN("ggml-hex: FASTRPC_GET_DOMAINS fetch failed (0x%x), using static CDSP domains\n", (unsigned) err); + return cached; + } + + const int n_domains = std::min(domain_info.sys.num_domains, (int) domains.size()); + for (int i = 0; i < n_domains; i++) { + GGML_LOG_INFO("ggml-hex: FASTRPC_GET_DOMAINS[%d]: type=%d id=%d name='%s' status=%d instance_id=%d\n", + i, (int) domains[i].type, domains[i].id, domains[i].name, domains[i].status, domains[i].instance_id); + if (domains[i].type != FASTRPC_NSP) { + GGML_LOG_DEBUG("ggml-hex: skipping non-NSP domain (type=%d)\n", (int) domains[i].type); + continue; + } + if (!domains[i].status) { + GGML_LOG_WARN("ggml-hex: skipping NSP domain id=%d (status=down)\n", domains[i].id); + continue; + } + cached.push_back({ domains[i].id, std::string(domains[i].name) }); + GGML_LOG_INFO("ggml-hex: using NSP domain[%zu]: id=%d name='%s'\n", + cached.size() - 1, domains[i].id, domains[i].name); + } + return cached; +} + static int get_domain_id(int physical_idx) { + const auto & domains = ggml_hexagon_discover_domains(); + if (physical_idx >= 0 && physical_idx < (int) domains.size()) { + return domains[physical_idx].id; + } switch (physical_idx) { case 0: return 3; // CDSP0 (all devices) case 1: return 4; // CDSP1 (IQ9, IQ10) @@ -87,6 +152,10 @@ static int get_domain_id(int physical_idx) { } static std::string get_domain_name(int physical_idx) { + const auto & domains = ggml_hexagon_discover_domains(); + if (physical_idx >= 0 && physical_idx < (int) domains.size()) { + return domains[physical_idx].name; + } if (physical_idx == 0) { return CDSP_DOMAIN_NAME; } @@ -2874,13 +2943,21 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str()); domain * my_domain = htpdrv_get_domain(this->domain_id); - if (my_domain == NULL) { - GGML_LOG_ERROR("ggml-hex: unable to get domain struct for CDSP (domain_id %d)\n", this->domain_id); - throw std::runtime_error("ggml-hex: failed to get CDSP domain (see log for details)"); - } std::string dom_name = get_domain_name(phys_idx); + // Enable Unsigned PD for all domains + { + struct remote_rpc_control_unsigned_module u; + u.domain = -1; + u.enable = 1; + int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); + if (err != AEE_SUCCESS) { + GGML_LOG_ERROR("ggml-hex: failed to enable unsigned PD for session %d : error 0x%x\n", dev_id, err); + throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); + } + } + // Create new session if virtual_idx > 0 if (virt_idx > 0) { struct remote_rpc_reserve_new_session n; @@ -2899,6 +2976,19 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { this->session_id = n.session_id; this->domain_id = n.effective_domain_id; this->valid_session = true; + } else { + struct remote_rpc_effective_domain_id eff = {}; + eff.domain_name = const_cast(dom_name.c_str()); + eff.domain_name_len = dom_name.size(); + eff.session_id = 0; + + int err = remote_session_control(FASTRPC_GET_EFFECTIVE_DOMAIN_ID, (void *) &eff, sizeof(eff)); + if (err == AEE_SUCCESS) { + this->domain_id = eff.effective_domain_id; + } else { + GGML_LOG_DEBUG("ggml-hex: %s FASTRPC_GET_EFFECTIVE_DOMAIN_ID returned 0x%x, using domain_id %d\n", + this->name.c_str(), err, this->domain_id); + } } // Get session URI @@ -2919,6 +3009,13 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u)); if (err != AEE_SUCCESS) { + if (my_domain == NULL) { + GGML_LOG_ERROR("ggml-hex: failed to get URI for session %d (physical %d, virtual %d) : error 0x%x, " + "and no static fallback URI is known for domain_id %d\n", + dev_id, phys_idx, virt_idx, err, this->domain_id); + throw std::runtime_error("ggml-hex: failed to get session URI (see log for details)"); + } + // fallback to single session uris int htp_URI_domain_len = strlen(htp_uri) + MAX_DOMAIN_NAMELEN; @@ -2928,18 +3025,6 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { } } - // Enable Unsigned PD - { - struct remote_rpc_control_unsigned_module u; - u.domain = this->domain_id; - u.enable = 1; - int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); - if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: failed to enable unsigned PD for session %d : error 0x%x\n", dev_id, err); - throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); - } - } - // Open session int err = htp_iface_open(session_uri, &this->handle); if (err != AEE_SUCCESS) { diff --git a/ggml/src/ggml-hexagon/htp-drv.cpp b/ggml/src/ggml-hexagon/htp-drv.cpp index 4f0790801731..437e367c9d35 100644 --- a/ggml/src/ggml-hexagon/htp-drv.cpp +++ b/ggml/src/ggml-hexagon/htp-drv.cpp @@ -73,6 +73,7 @@ typedef int (*remote_handle64_close_pfn_t)(remote_handle h); typedef int (*remote_handle_control_pfn_t)(uint32_t req, void* data, uint32_t datalen); typedef int (*remote_handle64_control_pfn_t)(remote_handle64 h, uint32_t req, void* data, uint32_t datalen); typedef int (*remote_session_control_pfn_t)(uint32_t req, void *data, uint32_t datalen); +typedef int (*remote_system_request_pfn_t)(system_req_payload * req); // // Driver API pfns @@ -99,6 +100,7 @@ remote_handle64_close_pfn_t remote_handle64_close_pfn = nullptr; remote_handle_control_pfn_t remote_handle_control_pfn = nullptr; remote_handle64_control_pfn_t remote_handle64_control_pfn = nullptr; remote_session_control_pfn_t remote_session_control_pfn = nullptr; +remote_system_request_pfn_t remote_system_request_pfn = nullptr; // // Driver API @@ -206,6 +208,13 @@ HTPDRV_API int remote_session_control(uint32_t req, void * data, uint32_t datale return remote_session_control_pfn(req, data, datalen); } +HTPDRV_API int remote_system_request(system_req_payload * req) { + if (!remote_system_request_pfn) { + return AEE_EUNSUPPORTEDAPI; + } + return remote_system_request_pfn(req); +} + #ifdef _WIN32 static std::string wstr_to_str(std::wstring_view wstr) { @@ -367,6 +376,7 @@ int htpdrv_init() { dlsym(handle.get(), remote_handle64_control_pfn_t, remote_handle64_control_pfn, remote_handle64_control, false); dlsym(handle.get(), remote_session_control_pfn_t, remote_session_control_pfn, remote_session_control, false); dlsym(handle.get(), remote_handle64_close_pfn_t, remote_handle64_close_pfn, remote_handle64_close, false); + dlsym(handle.get(), remote_system_request_pfn_t, remote_system_request_pfn, remote_system_request, true); lib_cdsp_rpc_handle = std::move(handle); initialized = true; diff --git a/ggml/src/ggml-hexagon/htp-drv.h b/ggml/src/ggml-hexagon/htp-drv.h index f3cc0da75c28..8232780e7fdc 100644 --- a/ggml/src/ggml-hexagon/htp-drv.h +++ b/ggml/src/ggml-hexagon/htp-drv.h @@ -116,6 +116,8 @@ HTPDRV_API domain * htpdrv_get_domain(int domain_id); */ HTPDRV_API int htpdrv_get_arch(int domain, int * arch); +HTPDRV_API int remote_system_request(system_req_payload * req); + #ifdef __cplusplus } #endif From 898e766554562738227affddd9b788d229db1c17 Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Wed, 26 Aug 2026 17:01:52 -0700 Subject: [PATCH 3/3] hex-devices: reject non-existing devices early during init --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 255 ++++++++++++------------- 1 file changed, 118 insertions(+), 137 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index ac73f9a87255..571ace7c8a09 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -69,99 +69,15 @@ using u32vec = std::vector; #define GGML_HEXAGON_FENCE_SLOT_SIZE 128 struct ggml_hexagon_device_config { - int physical_idx = 0; - int virtual_idx = 0; + int physical_idx = 0; + int virtual_idx = 0; + int domain_id = 0; + std::string domain_name; std::string name; }; static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS]; -struct ggml_hexagon_domain_info { - int id; - std::string name; -}; - -// Enumerate NSP domains via FASTRPC_GET_DOMAINS if supported. -// Filters to FASTRPC_NSP type only so non-compute domains are excluded. -static const std::vector & ggml_hexagon_discover_domains() { - static std::vector cached; - static bool discovered = false; - if (discovered) { - return cached; - } - discovered = true; - - system_req_payload domain_info = {}; - domain_info.id = FASTRPC_GET_DOMAINS; - domain_info.sys.domains = nullptr; - domain_info.sys.max_domains = 0; - domain_info.sys.flags = DOMAINS_LIST_FLAGS_SET_TYPE(0, FASTRPC_NSP); - - int err = remote_system_request(&domain_info); - if (err != AEE_SUCCESS) { - GGML_LOG_DEBUG("ggml-hex: FASTRPC_GET_DOMAINS query failed (0x%x), using static CDSP domains\n", (unsigned) err); - return cached; - } - - if (domain_info.sys.num_domains <= 0) { - GGML_LOG_DEBUG("ggml-hex: FASTRPC_GET_DOMAINS reported 0 domains, using static CDSP domains\n"); - return cached; - } - - std::vector domains(domain_info.sys.num_domains); - domain_info.sys.domains = domains.data(); - domain_info.sys.max_domains = (int) domains.size(); - - err = remote_system_request(&domain_info); - if (err != AEE_SUCCESS) { - GGML_LOG_WARN("ggml-hex: FASTRPC_GET_DOMAINS fetch failed (0x%x), using static CDSP domains\n", (unsigned) err); - return cached; - } - - const int n_domains = std::min(domain_info.sys.num_domains, (int) domains.size()); - for (int i = 0; i < n_domains; i++) { - GGML_LOG_INFO("ggml-hex: FASTRPC_GET_DOMAINS[%d]: type=%d id=%d name='%s' status=%d instance_id=%d\n", - i, (int) domains[i].type, domains[i].id, domains[i].name, domains[i].status, domains[i].instance_id); - if (domains[i].type != FASTRPC_NSP) { - GGML_LOG_DEBUG("ggml-hex: skipping non-NSP domain (type=%d)\n", (int) domains[i].type); - continue; - } - if (!domains[i].status) { - GGML_LOG_WARN("ggml-hex: skipping NSP domain id=%d (status=down)\n", domains[i].id); - continue; - } - cached.push_back({ domains[i].id, std::string(domains[i].name) }); - GGML_LOG_INFO("ggml-hex: using NSP domain[%zu]: id=%d name='%s'\n", - cached.size() - 1, domains[i].id, domains[i].name); - } - return cached; -} - -static int get_domain_id(int physical_idx) { - const auto & domains = ggml_hexagon_discover_domains(); - if (physical_idx >= 0 && physical_idx < (int) domains.size()) { - return domains[physical_idx].id; - } - switch (physical_idx) { - case 0: return 3; // CDSP0 (all devices) - case 1: return 4; // CDSP1 (IQ9, IQ10) - case 2: return 18; // CDSP2 (IQ10) - case 3: return 19; // CDSP3 (IQ10) - default: return CDSP_DOMAIN_ID + physical_idx; - } -} - -static std::string get_domain_name(int physical_idx) { - const auto & domains = ggml_hexagon_discover_domains(); - if (physical_idx >= 0 && physical_idx < (int) domains.size()) { - return domains[physical_idx].name; - } - if (physical_idx == 0) { - return CDSP_DOMAIN_NAME; - } - return std::string("cdsp") + std::to_string(physical_idx); -} - static int opt_arch = 0; // autodetect static size_t opt_ndev = 1; static size_t opt_nhvx = 0; // use all @@ -430,7 +346,6 @@ struct ggml_hexagon_session { uint32_t session_id; uint32_t domain_id; uint64_t queue_id; - int dev_id; int phys_idx; int virt_idx; bool valid_session; @@ -458,12 +373,12 @@ struct ggml_hexagon_session { mutable std::unordered_set needs_repack; - ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false); + ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr) noexcept(false); ~ggml_hexagon_session() noexcept(true); const char* c_name() const { return name.c_str(); } - void allocate(int dev_id) noexcept(false); + void allocate(const ggml_hexagon_device_config & config) noexcept(false); void release() noexcept(true); void enqueue_op(const htp_opnode & node); @@ -497,26 +412,24 @@ struct ggml_hexagon_session { // ** backend buffers struct ggml_backend_hexagon_device_context { - int dev_id; - int phys_idx; - int virt_idx; - std::string name; - ggml_backend_dev_t dev = nullptr; - size_t max_bufsize = 0; + int dev_id; + ggml_hexagon_device_config config; + ggml_backend_dev_t dev = nullptr; + size_t max_bufsize = 0; ggml_backend_buffer_type buffer_type = {}; ggml_backend_buffer_type host_buffer_type = {}; std::unique_ptr sess; - ggml_backend_hexagon_device_context(int dev_id, int phys_idx, int virt_idx, const std::string & name, ggml_backend_dev_t dev); + ggml_backend_hexagon_device_context(int dev_id, const ggml_hexagon_device_config & config, ggml_backend_dev_t dev); ~ggml_backend_hexagon_device_context(); - const char * c_name() const { return name.c_str(); } + const char * c_name() const { return config.name.c_str(); } ggml_hexagon_session * session() { if (!sess) { - sess = std::make_unique(dev_id, dev); + sess = std::make_unique(config, dev); } return sess.get(); } @@ -1662,15 +1575,15 @@ static ggml_backend_buffer_type_i ggml_backend_hexagon_host_buffer_type_interfac /* .is_host = */ ggml_backend_hexagon_host_buffer_type_is_host, }; -ggml_backend_hexagon_device_context::ggml_backend_hexagon_device_context(int dev_id, int phys_idx, int virt_idx, const std::string & name, ggml_backend_dev_t dev) - : dev_id(dev_id), phys_idx(phys_idx), virt_idx(virt_idx), name(name), dev(dev), max_bufsize(opt_mbuf) { +ggml_backend_hexagon_device_context::ggml_backend_hexagon_device_context(int dev_id, const ggml_hexagon_device_config & config, ggml_backend_dev_t dev) + : dev_id(dev_id), config(config), dev(dev), max_bufsize(opt_mbuf) { buffer_type.device = dev; buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; - buffer_type.context = new ggml_backend_hexagon_buffer_type_context(name, this); + buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name, this); host_buffer_type.device = dev; host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface; - host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(name + "-HOST", this); + host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-HOST", this); } ggml_backend_hexagon_device_context::~ggml_backend_hexagon_device_context() { @@ -2922,8 +2835,7 @@ static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) { return vmem - step; // backoff to account for overhead from internal mappings } -void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { - const auto & config = opt_device_configs[dev_id]; +void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) noexcept(false) { int phys_idx = config.physical_idx; int virt_idx = config.virtual_idx; @@ -2934,17 +2846,19 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { this->phys_idx = phys_idx; this->virt_idx = virt_idx; - this->domain_id = get_domain_id(phys_idx); + this->domain_id = config.domain_id; this->session_id = 0; - this->dev_id = dev_id; this->name = config.name; this->op_pending = 0; GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str()); - domain * my_domain = htpdrv_get_domain(this->domain_id); + if (config.domain_id < 0 || config.domain_name.empty()) { + GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx); + throw std::runtime_error("ggml-hex: invalid physical CDSP core"); + } - std::string dom_name = get_domain_name(phys_idx); + const std::string & dom_name = config.domain_name; // Enable Unsigned PD for all domains { @@ -2953,7 +2867,7 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { u.enable = 1; int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: failed to enable unsigned PD for session %d : error 0x%x\n", dev_id, err); + GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err); throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); } } @@ -2968,7 +2882,8 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n)); if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: failed to reserve new session %d (physical %d, virtual %d) : error 0x%x\n", dev_id, phys_idx, virt_idx, err); + GGML_LOG_ERROR("ggml-hex: %s failed to reserve new session (physical %d, virtual %d) : error 0x%x\n", + this->c_name(), phys_idx, virt_idx, err); throw std::runtime_error("ggml-hex: remote_session_control(new-sess) failed (see log for details)"); } @@ -2991,8 +2906,6 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { } } - // Get session URI - char session_uri[256]; { char htp_uri[256]; @@ -3009,26 +2922,18 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u)); if (err != AEE_SUCCESS) { - if (my_domain == NULL) { - GGML_LOG_ERROR("ggml-hex: failed to get URI for session %d (physical %d, virtual %d) : error 0x%x, " - "and no static fallback URI is known for domain_id %d\n", - dev_id, phys_idx, virt_idx, err, this->domain_id); - throw std::runtime_error("ggml-hex: failed to get session URI (see log for details)"); - } - - // fallback to single session uris - int htp_URI_domain_len = strlen(htp_uri) + MAX_DOMAIN_NAMELEN; + snprintf(session_uri, sizeof(session_uri), "%s&_dom=%s&_session=%u", + htp_uri, dom_name.c_str(), this->session_id); - snprintf(session_uri, htp_URI_domain_len, "%s%s", htp_uri, my_domain->uri); - - GGML_LOG_WARN("ggml-hex: failed to get URI for session %d (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n", dev_id, phys_idx, virt_idx, err, session_uri); + GGML_LOG_WARN("ggml-hex: %s failed to get URI (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n", + this->c_name(), phys_idx, virt_idx, err, session_uri); } } // Open session int err = htp_iface_open(session_uri, &this->handle); if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: failed to open session %d : error 0x%x\n", dev_id, err); + GGML_LOG_ERROR("ggml-hex: %s failed to open session : error 0x%x\n", this->c_name(), err); throw std::runtime_error("ggml-hex: failed to open session (see log for details)"); } @@ -3118,7 +3023,7 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem); // Start dspqueue/opbatch processing - err = htp_iface_start(this->handle, dev_id, this->queue_id, opt_nhvx, opt_nhmx, this->max_vmem); + err = htp_iface_start(this->handle, this->session_id, this->queue_id, opt_nhvx, opt_nhmx, this->max_vmem); if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err); throw std::runtime_error("ggml-hex: iface start failed (see log for details)"); @@ -3181,13 +3086,13 @@ void ggml_hexagon_session::release() noexcept(true) { this->cloned_buffers.clear(); } -ggml_hexagon_session::ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false) { +ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev) noexcept(false) { op_batch = nullptr; op_queue = nullptr; fence_seq = ((uintptr_t)this) & 0xFFFF; try { - allocate(dev_id); + allocate(config); } catch (const std::exception & exc) { release(); throw; @@ -5750,13 +5655,7 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) { for (size_t i = 0; i < opt_ndev; i++) { devices[i].iface = ggml_backend_hexagon_device_i; devices[i].reg = reg; - devices[i].context = new ggml_backend_hexagon_device_context( - i, - opt_device_configs[i].physical_idx, - opt_device_configs[i].virtual_idx, - opt_device_configs[i].name, - &devices[i] - ); + devices[i].context = new ggml_backend_hexagon_device_context(i, opt_device_configs[i], &devices[i]); } } @@ -5936,6 +5835,85 @@ template std::string vec_to_str(std::vector v) { return str; } +// Enumerate NPU (aka CDSP) domains via FASTRPC_GET_DOMAINS if supported, +// and populate domain_id and domain_name for all configured devices. +static void ggml_hexagon_discover_devices() { + std::unordered_map cdsp_map; + bool discovery_supported = false; + + system_req_payload domain_info = {}; + domain_info.id = FASTRPC_GET_DOMAINS; + domain_info.sys.domains = nullptr; + domain_info.sys.max_domains = 0; + domain_info.sys.flags = DOMAINS_LIST_FLAGS_SET_TYPE(0, FASTRPC_NSP); + + int err = remote_system_request(&domain_info); + if (err == AEE_SUCCESS && domain_info.sys.num_domains > 0) { + std::vector domains(domain_info.sys.num_domains); + domain_info.sys.domains = domains.data(); + domain_info.sys.max_domains = (int) domains.size(); + + err = remote_system_request(&domain_info); + if (err == AEE_SUCCESS) { + discovery_supported = true; + const int n_domains = std::min(domain_info.sys.num_domains, (int) domains.size()); + for (int i = 0; i < n_domains; i++) { + GGML_LOG_INFO("ggml-hex: FASTRPC_GET_DOMAINS[%d]: type %d id %d name '%s' status %d instance-id %d\n", + i, (int) domains[i].type, domains[i].id, domains[i].name, domains[i].status, domains[i].instance_id); + if (domains[i].type != FASTRPC_NSP) { + GGML_LOG_DEBUG("ggml-hex: skipping non-CDSP domain (type=%d)\n", (int) domains[i].type); + continue; + } + if (!domains[i].status) { + GGML_LOG_WARN("ggml-hex: skipping CDSP domain id=%d (status=down)\n", domains[i].id); + continue; + } + cdsp_map[domains[i].instance_id] = domains[i]; + GGML_LOG_INFO("ggml-hex: using CDSP domain: instance-id %d id %d name '%s'\n", + domains[i].instance_id, domains[i].id, domains[i].name); + } + } else { + GGML_LOG_WARN("ggml-hex: FASTRPC_GET_DOMAINS fetch failed (0x%x), using static CDSP domains\n", (unsigned) err); + } + } else if (err != AEE_SUCCESS) { + GGML_LOG_DEBUG("ggml-hex: FASTRPC_GET_DOMAINS query failed (0x%x), using static CDSP domains\n", (unsigned) err); + } + + // Populate domain IDs and names for all configured devices + for (size_t i = 0; i < opt_ndev; i++) { + auto & cfg = opt_device_configs[i]; + if (discovery_supported) { + auto it = cdsp_map.find(cfg.physical_idx); + if (it != cdsp_map.end()) { + cfg.domain_id = it->second.id; + cfg.domain_name = it->second.name; + } else { + GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not found on device (%zu CDSP core(s) available)\n", + cfg.physical_idx, cdsp_map.size()); + cfg.domain_id = -1; + cfg.domain_name = ""; + } + } else { + switch (cfg.physical_idx) { + case 0: + cfg.domain_id = 3; + cfg.domain_name = CDSP_DOMAIN_NAME; + break; + case 1: + cfg.domain_id = 4; + cfg.domain_name = "cdsp1"; + break; + default: + GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not supported without dynamic discovery\n", + cfg.physical_idx); + cfg.domain_id = -1; + cfg.domain_name = ""; + break; + } + } + } +} + static void ggml_hexagon_init(ggml_backend_reg * reg) { // Basic sanity checks to make sure definitions match static_assert((unsigned int) HTP_TYPE_Q4_0 == (unsigned int) GGML_TYPE_Q4_0, @@ -6101,6 +6079,9 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { } #endif + // Resolve domain info for all configured devices + ggml_hexagon_discover_devices(); + if (str_profile) { opt_pmu_evt = [&]() -> std::vector { auto v = str_to_vec(str_profile);