From 486de5d8b0a1c0bc07d332fff5a1ff0a6eb03209 Mon Sep 17 00:00:00 2001 From: Jiajia Qin Date: Fri, 22 May 2026 15:32:47 +0800 Subject: [PATCH 1/8] Forward user provider options to allocator init session EnsureDeviceOrtInit creates a dummy ORT session to obtain a device allocator before the real user session. For WebGPU, this first session creates the WebGpuContext singleton whose Dawn device toggles are immutable after creation (via std::call_once). Previously the dummy session used empty provider options, so the context was always created with the default validation_mode=Basic, ignoring the user's validationMode=disabled from genai_config.json. Forward the user's provider options to the dummy session so that singleton resources are initialized with the correct settings. enableGraphCapture is excluded since the trivial model cannot use it. --- src/models/model.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/models/model.cpp b/src/models/model.cpp index 8a4c248cf5..97f25b1994 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -433,7 +433,24 @@ void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config) { // Create an OrtSessionOptions and set the options to use the DeviceType we're using here auto session_options = OrtSessionOptions::Create(); std::vector provider_options_list; - provider_options_list.emplace_back(Config::ProviderOptions{device_type_names[static_cast(type)], {}}); + const char* provider_name = device_type_names[static_cast(type)]; + Config::ProviderOptions dummy_provider_options{provider_name, {}}; + + // Forward the user's provider options (e.g. validationMode) to the dummy session + // so that singleton resources (like WebGpuContext) are initialized with the correct settings. + // Exclude enableGraphCapture since the trivial model can't use graph capture. + for (const auto& user_po : config.model.decoder.session_options.provider_options) { + if (user_po.name == provider_name) { + for (const auto& opt : user_po.options) { + if (opt.first != "enableGraphCapture") { + dummy_provider_options.options.emplace_back(opt); + } + } + break; + } + } + + provider_options_list.emplace_back(std::move(dummy_provider_options)); // QnnHtpShared is a special case. This allocator is only made available when the provider option // 'enable_htp_shared_memory_allocator' is set to 1. if (type == DeviceType::QNN) { From 2ee1529829a7bb4d72d486c957a38ed48f6ca01a Mon Sep 17 00:00:00 2001 From: Jiajia Qin Date: Fri, 22 May 2026 15:50:03 +0800 Subject: [PATCH 2/8] Forward device filtering options to allocator init session Also copy device_filtering_options from user's provider options so the dummy session selects the same EP device as the real session. --- src/models/model.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/models/model.cpp b/src/models/model.cpp index 97f25b1994..bba917ceaa 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -446,6 +446,7 @@ void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config) { dummy_provider_options.options.emplace_back(opt); } } + dummy_provider_options.device_filtering_options = user_po.device_filtering_options; break; } } From 615b98b5ea3c983732599314ea2854575a3cfcce Mon Sep 17 00:00:00 2001 From: Jiajia Qin Date: Fri, 22 May 2026 16:05:14 +0800 Subject: [PATCH 3/8] Scope provider option forwarding to WebGPU and add missing declaration Limit the provider option forwarding in EnsureDeviceOrtInit to WebGPU only, since the singleton context issue is WebGPU-specific. Also add missing is_nemotron_speech_model_ declaration from main merge. --- src/generators.h | 1 + src/models/model.cpp | 16 +++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/generators.h b/src/generators.h index 89eebffb7b..532dd0a9ec 100644 --- a/src/generators.h +++ b/src/generators.h @@ -123,6 +123,7 @@ struct Generator : LeakChecked { bool computed_logits_{}; // Set to true in ComputeLogits() and false after appending a token to ensure a 1 to 1 call ratio bool set_extra_inputs_{true}; // Set to false once SetExtraInputs() is called once + bool is_nemotron_speech_model_{}; private: DeviceSpan AllocateInputIdsOnDevice(cpu_span input_ids); diff --git a/src/models/model.cpp b/src/models/model.cpp index b0984194c8..0658840f3e 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -441,15 +441,17 @@ void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config) { // Forward the user's provider options (e.g. validationMode) to the dummy session // so that singleton resources (like WebGpuContext) are initialized with the correct settings. // Exclude enableGraphCapture since the trivial model can't use graph capture. - for (const auto& user_po : config.model.decoder.session_options.provider_options) { - if (user_po.name == provider_name) { - for (const auto& opt : user_po.options) { - if (opt.first != "enableGraphCapture") { - dummy_provider_options.options.emplace_back(opt); + if (type == DeviceType::WEBGPU) { + for (const auto& user_po : config.model.decoder.session_options.provider_options) { + if (user_po.name == provider_name) { + for (const auto& opt : user_po.options) { + if (opt.first != "enableGraphCapture") { + dummy_provider_options.options.emplace_back(opt); + } } + dummy_provider_options.device_filtering_options = user_po.device_filtering_options; + break; } - dummy_provider_options.device_filtering_options = user_po.device_filtering_options; - break; } } From 94b92e149e4e4a9bbb41ab1fbe2500a4db0aecd7 Mon Sep 17 00:00:00 2001 From: Jiajia Qin Date: Tue, 26 May 2026 09:32:21 +0800 Subject: [PATCH 4/8] Remove unused is_nemotron_speech_model_ field after merging main --- src/generators.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/generators.h b/src/generators.h index 532dd0a9ec..89eebffb7b 100644 --- a/src/generators.h +++ b/src/generators.h @@ -123,7 +123,6 @@ struct Generator : LeakChecked { bool computed_logits_{}; // Set to true in ComputeLogits() and false after appending a token to ensure a 1 to 1 call ratio bool set_extra_inputs_{true}; // Set to false once SetExtraInputs() is called once - bool is_nemotron_speech_model_{}; private: DeviceSpan AllocateInputIdsOnDevice(cpu_span input_ids); From c54404f88eaed0ea378a169441cac7d0364c1212 Mon Sep 17 00:00:00 2001 From: Jiajia Qin Date: Mon, 1 Jun 2026 15:52:18 +0800 Subject: [PATCH 5/8] Forward only global/singleton WebGPU options in EnsureDeviceOrtInit Change the dummy session's provider option forwarding from a blocklist (skip enableGraphCapture) to a whitelist of 7 global/singleton options (deviceId, webgpuInstance, webgpuDevice, dawnBackendType, powerPreference, validationMode, dawnProcTable). Per-session options like enableGraphCapture, bufferCacheMode, enablePIXCapture are excluded because they are meaningless for the trivial initialization model. --- src/models/model.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/models/model.cpp b/src/models/model.cpp index f2c3157a0c..82030974bb 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -409,14 +409,20 @@ void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config) { const char* provider_name = device_type_names[static_cast(type)]; Config::ProviderOptions dummy_provider_options{provider_name, {}}; - // Forward the user's provider options (e.g. validationMode) to the dummy session - // so that singleton resources (like WebGpuContext) are initialized with the correct settings. - // Exclude enableGraphCapture since the trivial model can't use graph capture. + // Forward only global/singleton WebGPU options to the dummy session so that the + // process-wide WebGpuContext singleton is initialized with the correct settings. + // Per-session options (enableGraphCapture, bufferCacheMode, etc.) are excluded + // because they are meaningless for the trivial initialization model. if (type == DeviceType::WEBGPU) { + static const std::unordered_set global_options = { + "deviceId", "webgpuInstance", "webgpuDevice", + "dawnBackendType", "powerPreference", + "validationMode", "dawnProcTable", + }; for (const auto& user_po : config.model.decoder.session_options.provider_options) { if (user_po.name == provider_name) { for (const auto& opt : user_po.options) { - if (opt.first != "enableGraphCapture") { + if (global_options.count(opt.first)) { dummy_provider_options.options.emplace_back(opt); } } From 47f4917f5a52a42990133b37610b5e248a089015 Mon Sep 17 00:00:00 2001 From: Jiajia Qin Date: Tue, 2 Jun 2026 09:27:58 +0800 Subject: [PATCH 6/8] Fix clang-format: one initializer entry per line --- src/models/model.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/models/model.cpp b/src/models/model.cpp index 82030974bb..ae3b473f1d 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -415,9 +415,13 @@ void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config) { // because they are meaningless for the trivial initialization model. if (type == DeviceType::WEBGPU) { static const std::unordered_set global_options = { - "deviceId", "webgpuInstance", "webgpuDevice", - "dawnBackendType", "powerPreference", - "validationMode", "dawnProcTable", + "deviceId", + "webgpuInstance", + "webgpuDevice", + "dawnBackendType", + "powerPreference", + "validationMode", + "dawnProcTable", }; for (const auto& user_po : config.model.decoder.session_options.provider_options) { if (user_po.name == provider_name) { From 7991db8b6a0c9aa8c64e9db7a411f136fab9b4e0 Mon Sep 17 00:00:00 2001 From: Jiajia Qin Date: Thu, 11 Jun 2026 09:53:32 +0800 Subject: [PATCH 7/8] Use constexpr std::array for WebGPU global options whitelist Replace the function-local static std::unordered_set with a constexpr std::array and look up keys via std::find. Removes the hash-table allocation and uses zero-copy string_view comparison against the literal keys. Rename dummy_provider_options to init_session_provider_options so the variable reflects its role (provider options for the allocator init session) and matches the vocabulary used in the PR description. --- src/models/model.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/models/model.cpp b/src/models/model.cpp index ae3b473f1d..bd66f76560 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -4,10 +4,12 @@ // Modifications Copyright (C) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. // Portions of this file consist of AI generated content. #include +#include #include #include #include #include +#include #include #include "../generators.h" @@ -407,14 +409,14 @@ void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config) { auto session_options = OrtSessionOptions::Create(); std::vector provider_options_list; const char* provider_name = device_type_names[static_cast(type)]; - Config::ProviderOptions dummy_provider_options{provider_name, {}}; + Config::ProviderOptions init_session_provider_options{provider_name, {}}; // Forward only global/singleton WebGPU options to the dummy session so that the // process-wide WebGpuContext singleton is initialized with the correct settings. // Per-session options (enableGraphCapture, bufferCacheMode, etc.) are excluded // because they are meaningless for the trivial initialization model. if (type == DeviceType::WEBGPU) { - static const std::unordered_set global_options = { + constexpr std::array kWebGpuGlobalOptions = { "deviceId", "webgpuInstance", "webgpuDevice", @@ -426,17 +428,17 @@ void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config) { for (const auto& user_po : config.model.decoder.session_options.provider_options) { if (user_po.name == provider_name) { for (const auto& opt : user_po.options) { - if (global_options.count(opt.first)) { - dummy_provider_options.options.emplace_back(opt); + if (std::find(kWebGpuGlobalOptions.begin(), kWebGpuGlobalOptions.end(), opt.first) != kWebGpuGlobalOptions.end()) { + init_session_provider_options.options.emplace_back(opt); } } - dummy_provider_options.device_filtering_options = user_po.device_filtering_options; + init_session_provider_options.device_filtering_options = user_po.device_filtering_options; break; } } } - provider_options_list.emplace_back(std::move(dummy_provider_options)); + provider_options_list.emplace_back(std::move(init_session_provider_options)); // QnnHtpShared is a special case. This allocator is only made available when the provider option // 'enable_htp_shared_memory_allocator' is set to 1. if (type == DeviceType::QNN) { From 729eff942a446722a619ab4befcf902048523c80 Mon Sep 17 00:00:00 2001 From: Jiajia Qin Date: Thu, 11 Jun 2026 10:02:14 +0800 Subject: [PATCH 8/8] Update comment after dummy_provider_options rename The block comment above kWebGpuGlobalOptions still referred to the dummy session and cited bufferCacheMode, which is not a real per-session option key. Update both to match the new init_session_provider_options name and use enableInt64 as a real per-session example. --- src/models/model.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/models/model.cpp b/src/models/model.cpp index bd66f76560..daf4d18c51 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -411,9 +411,9 @@ void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config) { const char* provider_name = device_type_names[static_cast(type)]; Config::ProviderOptions init_session_provider_options{provider_name, {}}; - // Forward only global/singleton WebGPU options to the dummy session so that the + // Forward only global/singleton WebGPU options to the init session so that the // process-wide WebGpuContext singleton is initialized with the correct settings. - // Per-session options (enableGraphCapture, bufferCacheMode, etc.) are excluded + // Per-session options (enableGraphCapture, enableInt64, etc.) are excluded // because they are meaningless for the trivial initialization model. if (type == DeviceType::WEBGPU) { constexpr std::array kWebGpuGlobalOptions = {