Skip to content

Updated load_config mapping to make it a pass through for OV properties #782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MayureshV1 merged 1 commit into from
Aug 22, 2025
Merged

Updated load_config mapping to make it a pass through for OV properties #782

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 16 additions & 94 deletions onnxruntime/core/providers/openvino/backends/basic_backend.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,107 +214,29 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) {
if (!session_context_.load_config.empty()) {
const std::map<std::string, ov::AnyMap>& target_config = session_context_.load_config;

if ((session_context_.device_type.find("NPU") != std::string::npos) && session_context_.enable_causallm) {
if (target_config.find("NPU") != target_config.end()) {
auto npu_genai_config = target_config.at("NPU");
CausalLMConfig().ApplyConfig(npu_genai_config, device_config);
} else {
LOGS_DEFAULT(WARNING) << "ORT GenAI CausalLMConfig Configuration not found.";
}
}

if (session_context_.device_type.find("NPU") != std::string::npos) {
auto npuw_config = target_config.at("NPU");

// Check if "NPU_USE_NPUW" exists and is set to "YES"
auto npu_use_npuw_it = npuw_config.find("NPU_USE_NPUW");
if (npu_use_npuw_it != npuw_config.end() &&
npu_use_npuw_it->second.is<std::string>() &&
npu_use_npuw_it->second.as<std::string>() == "YES") {
// Only add NPUW-related keys if NPU_USE_NPUW is "YES"
for (const auto& [key, value] : npuw_config) {
if (key.find("NPUW") != std::string::npos) {
if (!value.is<std::string>()) {
LOGS_DEFAULT(ERROR) << "Invalid value type for key: " << key;
continue;
}
device_config[key] = value;
}
}
} else {
// Check if there are any "NPUW" keys and log a warning
if (std::any_of(npuw_config.begin(), npuw_config.end(),
[&](const auto& pair) { return pair.first.find("NPUW") != std::string::npos; })) {
LOGS_DEFAULT(WARNING) << "Skipping NPUW-related configurations as NPU_USE_NPUW is not set to 'YES'.";
}
}
}
auto find_device_type_mode = [&](const std::string& device_type) -> std::string {
std::string device_mode = "";
auto delimiter_pos = device_type.find(':');
if (delimiter_pos != std::string::npos) {
std::stringstream str_stream(device_type.substr(0, delimiter_pos));
std::getline(str_stream, device_mode, ',');
}
return device_mode;
};
// Extract device names from device string and apply their configs
// Examples: "GPU" -> ["GPU"], "AUTO:GPU.0,CPU" -> ["AUTO", "GPU", "CPU"]
auto apply_device_config = [&](std::string_view device) {
if (device.empty()) return;

// Parse device types like "AUTO:CPU,GPU" and extract individual devices
auto parse_individual_devices = [&](const std::string& device_type) -> std::vector<std::string> {
std::vector<std::string> devices;
auto delimiter_pos = device_type.find(':');
if (delimiter_pos != std::string::npos) {
std::stringstream str_stream(device_type.substr(delimiter_pos + 1));
std::string device;
while (std::getline(str_stream, device, ',')) {
devices.emplace_back(device);
}
} else {
devices.emplace_back(device_type);
}
return devices;
};
// Remove device index: "GPU.0" -> "GPU"
auto base_device = device.substr(0, device.find('.'));

// Set properties, Validation will be handled by OpenVINO Core
auto set_target_properties = [&](const std::string& device, const ov::AnyMap& config_options) {
for (const auto& [key, value] : config_options) {
// Update the device_config map from the target_config to avoid load_config being overridden
// by the device_config set by the OpenVINO EP.
auto it = device_config.find(key);
if (it != device_config.end()) {
it->second = value;
}
if ((key.find("NPUW") != std::string::npos) ||
((it != device_config.end()) && session_context_.enable_causallm)) {
continue;
if (auto config_it = target_config.find(std::string(base_device)); config_it != target_config.end()) {
for (const auto& [key, value] : config_it->second) {
device_config[key] = value;
}
OVCore::Get()->core.set_property(device, ov::AnyMap{{key, value}});
}
};

// Check if the device type is AUTO, HETERO, or MULTI
if (session_context_.device_type.find("AUTO") == 0 ||
session_context_.device_type.find("HETERO") == 0 ||
session_context_.device_type.find("MULTI") == 0) {
//// Parse to get the device mode (e.g., "AUTO:CPU,GPU" -> "AUTO")
std::unordered_set<std::string> supported_mode = {"AUTO", "HETERO", "MULTI"};
auto device_mode = find_device_type_mode(session_context_.device_type);
ORT_ENFORCE(supported_mode.find(device_mode) != supported_mode.end(), " Invalid device mode is passed : ", session_context_.device_type);
// Parse individual devices (e.g., "AUTO:CPU,GPU" -> ["CPU", "GPU"])
auto individual_devices = parse_individual_devices(session_context_.device_type);
if (!device_mode.empty()) individual_devices.emplace_back(device_mode);

// Set properties only for individual devices (e.g., "CPU", "GPU")
for (const std::string& device : individual_devices) {
if (target_config.count(device)) {
// Set properties for the device
set_target_properties(device, target_config.at(device));
// Parse device string by splitting on ':' and ',' delimiters
const auto& device_str = session_context_.device_type;
for (size_t start = 0, pos = 0; pos <= device_str.size(); ++pos) {
if (pos == device_str.size() || device_str[pos] == ':' || device_str[pos] == ',') {
if (pos > start) {
apply_device_config(std::string_view(device_str).substr(start, pos - start));
}
}
} else {
if (target_config.count(session_context_.device_type)) {
set_target_properties(session_context_.device_type,
target_config.at(session_context_.device_type));
start = pos + 1;
}
}
}
Expand Down
Loading