Skip to content
10 changes: 10 additions & 0 deletions src/models/kv_cache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,16 @@ DefaultKeyValueCache::DefaultKeyValueCache(State& state)
shape_[2] = state_.params_->search.max_length;
}

if (past_present_share_buffer_) { // AMD-specific override
for (const auto& [key, val] : state_.params_->config.model.decoder.session_options.config_entries) {
if (key == "max_length_for_kv_cache") try {
Comment thread
BoarQing marked this conversation as resolved.
Outdated
shape_[2] = std::atoi(val.c_str());
} catch (const std::exception& ex) {
throw std::runtime_error{std::string{"config_entries/max_length_for_kv_cache format error: "} + ex.what()};
}
}
}

try {
// Allocate KV cache tensors - 2 per layer (key and value)
// For per-layer shapes: alternates between key and value for each layer
Expand Down
22 changes: 22 additions & 0 deletions src/models/model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,7 @@ DeviceInterface* SetProviderSessionOptions(OrtSessionOptions& session_options,
else if (provider_options.name == "VitisAI") {
session_options.AddConfigEntry("session.inter_op.allow_spinning", "0");
session_options.AddConfigEntry("session.intra_op.allow_spinning", "0");
session_options.AddConfigEntry("model_root", config.config_path.string().c_str());
} else if (provider_options.name == "NvTensorRtRtx") {
bool is_multi_profile_enabled = IsMultiProfileEnabled(config.model.decoder.session_options);
ConfigureNvTensorRtRtxProfile(config, session_options, is_multi_profile_enabled);
Expand Down Expand Up @@ -812,6 +813,27 @@ DeviceInterface* SetProviderSessionOptions(OrtSessionOptions& session_options,
values.emplace_back(option.second.c_str());
}
session_options.AppendExecutionProvider(provider_options.name.c_str(), keys.data(), values.data(), keys.size());
if (provider_options.name == "VitisAI") {
if (const auto opt_it = std::find_if(provider_options.options.begin(), provider_options.options.end(),
[](const auto& pair) { return pair.first == "external_ep_libray"; });
Comment thread
BoarQing marked this conversation as resolved.
opt_it != provider_options.options.end()) {
auto lib_name = opt_it->second;
auto lib = LoadLibrary(lib_name.c_str());
if (const auto func = (void (*)(void*, const OrtApiBase*, void*, OrtEpFactory**, size_t, size_t*))GetProcAddress(lib, "CreateEpFactories")) {
OrtEpFactory* factory = nullptr;
size_t num = 1;

func(nullptr, OrtGetApiBase(), nullptr, &factory, num, &num);
}
if (const auto func = (void (*)(OrtSessionOptions*))GetProcAddress(lib, "RyzenAI_SetSessionOptions"))
func(&session_options);

if (const auto api = OrtGetApiBase()->GetApi(ORT_API_VERSION)) {
void* unused;
Comment thread
BoarQing marked this conversation as resolved.
Outdated
api->RegisterCustomOpsLibrary(&session_options, lib_name.c_str(), &unused);
}
}
}

#endif
}
Expand Down
Loading