diff --git a/src/models/kv_cache.cpp b/src/models/kv_cache.cpp
index 60c7195d0a..dbdbd828d6 100644
--- a/src/models/kv_cache.cpp
+++ b/src/models/kv_cache.cpp
@@ -176,7 +176,7 @@ DefaultKeyValueCache::DefaultKeyValueCache(State& state)
   }
 
   // Set the size after empty_past_ has been created with 0 for this field
-  if (model_.config_->model.decoder.sliding_window.has_value() &&
+  if (state_.model_.p_device_->GetType() == DeviceType::NvTensorRtRtx && model_.config_->model.decoder.sliding_window.has_value() &&
       model_.config_->model.decoder.sliding_window->window_size > 0) {
     const int sliding_window_size = model_.config_->model.decoder.sliding_window->window_size;
     const int max_length = state_.params_->search.max_length;