Enable CUDA graph capture for CUDA EP to improve decode throughput

src/config.cpp

@@ -1377,12 +1377,12 @@ bool IsGraphCaptureEnabled(const Config::SessionOptions& session_options) {
                                                });
     if (provider_options != session_options.provider_options.end()) {
       if (provider_options->name == "cuda") {
-        // Graph Capture is currently broken for CUDA
         for (const auto& value : provider_options->options) {
           if (value.first == "enable_cuda_graph" && value.second == "1") {
-            throw std::runtime_error("Graph Capture is currently unsupported for CUDA");
+            return true;
           }
         }
+        return false;
       } else if (provider_options->name == "DML") {
         return true;
       } else if (provider_options->name == "WebGPU") {

src/cuda/session_options.cpp

@@ -81,15 +81,27 @@ void AddCudaStreamConfig(OrtSessionOptions& session_options, DeviceInterface* de
 DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options,
                                          const Config::ProviderOptions& provider_options,
                                          const Config& /*config*/,
-                                         bool /*disable_graph_capture*/) {
+                                         bool disable_graph_capture) {
   auto device = GetDeviceInterface(DeviceType::CUDA);
   AddCudaStreamConfig(session_options, device);
+
+  // For non-decoder sessions (vision, embedding), disable CUDA graph capture
+  // since they have dynamic shapes incompatible with graph capture.
+  Config::ProviderOptions effective_options = provider_options;
+  if (disable_graph_capture) {
+    for (auto& option : effective_options.options) {
+      if (option.first == "enable_cuda_graph") {
+        option.second = "0";
+      }
+    }
+  }
+
   // Try pre-registered plugin path first
-  if (!AppendExecutionProviderV2(session_options, provider_options,
+  if (!AppendExecutionProviderV2(session_options, effective_options,
                                  DeviceType::CUDA, "CUDAExecutionProvider")) {
     // Register the CUDA execution provider as a provider-bridge provider.
     CUDAExecutionProvider::AppendProviderBridgeExecutionProvider(
-        session_options, provider_options, device);
+        session_options, effective_options, device);
   }
 
   return device;

src/models/recurrent_state.cpp

@@ -125,10 +125,22 @@ void RecurrentState::Update() {
   if (layer_indices_.empty()) return;
 
   const int num_layers = static_cast<int>(layer_indices_.size());
-  for (int i = 0; i < num_layers * 2; ++i) {
-    std::swap(pasts_[i], presents_[i]);
-    state_.inputs_[input_index_ + i] = pasts_[i].get();
-    state_.outputs_[output_index_ + i] = presents_[i].get();
+
+  if (state_.params_->use_graph_capture) {
+    // When CUDA graph capture is enabled, we must not swap pointers because
+    // the graph has captured the original memory addresses. Instead, copy
+    // present→past in-place so the pointers remain stable.
+    auto& device = *model_.p_device_kvcache_;
+    for (int i = 0; i < num_layers * 2; ++i) {
+      ByteWrapTensor(device, *pasts_[i]).CopyFrom(ByteWrapTensor(device, *presents_[i]));
+    }
+    // No need to rebind state_.inputs_/outputs_ — pointers are unchanged.
+  } else {
+    for (int i = 0; i < num_layers * 2; ++i) {
+      std::swap(pasts_[i], presents_[i]);
+      state_.inputs_[input_index_ + i] = pasts_[i].get();
+      state_.outputs_[output_index_ + i] = presents_[i].get();
+    }
   }
 }