diff --git a/cmake/onnxruntime_providers_vitisai.cmake b/cmake/onnxruntime_providers_vitisai.cmake
index d40ae17e40545..d59c944c8926f 100644
--- a/cmake/onnxruntime_providers_vitisai.cmake
+++ b/cmake/onnxruntime_providers_vitisai.cmake
@@ -19,7 +19,16 @@
     "${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.cc"
   )
   source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_vitisai_cc_srcs})
-  onnxruntime_add_shared_library(onnxruntime_providers_vitisai ${onnxruntime_providers_vitisai_cc_srcs})
+  set(onnxruntime_providers_vitisai_all_srcs ${onnxruntime_providers_vitisai_cc_srcs})
+  if(WIN32)
+    # Sets the DLL version info on Windows: https://learn.microsoft.com/en-us/windows/win32/menurc/versioninfo-resource
+    list(APPEND onnxruntime_providers_vitisai_all_srcs "${ONNXRUNTIME_ROOT}/core/providers/vitisai/onnxruntime_providers_vitisai.rc")
+  endif()
+  onnxruntime_add_shared_library(onnxruntime_providers_vitisai ${onnxruntime_providers_vitisai_all_srcs})
+  if(WIN32)
+    # FILE_NAME preprocessor definition is used in onnxruntime_providers_vitisai.rc
+    target_compile_definitions(onnxruntime_providers_vitisai PRIVATE FILE_NAME=\"onnxruntime_providers_vitisai.dll\")
+  endif()
   onnxruntime_add_include_to_target(onnxruntime_providers_vitisai ${ONNXRUNTIME_PROVIDERS_SHARED} ${GSL_TARGET} safeint_interface flatbuffers::flatbuffers  Boost::mp11)
   target_link_libraries(onnxruntime_providers_vitisai PRIVATE ${ONNXRUNTIME_PROVIDERS_SHARED} ${ABSEIL_LIBS})
   if(MSVC)
diff --git a/onnxruntime/contrib_ops/cpu/bert/attention_base.cc b/onnxruntime/contrib_ops/cpu/bert/attention_base.cc
deleted file mode 100644
index 651f270230a75..0000000000000
--- a/onnxruntime/contrib_ops/cpu/bert/attention_base.cc
+++ /dev/null
@@ -1,341 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License.
-
-#include "contrib_ops/cpu/bert/attention_base.h"
-#include "contrib_ops/cpu/bert/multihead_attention_helper.h"
-#include "core/providers/common.h"
-
-namespace onnxruntime {
-namespace contrib {
-
-Status AttentionBase::CheckInputs(const TensorShape& input_shape,
-                                  const TensorShape& weights_shape,
-                                  const TensorShape& bias_shape,
-                                  const Tensor*& mask_index,
-                                  const Tensor* past,
-                                  const Tensor* attention_bias,
-                                  void* parameters,
-                                  const Tensor* past_seq_len) const {
-  // Abbreviation and Meanings:
-  //   B:    batch_size
-  //   S:    sequence_length (input sequence length of query)
-  //   P:    past_sequence_length (past sequence length of key or value)
-  //   L:    kv_sequence_length (input sequence length of key or value)
-  //   M:    max_sequence_length
-  //   T:    total_sequence_length = past_sequence_length + kv_sequence_length
-  //   N:    num_heads
-  //   H:    head size for Q and K, aka q_head_size or k_head_size or qk_head_size
-  //   H_v:  v_head_size
-  //   D_i:  input hidden size
-  //   D:    hidden size for Q and K (D = N * H), aka q_hidden_size or k_hidden_size or qk_hidden_size
-  //   D_v:  v_hidden_size = num_heads * v_head_size
-
-  // When past state is used, Q, K and V should have same hidden size (unless we split it into past_key and past_value).
-
-  // Input shapes:
-  //   input        (Q/K/V)    : (B, S, D_i)
-  //   weights      (Q/K/V)    : (D_i, D + D + D_v)
-  //   bias         (Q/K/V)    : (D + D + D_v)
-  //   mask_index              : see below
-  //   past         (K/V)      : (2, B, N, P, H) or NULL
-  //   attention_bias          : (B or 1, N or 1, S, T) or NULL
-
-  // For mask_index, the following shapes are supported:
-  //     NULL, (B, 1), (1, 1)
-  //     (B), (2 * B), (3 * B + 2)
-  //     (B, T)
-  //     (B, S, T)
-  //     (B, 1, M, M)
-  //
-  // When a model is pruned (like some attention heads are removed in Q/K/V), input_hidden_size could be larger
-  // than hidden dimension of Q, K and V.
-
-  if (past != nullptr && attention_bias != nullptr) {
-    // past is used on GPT-2 model with past state, we don't have a case for attention bias yet
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Attention cannot have both past and attention_bias");
-  }
-
-  const auto& dims = input_shape.GetDims();
-  if (dims.size() != 3) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'input' is expected to have 3 dimensions, got ",
-                           dims.size());
-  }
-
-  auto& batch_size = dims[0];
-  auto& sequence_length = dims[1];
-  int64_t input_hidden_size = dims[2];
-
-  const auto& bias_dims = bias_shape.GetDims();
-  if (bias_dims.size() != 1) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'bias' is expected to have 1 dimension, got ",
-                           bias_dims.size());
-  }
-
-  const auto& weights_dims = weights_shape.GetDims();
-  if (weights_dims.size() != 2) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'weights' is expected to have 2 dimensions, got ",
-                           weights_dims.size());
-  }
-  if (weights_dims[0] != input_hidden_size) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                           "Input 1 dimension 0 should have same length as dimension 2 of input 0");
-  }
-
-  if (bias_dims[0] != weights_dims[1]) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                           "Input 'bias' dimension 0 should have same length as dimension 1 of input 'weights'");
-  }
-
-  int64_t q_hidden_size = bias_dims[0] / static_cast<int64_t>(3);
-  int64_t k_hidden_size = q_hidden_size;
-  int64_t v_hidden_size = k_hidden_size;
-  if (qkv_hidden_sizes_.size() != 0) {
-    if (qkv_hidden_sizes_.size() != 3) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "qkv_hidden_sizes attribute should have 3 elements");
-    }
-
-    for (size_t i = 0; i < qkv_hidden_sizes_.size(); i++) {
-      if (qkv_hidden_sizes_[i] % num_heads_ != 0) {
-        return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                               "hidden_size should be divisible by num_heads:", qkv_hidden_sizes_[i]);
-      }
-    }
-
-    q_hidden_size = qkv_hidden_sizes_[0];
-    k_hidden_size = qkv_hidden_sizes_[1];
-    v_hidden_size = qkv_hidden_sizes_[2];
-  }
-
-  int64_t kv_sequence_length = sequence_length;
-
-  if (q_hidden_size != k_hidden_size) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                           "qkv_hidden_sizes first element should be same as the second");
-  }
-
-  if (this->require_same_hidden_size_ && k_hidden_size != v_hidden_size) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Hidden size of Q, K and V shall be same");
-  }
-
-  if (bias_dims[0] != q_hidden_size + k_hidden_size + v_hidden_size) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                           "Input 'bias' dimension 0 should have same length as sum of Q/K/V hidden sizes:",
-                           " q_hidden_size=", q_hidden_size, " k_hidden_size=", k_hidden_size, " v_hidden_size=",
-                           v_hidden_size, "bias_dims[0]=", bias_dims[0]);
-  }
-
-  int64_t past_sequence_length = 0;
-  if (past != nullptr) {  // past is optional
-    if (k_hidden_size != v_hidden_size) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'past' expect k_hidden_size == v_hidden_size");
-    }
-
-    const auto& past_dims = past->Shape().GetDims();
-    if (past_dims.size() != 5) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'past' is expected to have 5 dimension, got ",
-                             past_dims.size());
-    }
-
-    if (past_dims[0] != 2) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'past' dimension 0 shall have length of 2");
-    }
-
-    if (past_dims[1] != batch_size) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "Inputs 'past' dimension 1 shall have same length as dimension 0 of input 0");
-    }
-
-    if (static_cast<int>(past_dims[2]) != num_heads_) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "Inputs 'past' dimension 2 shall have length of num_heads", num_heads_);
-    }
-
-    if (static_cast<int>(past_dims[4]) != k_hidden_size / num_heads_) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "Inputs 'past' dimension 2 shall have length of ", k_hidden_size / num_heads_);
-    }
-
-    if (!past_present_share_buffer_) {
-      past_sequence_length = past_dims[3];
-    } else {
-      if (past_seq_len == nullptr || !onnxruntime::IsScalarOr1ElementVector(past_seq_len)) {
-        return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                               "past_sequence_length tensor must be of one element when past_present_share_buffer is set");
-      }
-      past_sequence_length = *past_seq_len->Data<int32_t>();
-    }
-  }
-
-  int64_t total_sequence_length = kv_sequence_length + past_sequence_length;
-  if (past != nullptr && past_present_share_buffer_) {
-    const auto& past_dims = past->Shape().GetDims();
-    if (past_dims[3] < total_sequence_length) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "when past_present_share_buffer, past tensor sequence must not smaller than total_sequqnce_length ");
-    }
-  }
-
-  int64_t max_sequence_length = -1;
-  AttentionMaskType mask_type = AttentionMaskType::MASK_NONE;
-  if (mask_index != nullptr) {  // mask_index is optional
-    mask_type = AttentionMaskType::MASK_UNKNOWN;
-    auto status = this->CheckMask(mask_index, mask_type,
-                                  max_sequence_length, batch_size, sequence_length, total_sequence_length);
-    if (status != Status::OK()) {
-      return status;
-    }
-
-    if (mask_type == AttentionMaskType::MASK_2D_DUMMY) {
-      mask_index = nullptr;
-      mask_type = AttentionMaskType::MASK_NONE;
-    }
-  }
-
-  gsl::span<const int64_t> attention_bias_dims;
-  if (attention_bias != nullptr) {
-    attention_bias_dims = attention_bias->Shape().GetDims();
-
-    ORT_RETURN_IF_ERROR(multihead_attention_helper::CheckAttentionBias(
-        attention_bias_dims, batch_size, num_heads_, sequence_length, total_sequence_length));
-  }
-
-  if (past != nullptr && past_present_share_buffer_) {
-    if (max_sequence_length <= 0) {
-      max_sequence_length = past->Shape().GetDims()[3];
-    }
-    if (max_sequence_length != past->Shape().GetDims()[3]) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "max_sequence_length not matching from mask and past when past_present_share_buffer_ is set");
-    }
-  }
-
-  if (parameters != nullptr) {
-    AttentionParameters* output_parameters = reinterpret_cast<AttentionParameters*>(parameters);
-    output_parameters->batch_size = static_cast<int>(batch_size);
-    output_parameters->sequence_length = static_cast<int>(sequence_length);
-    output_parameters->past_sequence_length = static_cast<int>(past_sequence_length);
-    output_parameters->kv_sequence_length = static_cast<int>(kv_sequence_length);
-    output_parameters->total_sequence_length = static_cast<int>(total_sequence_length);
-    output_parameters->max_sequence_length = static_cast<int>(max_sequence_length);
-    output_parameters->input_hidden_size = static_cast<int>(input_hidden_size);
-    output_parameters->hidden_size = static_cast<int>(q_hidden_size);
-    output_parameters->v_hidden_size = static_cast<int>(v_hidden_size);
-    output_parameters->head_size = static_cast<int>(q_hidden_size) / num_heads_;
-    output_parameters->v_head_size = static_cast<int>(v_hidden_size) / num_heads_;
-    output_parameters->num_heads = num_heads_;
-    output_parameters->is_unidirectional = is_unidirectional_;
-    output_parameters->past_present_share_buffer = (past_present_share_buffer_ != 0 && past != nullptr);
-    output_parameters->do_rotary = do_rotary_;
-    output_parameters->rotary_dim = rotary_embedding_ == 0 ? (int)(output_parameters->head_size) : rotary_embedding_;
-    output_parameters->mask_filter_value = mask_filter_value_;
-    output_parameters->scale = scale_;
-    output_parameters->mask_type = mask_type;
-    output_parameters->broadcast_attn_bias_dim_0 = attention_bias_dims.size() > 0 && attention_bias_dims[0] == 1;
-    output_parameters->broadcast_attn_bias_dim_1 = attention_bias_dims.size() > 1 && attention_bias_dims[1] == 1;
-    output_parameters->qkv_format = Q_K_V_BNSH;
-  }
-
-  return Status::OK();
-}
-
-Status AttentionBase::CheckMask(const Tensor* mask_index,
-                                AttentionMaskType& mask_type,
-                                int64_t& max_sequence_length,
-                                int64_t batch_size,
-                                int64_t sequence_length,
-                                int64_t total_sequence_length) const {
-  const auto& mask_dims = mask_index->Shape().GetDims();
-  if (mask_dims.size() == 1) {
-    if (mask_dims[0] != batch_size && mask_dims[0] != 2 * batch_size && mask_dims[0] != 3 * batch_size + 2) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "Inputs 'mask_index' with 1D data shall have length of batch_size or 2 * batch_size or 3 * batch_size + 2");
-    }
-    mask_type = (mask_dims[0] == batch_size ? AttentionMaskType::MASK_1D_KEY_SEQ_LEN : mask_dims[0] == 2 * batch_size ? AttentionMaskType::MASK_1D_END_START
-                                                                                                                      : AttentionMaskType::MASK_1D_KEY_SEQ_LEN_START);
-  } else if (mask_dims.size() == 2) {
-    if (mask_dims[0] == batch_size && mask_dims[1] == total_sequence_length) {
-      mask_type = AttentionMaskType::MASK_2D_KEY_PADDING;
-    } else {
-      // Add operator supports broadcasting. Here we handle a case with only one element in the 2nd dimension.
-      if ((mask_dims[0] == batch_size || mask_dims[0] == 1) && mask_dims[1] == 1) {
-        // Mask will have same value after propagation, which has same effect as no mask.
-        mask_type = AttentionMaskType::MASK_2D_DUMMY;
-      } else {
-        return ORT_MAKE_STATUS(
-            ONNXRUNTIME, INVALID_ARGUMENT,
-            "Inputs 'mask_index' with 2D data shall have shape "
-            "batch_size x total_sequence_length");
-      }
-    }
-  } else if (mask_dims.size() == 3) {
-    if (mask_dims[0] != batch_size || mask_dims[1] != sequence_length || mask_dims[2] != total_sequence_length) {
-      return ORT_MAKE_STATUS(
-          ONNXRUNTIME, INVALID_ARGUMENT,
-          "Inputs 'mask_index' with 3D data shall have shape "
-          "batch_size x sequence_length x total_sequence_length");
-    }
-    mask_type = AttentionMaskType::MASK_3D_ATTENTION;
-  } else if (mask_dims.size() == 4) {
-    if (mask_dims[0] != batch_size || mask_dims[1] != 1 || mask_dims[2] != mask_dims[3] ||
-        mask_dims[2] < total_sequence_length) {
-      return ORT_MAKE_STATUS(
-          ONNXRUNTIME, INVALID_ARGUMENT,
-          "Inputs 'mask_index' with 4D data shall have shape "
-          "batch_size x 1 x max_sequence_length x max_sequence_length)");
-    }
-    max_sequence_length = mask_dims[3];
-    mask_type = AttentionMaskType::MASK_4D_MEGATRON;
-    if (this->is_unidirectional_) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "Inputs 'mask_index' with 4D data shall have is_unidirectional set to false");
-    }
-  } else {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                           "Input 'mask_index' is expected to have 1, 2, 3 or 4 dimensions, got ",
-                           mask_dims.size());
-  }
-
-  return Status::OK();
-}
-
-Status AttentionBase::CheckInputs(const TensorShape& input_shape,
-                                  const TensorShape& weights_shape,
-                                  const TensorShape& bias_shape,
-                                  const Tensor*& mask_index,
-                                  const Tensor* past,
-                                  const Tensor* attention_bias,
-                                  void* parameters,
-                                  const int max_threads_per_block,
-                                  const Tensor* past_seq_len) const {
-  if (num_heads_ > max_threads_per_block) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "num_heads should be no larger than ", max_threads_per_block);
-  }
-
-  return CheckInputs(input_shape, weights_shape, bias_shape, mask_index, past, attention_bias, parameters, past_seq_len);
-}
-
-Tensor* AttentionBase::GetPresent(OpKernelContext* context,
-                                  const Tensor* past,
-                                  int batch_size,
-                                  int head_size,
-                                  int kv_sequence_length,
-                                  int& past_sequence_length) const {
-  // Input and output shapes:
-  //   past        : (2, batch_size, num_heads, past_sequence_length, head_size)
-  //   present     : (2, batch_size, num_heads, past_sequence_length + kv_sequence_length, head_size)
-
-  past_sequence_length = (nullptr != past) ? static_cast<int>(past->Shape().GetDims()[3]) : 0;
-  std::array<int64_t, 5> present_dims{2, batch_size, num_heads_, static_cast<int64_t>(kv_sequence_length) + past_sequence_length, head_size};
-
-  TensorShape present_shape(present_dims);
-  Tensor* present = context->Output(1, present_shape);
-  if (nullptr != past && nullptr == present) {
-    ORT_THROW("Expect to have present state output when past state input is given");
-  }
-
-  return present;
-}
-
-}  // namespace contrib
-}  // namespace onnxruntime
diff --git a/onnxruntime/contrib_ops/cpu/bert/attention_base.h b/onnxruntime/contrib_ops/cpu/bert/attention_base.h
index bd7f03379b2f0..2872fcfda5bbf 100644
--- a/onnxruntime/contrib_ops/cpu/bert/attention_base.h
+++ b/onnxruntime/contrib_ops/cpu/bert/attention_base.h
@@ -3,12 +3,19 @@
 
 #pragma once
 
+#include <array>
 #include <vector>
 #include "core/common/common.h"
-#include "core/framework/op_kernel.h"
 #include "core/providers/cpu/mlas_backend_kernel_selector_config_utils.h"
+#ifndef SHARED_PROVIDER
+#include "core/framework/op_kernel.h"
+#include "core/providers/common.h"
+#endif
 #include "contrib_ops/cpu/bert/attention_common.h"
 #include "contrib_ops/cpu/bert/attention_parameters.h"
+#ifndef SHARED_PROVIDER
+#include "contrib_ops/cpu/bert/multihead_attention_helper.h"
+#endif
 
 namespace onnxruntime {
 namespace contrib {
@@ -25,14 +32,25 @@ class AttentionBase {
                      const int max_threads_per_block,  // for CUDA
                      const Tensor* past_seq_len = nullptr) const;
 
+#ifdef SHARED_PROVIDER
   Tensor* GetPresent(OpKernelContext* context,
                      const Tensor* past,
                      int batch_size,
                      int head_size,
                      int kv_sequence_length,
                      int& past_sequence_length) const;
+#else
+  template <typename TOpKernelContext>
+  Tensor* GetPresent(TOpKernelContext* context,
+                     const Tensor* past,
+                     int batch_size,
+                     int head_size,
+                     int kv_sequence_length,
+                     int& past_sequence_length) const;
+#endif
 
  protected:
+  // Keep the class layout identical in SHARED_PROVIDER and non-SHARED_PROVIDER builds.
   MLAS_BACKEND_KERNEL_SELECTOR_CONFIG mlas_backend_kernel_selector_config_;
 
   template <typename KernelInfoType>
@@ -54,7 +72,9 @@ class AttentionBase {
 
     require_same_hidden_size_ = require_same_hidden_size;
 
+#ifndef SHARED_PROVIDER
     SetupMlasBackendKernelSelectorFromConfigOptions(mlas_backend_kernel_selector_config_, info.GetConfigOptions());
+#endif
   }
 
   Status CheckMask(const Tensor* mask_index,
@@ -84,5 +104,299 @@ class AttentionBase {
   float scale_;                            // the scale to be used for softmax
 };
 
+#ifndef SHARED_PROVIDER
+// Inline implementations of out-of-line methods for non-SHARED_PROVIDER builds
+// (attention_base.cc definitions are used only in the SHARED_PROVIDER bridge path).
+inline Status AttentionBase::CheckMask(const Tensor* mask_index,
+                                       AttentionMaskType& mask_type,
+                                       int64_t& max_sequence_length,
+                                       int64_t batch_size,
+                                       int64_t sequence_length,
+                                       int64_t total_sequence_length) const {
+  const auto& mask_dims = mask_index->Shape().GetDims();
+  if (mask_dims.size() == 1) {
+    if (mask_dims[0] != batch_size && mask_dims[0] != 2 * batch_size && mask_dims[0] != 3 * batch_size + 2) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "Inputs 'mask_index' with 1D data shall have length of batch_size or 2 * batch_size or 3 * batch_size + 2");
+    }
+    mask_type = (mask_dims[0] == batch_size ? AttentionMaskType::MASK_1D_KEY_SEQ_LEN : mask_dims[0] == 2 * batch_size ? AttentionMaskType::MASK_1D_END_START
+                                                                                                                      : AttentionMaskType::MASK_1D_KEY_SEQ_LEN_START);
+  } else if (mask_dims.size() == 2) {
+    if (mask_dims[0] == batch_size && mask_dims[1] == total_sequence_length) {
+      mask_type = AttentionMaskType::MASK_2D_KEY_PADDING;
+    } else {
+      if ((mask_dims[0] == batch_size || mask_dims[0] == 1) && mask_dims[1] == 1) {
+        mask_type = AttentionMaskType::MASK_2D_DUMMY;
+      } else {
+        return ORT_MAKE_STATUS(
+            ONNXRUNTIME, INVALID_ARGUMENT,
+            "Inputs 'mask_index' with 2D data shall have shape "
+            "batch_size x total_sequence_length");
+      }
+    }
+  } else if (mask_dims.size() == 3) {
+    if (mask_dims[0] != batch_size || mask_dims[1] != sequence_length || mask_dims[2] != total_sequence_length) {
+      return ORT_MAKE_STATUS(
+          ONNXRUNTIME, INVALID_ARGUMENT,
+          "Inputs 'mask_index' with 3D data shall have shape "
+          "batch_size x sequence_length x total_sequence_length");
+    }
+    mask_type = AttentionMaskType::MASK_3D_ATTENTION;
+  } else if (mask_dims.size() == 4) {
+    if (mask_dims[0] != batch_size || mask_dims[1] != 1 || mask_dims[2] != mask_dims[3] ||
+        mask_dims[2] < total_sequence_length) {
+      return ORT_MAKE_STATUS(
+          ONNXRUNTIME, INVALID_ARGUMENT,
+          "Inputs 'mask_index' with 4D data shall have shape "
+          "batch_size x 1 x max_sequence_length x max_sequence_length)");
+    }
+    max_sequence_length = mask_dims[3];
+    mask_type = AttentionMaskType::MASK_4D_MEGATRON;
+    if (this->is_unidirectional_) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "Inputs 'mask_index' with 4D data shall have is_unidirectional set to false");
+    }
+  } else {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                           "Input 'mask_index' is expected to have 1, 2, 3 or 4 dimensions, got ",
+                           mask_dims.size());
+  }
+
+  return Status::OK();
+}
+
+inline Status AttentionBase::CheckInputs(const TensorShape& input_shape,
+                                         const TensorShape& weights_shape,
+                                         const TensorShape& bias_shape,
+                                         const Tensor*& mask_index,
+                                         const Tensor* past,
+                                         const Tensor* attention_bias,
+                                         void* parameters,
+                                         const Tensor* past_seq_len) const {
+  if (past != nullptr && attention_bias != nullptr) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Attention cannot have both past and attention_bias");
+  }
+
+  const auto& dims = input_shape.GetDims();
+  if (dims.size() != 3) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'input' is expected to have 3 dimensions, got ",
+                           dims.size());
+  }
+
+  auto& batch_size = dims[0];
+  auto& sequence_length = dims[1];
+  int64_t input_hidden_size = dims[2];
+
+  const auto& bias_dims = bias_shape.GetDims();
+  if (bias_dims.size() != 1) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'bias' is expected to have 1 dimension, got ",
+                           bias_dims.size());
+  }
+
+  const auto& weights_dims = weights_shape.GetDims();
+  if (weights_dims.size() != 2) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'weights' is expected to have 2 dimensions, got ",
+                           weights_dims.size());
+  }
+  if (weights_dims[0] != input_hidden_size) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                           "Input 1 dimension 0 should have same length as dimension 2 of input 0");
+  }
+
+  if (bias_dims[0] != weights_dims[1]) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                           "Input 'bias' dimension 0 should have same length as dimension 1 of input 'weights'");
+  }
+
+  int64_t q_hidden_size = bias_dims[0] / static_cast<int64_t>(3);
+  int64_t k_hidden_size = q_hidden_size;
+  int64_t v_hidden_size = k_hidden_size;
+  if (qkv_hidden_sizes_.size() != 0) {
+    if (qkv_hidden_sizes_.size() != 3) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "qkv_hidden_sizes attribute should have 3 elements");
+    }
+
+    for (size_t i = 0; i < qkv_hidden_sizes_.size(); i++) {
+      if (qkv_hidden_sizes_[i] % num_heads_ != 0) {
+        return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                               "hidden_size should be divisible by num_heads:", qkv_hidden_sizes_[i]);
+      }
+    }
+
+    q_hidden_size = qkv_hidden_sizes_[0];
+    k_hidden_size = qkv_hidden_sizes_[1];
+    v_hidden_size = qkv_hidden_sizes_[2];
+  }
+
+  int64_t kv_sequence_length = sequence_length;
+
+  if (q_hidden_size != k_hidden_size) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                           "qkv_hidden_sizes first element should be same as the second");
+  }
+
+  if (this->require_same_hidden_size_ && k_hidden_size != v_hidden_size) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Hidden size of Q, K and V shall be same");
+  }
+
+  if (bias_dims[0] != q_hidden_size + k_hidden_size + v_hidden_size) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                           "Input 'bias' dimension 0 should have same length as sum of Q/K/V hidden sizes:",
+                           " q_hidden_size=", q_hidden_size, " k_hidden_size=", k_hidden_size, " v_hidden_size=",
+                           v_hidden_size, "bias_dims[0]=", bias_dims[0]);
+  }
+
+  int64_t past_sequence_length = 0;
+  if (past != nullptr) {
+    if (k_hidden_size != v_hidden_size) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'past' expect k_hidden_size == v_hidden_size");
+    }
+
+    const auto& past_dims = past->Shape().GetDims();
+    if (past_dims.size() != 5) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'past' is expected to have 5 dimension, got ",
+                             past_dims.size());
+    }
+
+    if (past_dims[0] != 2) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'past' dimension 0 shall have length of 2");
+    }
+
+    if (past_dims[1] != batch_size) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "Inputs 'past' dimension 1 shall have same length as dimension 0 of input 0");
+    }
+
+    if (static_cast<int>(past_dims[2]) != num_heads_) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "Inputs 'past' dimension 2 shall have length of num_heads", num_heads_);
+    }
+
+    if (static_cast<int>(past_dims[4]) != k_hidden_size / num_heads_) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "Inputs 'past' dimension 2 shall have length of ", k_hidden_size / num_heads_);
+    }
+
+    if (!past_present_share_buffer_) {
+      past_sequence_length = past_dims[3];
+    } else {
+      if (past_seq_len == nullptr || !::onnxruntime::IsScalarOr1ElementVector(past_seq_len)) {
+        return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                               "past_sequence_length tensor must be of one element when past_present_share_buffer is set");
+      }
+      past_sequence_length = *past_seq_len->Data<int32_t>();
+    }
+  }
+
+  int64_t total_sequence_length = kv_sequence_length + past_sequence_length;
+  if (past != nullptr && past_present_share_buffer_) {
+    const auto& past_dims = past->Shape().GetDims();
+    if (past_dims[3] < total_sequence_length) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "when past_present_share_buffer, past tensor sequence must not smaller than total_sequence_length ");
+    }
+  }
+
+  int64_t max_sequence_length = -1;
+  AttentionMaskType mask_type = AttentionMaskType::MASK_NONE;
+  if (mask_index != nullptr) {
+    mask_type = AttentionMaskType::MASK_UNKNOWN;
+    auto status = this->CheckMask(mask_index, mask_type,
+                                  max_sequence_length, batch_size, sequence_length, total_sequence_length);
+    if (status != Status::OK()) {
+      return status;
+    }
+
+    if (mask_type == AttentionMaskType::MASK_2D_DUMMY) {
+      mask_index = nullptr;
+      mask_type = AttentionMaskType::MASK_NONE;
+    }
+  }
+
+  gsl::span<const int64_t> attention_bias_dims;
+  if (attention_bias != nullptr) {
+    attention_bias_dims = attention_bias->Shape().GetDims();
+
+    ORT_RETURN_IF_ERROR(::onnxruntime::contrib::multihead_attention_helper::CheckAttentionBias(
+        attention_bias_dims, batch_size, num_heads_, sequence_length, total_sequence_length));
+  }
+
+  if (past != nullptr && past_present_share_buffer_) {
+    if (max_sequence_length <= 0) {
+      max_sequence_length = past->Shape().GetDims()[3];
+    }
+    if (max_sequence_length != past->Shape().GetDims()[3]) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "max_sequence_length not matching from mask and past when past_present_share_buffer_ is set");
+    }
+  }
+
+  if (parameters != nullptr) {
+    AttentionParameters* output_parameters = reinterpret_cast<AttentionParameters*>(parameters);
+    output_parameters->batch_size = static_cast<int>(batch_size);
+    output_parameters->sequence_length = static_cast<int>(sequence_length);
+    output_parameters->past_sequence_length = static_cast<int>(past_sequence_length);
+    output_parameters->kv_sequence_length = static_cast<int>(kv_sequence_length);
+    output_parameters->total_sequence_length = static_cast<int>(total_sequence_length);
+    output_parameters->max_sequence_length = static_cast<int>(max_sequence_length);
+    output_parameters->input_hidden_size = static_cast<int>(input_hidden_size);
+    output_parameters->hidden_size = static_cast<int>(q_hidden_size);
+    output_parameters->v_hidden_size = static_cast<int>(v_hidden_size);
+    output_parameters->head_size = static_cast<int>(q_hidden_size) / num_heads_;
+    output_parameters->v_head_size = static_cast<int>(v_hidden_size) / num_heads_;
+    output_parameters->num_heads = num_heads_;
+    output_parameters->is_unidirectional = is_unidirectional_;
+    output_parameters->past_present_share_buffer = (past_present_share_buffer_ != 0 && past != nullptr);
+    output_parameters->do_rotary = do_rotary_;
+    output_parameters->rotary_dim = rotary_embedding_ == 0 ? (int)(output_parameters->head_size) : rotary_embedding_;
+    output_parameters->mask_filter_value = mask_filter_value_;
+    output_parameters->scale = scale_;
+    output_parameters->mask_type = mask_type;
+    output_parameters->broadcast_attn_bias_dim_0 = attention_bias_dims.size() > 0 && attention_bias_dims[0] == 1;
+    output_parameters->broadcast_attn_bias_dim_1 = attention_bias_dims.size() > 1 && attention_bias_dims[1] == 1;
+    output_parameters->qkv_format = Q_K_V_BNSH;
+  }
+
+  return Status::OK();
+}
+
+inline Status AttentionBase::CheckInputs(const TensorShape& input_shape,
+                                         const TensorShape& weights_shape,
+                                         const TensorShape& bias_shape,
+                                         const Tensor*& mask_index,
+                                         const Tensor* past,
+                                         const Tensor* attention_bias,
+                                         void* parameters,
+                                         const int max_threads_per_block,
+                                         const Tensor* past_seq_len) const {
+  if (num_heads_ > max_threads_per_block) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "num_heads should be no larger than ", max_threads_per_block);
+  }
+
+  return CheckInputs(input_shape, weights_shape, bias_shape, mask_index, past, attention_bias, parameters, past_seq_len);
+}
+
+template <typename TOpKernelContext>
+inline Tensor* AttentionBase::GetPresent(TOpKernelContext* context,
+                                         const Tensor* past,
+                                         int batch_size,
+                                         int head_size,
+                                         int kv_sequence_length,
+                                         int& past_sequence_length) const {
+  past_sequence_length = (nullptr != past) ? static_cast<int>(past->Shape().GetDims()[3]) : 0;
+  std::array<int64_t, 5> present_dims{2, batch_size, num_heads_,
+                                      static_cast<int64_t>(kv_sequence_length) + past_sequence_length, head_size};
+
+  TensorShape present_shape(present_dims);
+  Tensor* present = context->Output(1, present_shape);
+  if (nullptr != past && nullptr == present) {
+    ORT_THROW("Expect to have present state output when past state input is given");
+  }
+
+  return present;
+}
+#endif  // SHARED_PROVIDER
+
 }  // namespace contrib
 }  // namespace onnxruntime
diff --git a/onnxruntime/contrib_ops/cpu/bert/attention_parameters.h b/onnxruntime/contrib_ops/cpu/bert/attention_parameters.h
index 9a123e80adc18..f316a0dfdf91c 100644
--- a/onnxruntime/contrib_ops/cpu/bert/attention_parameters.h
+++ b/onnxruntime/contrib_ops/cpu/bert/attention_parameters.h
@@ -10,33 +10,33 @@ namespace contrib {
 
 // Parameters deduced from node attributes and inputs/outputs.
 struct AttentionParameters {
-  int batch_size;
-  int sequence_length;
-  int kv_sequence_length;     // input sequence length of K or V
-  int past_sequence_length;   // sequence length in past state of K or V
-  int total_sequence_length;  // total sequence length of K or V
-  int max_sequence_length;    // max sequence length from 4D mask
-  int input_hidden_size;      // first dimension of weights for input projection
-  int hidden_size;            // hidden size of Q or K
-  int head_size;              // hidden size per head of Q or K
-  int v_hidden_size;          // hidden size of V
-  int v_head_size;            // hidden size per head of V
-  int num_heads;
-  int num_splits;      // number of splits for splitkv
+  int batch_size = 0;
+  int sequence_length = 0;
+  int kv_sequence_length = 0;     // input sequence length of K or V
+  int past_sequence_length = 0;   // sequence length in past state of K or V
+  int total_sequence_length = 0;  // total sequence length of K or V
+  int max_sequence_length = 0;    // max sequence length from 4D mask
+  int input_hidden_size = 0;      // first dimension of weights for input projection
+  int hidden_size = 0;            // hidden size of Q or K
+  int head_size = 0;              // hidden size per head of Q or K
+  int v_hidden_size = 0;          // hidden size of V
+  int v_head_size = 0;            // hidden size per head of V
+  int num_heads = 0;
+  int num_splits = 0;  // number of splits for splitkv
   int rotary_dim = 0;  // rotary embedding dimension
-  int beam_width;
+  int beam_width = 0;
   bool is_unidirectional = false;
   bool past_present_share_buffer = false;
   bool is_packed_qkv = false;  // whether qkv is packed
   bool do_rotary = false;
   bool broadcast_attn_bias_dim_0 = false;
   bool broadcast_attn_bias_dim_1 = false;
-  float mask_filter_value;
-  float scale;
+  float mask_filter_value = 0.0f;
+  float scale = 0.0f;
   bool use_tf32 = false;
   bool is_output_bnsh = false;  // whether the output format is BNSH
-  AttentionMaskType mask_type;
-  AttentionQkvFormat qkv_format;
+  AttentionMaskType mask_type = AttentionMaskType::MASK_NONE;
+  AttentionQkvFormat qkv_format = AttentionQkvFormat::Q_K_V_BNSH;
 };
 
 // Parameters deduced from node attributes and inputs/outputs.
diff --git a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.cc b/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.cc
deleted file mode 100644
index 97f75d297d789..0000000000000
--- a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.cc
+++ /dev/null
@@ -1,141 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License.
-
-#include "longformer_attention_base.h"
-
-namespace onnxruntime {
-namespace contrib {
-
-Status LongformerAttentionBase::CheckInputs(const TensorShape& input_shape,
-                                            const TensorShape& weights_shape,
-                                            const TensorShape& bias_shape,
-                                            const TensorShape& attention_mask_shape,
-                                            const TensorShape& global_weights_shape,
-                                            const TensorShape& global_bias_shape,
-                                            const TensorShape& global_mask_shape) const {
-  // Input shapes:
-  //   input                 : (batch_size, sequence_length, hidden_size)
-  //   weights               : (hidden_size, 3 * hidden_size) -- format 1
-  //                           (3, hidden_size, hidden_size)  -- format 0
-  //   bias                  : (3 * hidden_size)              -- format 1 (bias for Q, K, V)
-  //                           (5 * hidden_size)              -- format 0 (bias for Q, K, V, Global_K, Global_V)
-  //   attention_mask        : (batch_size, sequence_length)
-  //   global_weights        : (hidden_size, 3 * hidden_size) -- format 1
-  //                           (3, hidden_size, hidden_size)  -- format 0
-  //   global_bias           : (3 * hidden_size)              -- format 1 (bias for Global_Q, Global_K, Global_V)
-  //                           (1 * hidden_size)              -- format 0 (bias for Global_Q)
-  //   global_attention_mask : (batch_size, sequence_length)
-
-  const auto& dims = input_shape.GetDims();
-  if (dims.size() != 3) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'input' is expected to have 3 dimensions, got ",
-                           dims.size());
-  }
-
-  int batch_size = static_cast<int>(dims[0]);
-  int sequence_length = static_cast<int>(dims[1]);
-  auto hidden_size = dims[2];
-  if (sequence_length % (2 * window_) != 0) {
-    return ORT_MAKE_STATUS(
-        ONNXRUNTIME, INVALID_ARGUMENT,
-        "Input 'input' dimension 1 should be divisible by 2W, where W is value of the window attribute.");
-  }
-  if (hidden_size % num_heads_ != 0) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                           "Input 'input' dimension 2 should be divisible by value of the num_heads attribute.");
-  }
-
-  const auto& weights_dims = weights_shape.GetDims();
-  bool use_merged_qkv_weights = (weights_shape.NumDimensions() == 2);
-  if (use_merged_qkv_weights) {
-    if (weights_dims[0] != hidden_size || weights_dims[1] != 3 * hidden_size) {
-      return ORT_MAKE_STATUS(
-          ONNXRUNTIME, INVALID_ARGUMENT,
-          "Input 'weights' shape should be (hidden_size, 3 * hidden_size) for format 1");
-    }
-  } else {
-    if (weights_dims.size() != 3 ||
-        weights_dims[0] != 3 || weights_dims[1] != hidden_size || weights_dims[2] != hidden_size) {
-      return ORT_MAKE_STATUS(
-          ONNXRUNTIME, INVALID_ARGUMENT,
-          "Input 'weights' shape should be (3, hidden_size, hidden_size) for format 0");
-    }
-  }
-
-  const auto& bias_dims = bias_shape.GetDims();
-  if (bias_dims.size() != 1) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'bias' is expected to have 1 dimension, got ",
-                           bias_dims.size());
-  }
-
-  if (use_merged_qkv_weights) {
-    if (bias_dims[0] != 3 * hidden_size) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "Input 'bias' shape should be (3 * hidden_size) for format 1");
-    }
-  } else {
-    if (bias_dims[0] != 5 * hidden_size) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "Input 'bias' shape should be (5 * hidden_size) for format 0");
-    }
-  }
-
-  const auto& mask_dims = attention_mask_shape.GetDims();
-  if (mask_dims.size() == 2) {
-    if (static_cast<int>(mask_dims[0]) != batch_size || static_cast<int>(mask_dims[1]) != sequence_length) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "Inputs 'attention_mask' shape shall be (batch_size, sequence_length)");
-    }
-  } else {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                           "Input 'attention_mask' is expected to have 2 dimensions, got ", mask_dims.size());
-  }
-
-  const auto& global_weights_dims = global_weights_shape.GetDims();
-  if (use_merged_qkv_weights) {
-    if (global_weights_dims.size() != 2 ||
-        global_weights_dims[0] != hidden_size || global_weights_dims[1] != 3 * hidden_size) {
-      return ORT_MAKE_STATUS(
-          ONNXRUNTIME, INVALID_ARGUMENT,
-          "Input 'global_weights' shape should be (hidden_size, 3 * hidden_size) for format 1");
-    }
-  } else {
-    if (global_weights_dims.size() != 3 || global_weights_dims[0] != 3 ||
-        global_weights_dims[1] != hidden_size || global_weights_dims[2] != hidden_size) {
-      return ORT_MAKE_STATUS(
-          ONNXRUNTIME, INVALID_ARGUMENT,
-          "Input 'global_weights' shape should be (3, hidden_size, hidden_size) for format 0");
-    }
-  }
-
-  const auto& global_bias_dims = global_bias_shape.GetDims();
-  if (global_bias_dims.size() != 1) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'global_bias' is expected to have 1 dimension, got ",
-                           global_bias_dims.size());
-  }
-
-  if (use_merged_qkv_weights) {
-    if (global_bias_dims[0] != 3 * hidden_size) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "Input 'global_bias' shape should be (3 * hidden_size) for format 1");
-    }
-  } else {
-    if (global_bias_dims[0] != hidden_size) {
-      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                             "Input 'global_bias' shape should be (hidden_size) for format 0");
-    }
-  }
-
-  const auto& global_mask_dims = global_mask_shape.GetDims();
-  if (global_mask_dims.size() != 2 ||
-      static_cast<int>(global_mask_dims[0]) != batch_size ||
-      static_cast<int>(global_mask_dims[1]) != sequence_length) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                           "Input 'global_attention_mask' shape shall be (batch_size, sequence_length)");
-  }
-
-  return Status::OK();
-}
-
-}  // namespace contrib
-}  // namespace onnxruntime
diff --git a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h b/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h
index ac1cccaa83cf9..bb1dfea38ae80 100644
--- a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h
+++ b/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h
@@ -4,7 +4,9 @@
 #pragma once
 
 #include "core/common/common.h"
+#ifndef SHARED_PROVIDER
 #include "core/framework/op_kernel.h"
+#endif
 
 namespace onnxruntime {
 namespace contrib {
@@ -20,7 +22,8 @@ class LongformerAttentionBase {
                      const TensorShape& global_attention_mask_shape) const;
 
  protected:
-  LongformerAttentionBase(const OpKernelInfo& info) {
+  template <typename KernelInfoType>
+  LongformerAttentionBase(const KernelInfoType& info) {
     int64_t num_heads = 0;
     ORT_ENFORCE(info.GetAttr("num_heads", &num_heads).IsOK() && num_heads > 0);
     num_heads_ = static_cast<int>(num_heads);
@@ -43,5 +46,126 @@ constexpr const char* kUseHalf4 = "ORT_LONGFORMER_USE_HALF4";
 
 }  // namespace longformer
 
+#ifndef SHARED_PROVIDER
+// Inline implementation of CheckInputs for non-SHARED_PROVIDER builds.
+inline Status LongformerAttentionBase::CheckInputs(const TensorShape& input_shape,
+                                                   const TensorShape& weights_shape,
+                                                   const TensorShape& bias_shape,
+                                                   const TensorShape& attention_mask_shape,
+                                                   const TensorShape& global_weights_shape,
+                                                   const TensorShape& global_bias_shape,
+                                                   const TensorShape& global_mask_shape) const {
+  const auto& dims = input_shape.GetDims();
+  if (dims.size() != 3) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'input' is expected to have 3 dimensions, got ",
+                           dims.size());
+  }
+
+  int batch_size = static_cast<int>(dims[0]);
+  int sequence_length = static_cast<int>(dims[1]);
+  auto hidden_size = dims[2];
+  if (sequence_length % (2 * window_) != 0) {
+    return ORT_MAKE_STATUS(
+        ONNXRUNTIME, INVALID_ARGUMENT,
+        "Input 'input' dimension 1 should be divisible by 2W, where W is value of the window attribute.");
+  }
+  if (hidden_size % num_heads_ != 0) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                           "Input 'input' dimension 2 should be divisible by value of the num_heads attribute.");
+  }
+
+  const auto& weights_dims = weights_shape.GetDims();
+  bool use_merged_qkv_weights = (weights_shape.NumDimensions() == 2);
+  if (use_merged_qkv_weights) {
+    if (weights_dims[0] != hidden_size || weights_dims[1] != 3 * hidden_size) {
+      return ORT_MAKE_STATUS(
+          ONNXRUNTIME, INVALID_ARGUMENT,
+          "Input 'weights' shape should be (hidden_size, 3 * hidden_size) for format 1");
+    }
+  } else {
+    if (weights_dims.size() != 3 ||
+        weights_dims[0] != 3 || weights_dims[1] != hidden_size || weights_dims[2] != hidden_size) {
+      return ORT_MAKE_STATUS(
+          ONNXRUNTIME, INVALID_ARGUMENT,
+          "Input 'weights' shape should be (3, hidden_size, hidden_size) for format 0");
+    }
+  }
+
+  const auto& bias_dims = bias_shape.GetDims();
+  if (bias_dims.size() != 1) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'bias' is expected to have 1 dimension, got ",
+                           bias_dims.size());
+  }
+
+  if (use_merged_qkv_weights) {
+    if (bias_dims[0] != 3 * hidden_size) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "Input 'bias' shape should be (3 * hidden_size) for format 1");
+    }
+  } else {
+    if (bias_dims[0] != 5 * hidden_size) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "Input 'bias' shape should be (5 * hidden_size) for format 0");
+    }
+  }
+
+  const auto& mask_dims = attention_mask_shape.GetDims();
+  if (mask_dims.size() == 2) {
+    if (static_cast<int>(mask_dims[0]) != batch_size || static_cast<int>(mask_dims[1]) != sequence_length) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "Inputs 'attention_mask' shape shall be (batch_size, sequence_length)");
+    }
+  } else {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                           "Input 'attention_mask' is expected to have 2 dimensions, got ", mask_dims.size());
+  }
+
+  const auto& global_weights_dims = global_weights_shape.GetDims();
+  if (use_merged_qkv_weights) {
+    if (global_weights_dims.size() != 2 ||
+        global_weights_dims[0] != hidden_size || global_weights_dims[1] != 3 * hidden_size) {
+      return ORT_MAKE_STATUS(
+          ONNXRUNTIME, INVALID_ARGUMENT,
+          "Input 'global_weights' shape should be (hidden_size, 3 * hidden_size) for format 1");
+    }
+  } else {
+    if (global_weights_dims.size() != 3 || global_weights_dims[0] != 3 ||
+        global_weights_dims[1] != hidden_size || global_weights_dims[2] != hidden_size) {
+      return ORT_MAKE_STATUS(
+          ONNXRUNTIME, INVALID_ARGUMENT,
+          "Input 'global_weights' shape should be (3, hidden_size, hidden_size) for format 0");
+    }
+  }
+
+  const auto& global_bias_dims = global_bias_shape.GetDims();
+  if (global_bias_dims.size() != 1) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'global_bias' is expected to have 1 dimension, got ",
+                           global_bias_dims.size());
+  }
+
+  if (use_merged_qkv_weights) {
+    if (global_bias_dims[0] != 3 * hidden_size) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "Input 'global_bias' shape should be (3 * hidden_size) for format 1");
+    }
+  } else {
+    if (global_bias_dims[0] != hidden_size) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                             "Input 'global_bias' shape should be (hidden_size) for format 0");
+    }
+  }
+
+  const auto& global_mask_dims = global_mask_shape.GetDims();
+  if (global_mask_dims.size() != 2 ||
+      static_cast<int>(global_mask_dims[0]) != batch_size ||
+      static_cast<int>(global_mask_dims[1]) != sequence_length) {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
+                           "Input 'global_attention_mask' shape shall be (batch_size, sequence_length)");
+  }
+
+  return Status::OK();
+}
+#endif  // SHARED_PROVIDER
+
 }  // namespace contrib
 }  // namespace onnxruntime
diff --git a/onnxruntime/contrib_ops/cpu/crop.h b/onnxruntime/contrib_ops/cpu/crop.h
index 3b72ef429c1f7..97577304e948e 100644
--- a/onnxruntime/contrib_ops/cpu/crop.h
+++ b/onnxruntime/contrib_ops/cpu/crop.h
@@ -4,7 +4,9 @@
 #pragma once
 
 #include "core/common/common.h"
+#ifndef SHARED_PROVIDER
 #include "core/framework/op_kernel.h"
+#endif
 
 #include <gsl/gsl>
 
@@ -13,9 +15,10 @@ namespace contrib {
 
 class CropBase {
  protected:
-  CropBase(const OpKernelInfo& info)
-      : border_(info.GetAttrsOrDefault<int64_t>("border")),
-        scale_(info.GetAttrsOrDefault<int64_t>("scale")) {
+  template <typename KernelInfoType>
+  CropBase(const KernelInfoType& info)
+      : border_(info.template GetAttrsOrDefault<int64_t>("border")),
+        scale_(info.template GetAttrsOrDefault<int64_t>("scale")) {
   }
 
   Status ValidateInput(const Tensor* X) const {
diff --git a/onnxruntime/core/providers/cpu/math/cumsum.cc b/onnxruntime/core/providers/cpu/math/cumsum.cc
index 8321b81021d19..14ea6712f7f46 100644
--- a/onnxruntime/core/providers/cpu/math/cumsum.cc
+++ b/onnxruntime/core/providers/cpu/math/cumsum.cc
@@ -13,29 +13,6 @@ using namespace onnxruntime;
 
 namespace onnxruntime {
 
-namespace cumsum_op {
-Status GetAxis(const Tensor* axis_tensor, int64_t input_rank, int64_t& axis_out) {
-  if (!axis_tensor)
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Axis tensor must be provided to the CumSum op");
-
-  if (axis_tensor->Shape().NumDimensions() > 1)
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Axis tensor should be 0D or 1D");
-
-  if (axis_tensor->IsDataType<int32_t>()) {
-    axis_out = static_cast<int64_t>(axis_tensor->Data<int32_t>()[0]);
-  } else if (axis_tensor->IsDataType<int64_t>()) {
-    axis_out = axis_tensor->Data<int64_t>()[0];
-  } else {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Axis tensor should be of type `int32_t` or `int64_t`");
-  }
-
-  axis_out = HandleNegativeAxis(axis_out, input_rank);
-
-  return Status::OK();
-}
-
-}  // namespace cumsum_op
-
 ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(
     CumSum,
     11,
diff --git a/onnxruntime/core/providers/cpu/math/cumsum.h b/onnxruntime/core/providers/cpu/math/cumsum.h
index fa1c1ceb0df10..b7443ada40861 100644
--- a/onnxruntime/core/providers/cpu/math/cumsum.h
+++ b/onnxruntime/core/providers/cpu/math/cumsum.h
@@ -1,11 +1,17 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
+#pragma once
 
 #include "core/common/common.h"
+#include "core/providers/common.h"
+
+#ifndef SHARED_PROVIDER
 #include "core/framework/op_kernel.h"
+#endif
 
 namespace onnxruntime {
 
+#ifndef SHARED_PROVIDER
 template <class T>
 class CumSum final : public OpKernel {
  public:
@@ -17,10 +23,33 @@ class CumSum final : public OpKernel {
   int64_t exclusive_;
   int64_t reverse_;
 };
+#endif
 
 namespace cumsum_op {
 
+#ifdef SHARED_PROVIDER
 Status GetAxis(const Tensor* axis_tensor, int64_t input_rank, int64_t& axis_out);
+#else
+inline Status GetAxis(const Tensor* axis_tensor, int64_t input_rank, int64_t& axis_out) {
+  if (!axis_tensor)
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Axis tensor must be provided to the CumSum op");
+
+  if (axis_tensor->Shape().NumDimensions() > 1)
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Axis tensor should be 0D or 1D");
+
+  if (axis_tensor->IsDataType<int32_t>()) {
+    axis_out = static_cast<int64_t>(axis_tensor->Data<int32_t>()[0]);
+  } else if (axis_tensor->IsDataType<int64_t>()) {
+    axis_out = axis_tensor->Data<int64_t>()[0];
+  } else {
+    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Axis tensor should be of type `int32_t` or `int64_t`");
+  }
+
+  axis_out = HandleNegativeAxis(axis_out, input_rank);
+
+  return Status::OK();
+}
+#endif  // SHARED_PROVIDER
 
 }  // namespace cumsum_op
 }  // namespace onnxruntime
diff --git a/onnxruntime/core/providers/cpu/object_detection/roialign.cc b/onnxruntime/core/providers/cpu/object_detection/roialign.cc
index 87958a9f7e2dd..0680be3aea49c 100644
--- a/onnxruntime/core/providers/cpu/object_detection/roialign.cc
+++ b/onnxruntime/core/providers/cpu/object_detection/roialign.cc
@@ -258,76 +258,6 @@ void RoiAlignForward(const TensorShape& output_shape, const T* bottom_data, floa
 }
 }  // namespace
 
-Status CheckROIAlignValidInput(const Tensor* X_ptr, const Tensor* rois_ptr, const Tensor* batch_indices_ptr) {
-  constexpr int64_t EXPECTED_NUM_ROI_DIMS = 2;
-  constexpr int64_t EXPECTED_SECOND_ROI_DIM = 4;
-  if (!X_ptr) {
-    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Null input X ptr");
-  }
-  if (!rois_ptr) {
-    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Null rois_ptr");
-  }
-  if (!batch_indices_ptr) {
-    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Null batch_indices_ptr");
-  }
-
-  const auto& rois_dims = rois_ptr->Shape();
-  const auto& batch_indices_dims = batch_indices_ptr->Shape();
-
-  if (batch_indices_dims.NumDimensions() != 1) {
-    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
-                  "Number of dimensions for batch indices should be exactly 1");
-  }
-
-  // validate rois_dims
-  if (rois_dims.NumDimensions() != EXPECTED_NUM_ROI_DIMS) {
-    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
-                  "Number of dimensions for rois should be exactly " + std::to_string(EXPECTED_NUM_ROI_DIMS));
-  }
-  if (rois_dims[1] != EXPECTED_SECOND_ROI_DIM) {
-    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
-                  "Second dimension for rois should be exactly " + std::to_string(EXPECTED_SECOND_ROI_DIM));
-  }
-
-  // first dimension of batch_indices and rois should match
-  if (batch_indices_dims[0] != rois_dims[0]) {
-    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
-                  "First dimension (num_rois) of batch_indices and rois don't match");
-  }
-
-  if (batch_indices_ptr->Location().device.Type() == OrtDevice::CPU) {
-    // Validate batch_indices values are within [0, batch_size) when the tensor
-    // data is accessible from the host (CPU).
-    const int64_t batch_size = X_ptr->Shape()[0];
-    const int64_t num_rois = batch_indices_dims[0];
-
-    auto check_bounds = [batch_size, num_rois](const auto* batch_indices_data) -> Status {
-      for (int64_t i = 0; i < num_rois; ++i) {
-        if (batch_indices_data[i] < 0 || batch_indices_data[i] >= batch_size) {
-          return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
-                        "batch_indices value " + std::to_string(batch_indices_data[i]) +
-                            " at index " + std::to_string(i) +
-                            " is out of range [0, " + std::to_string(batch_size) + ")");
-        }
-      }
-      return Status::OK();
-    };
-
-    if (batch_indices_ptr->IsDataType<int64_t>()) {
-      auto status = check_bounds(batch_indices_ptr->Data<int64_t>());
-      if (!status.IsOK()) return status;
-    } else if (batch_indices_ptr->IsDataType<int32_t>()) {
-      auto status = check_bounds(batch_indices_ptr->Data<int32_t>());
-      if (!status.IsOK()) return status;
-    } else {
-      return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
-                    "batch_indices must be of type int64_t or int32_t");
-    }
-  }
-
-  return Status::OK();
-}
-
 template <typename T>
 Status RoiAlign<T>::Compute(OpKernelContext* context) const {
   const auto* X_ptr = context->Input<Tensor>(0);
diff --git a/onnxruntime/core/providers/cpu/object_detection/roialign.h b/onnxruntime/core/providers/cpu/object_detection/roialign.h
index 1bb8bd34c5cb2..bb97de158369b 100644
--- a/onnxruntime/core/providers/cpu/object_detection/roialign.h
+++ b/onnxruntime/core/providers/cpu/object_detection/roialign.h
@@ -3,12 +3,86 @@
 
 #pragma once
 
-#include "core/framework/op_kernel.h"
+#include <algorithm>
 #include <cctype>
+#include <string>
+
+#include "core/common/common.h"
+#ifndef SHARED_PROVIDER
+#include "core/framework/op_kernel.h"
+#endif
 
 namespace onnxruntime {
 
+#ifdef SHARED_PROVIDER
 Status CheckROIAlignValidInput(const Tensor* X_ptr, const Tensor* rois_ptr, const Tensor* batch_indices_ptr);
+#else
+inline Status CheckROIAlignValidInput(const Tensor* X_ptr, const Tensor* rois_ptr, const Tensor* batch_indices_ptr) {
+  constexpr int64_t EXPECTED_NUM_ROI_DIMS = 2;
+  constexpr int64_t EXPECTED_SECOND_ROI_DIM = 4;
+  if (!X_ptr) {
+    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Null input X ptr");
+  }
+  if (!rois_ptr) {
+    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Null rois_ptr");
+  }
+  if (!batch_indices_ptr) {
+    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Null batch_indices_ptr");
+  }
+
+  const auto& rois_dims = rois_ptr->Shape();
+  const auto& batch_indices_dims = batch_indices_ptr->Shape();
+
+  if (batch_indices_dims.NumDimensions() != 1) {
+    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
+                  "Number of dimensions for batch indices should be exactly 1");
+  }
+
+  if (rois_dims.NumDimensions() != EXPECTED_NUM_ROI_DIMS) {
+    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
+                  "Number of dimensions for rois should be exactly " + std::to_string(EXPECTED_NUM_ROI_DIMS));
+  }
+  if (rois_dims[1] != EXPECTED_SECOND_ROI_DIM) {
+    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
+                  "Second dimension for rois should be exactly " + std::to_string(EXPECTED_SECOND_ROI_DIM));
+  }
+
+  if (batch_indices_dims[0] != rois_dims[0]) {
+    return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
+                  "First dimension (num_rois) of batch_indices and rois don't match");
+  }
+
+  if (batch_indices_ptr->Location().device.Type() == OrtDevice::CPU) {
+    const int64_t batch_size = X_ptr->Shape()[0];
+    const int64_t num_rois = batch_indices_dims[0];
+
+    auto check_bounds = [batch_size, num_rois](const auto* batch_indices_data) -> Status {
+      for (int64_t i = 0; i < num_rois; ++i) {
+        if (batch_indices_data[i] < 0 || batch_indices_data[i] >= batch_size) {
+          return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
+                        "batch_indices value " + std::to_string(batch_indices_data[i]) +
+                            " at index " + std::to_string(i) +
+                            " is out of range [0, " + std::to_string(batch_size) + ")");
+        }
+      }
+      return Status::OK();
+    };
+
+    if (batch_indices_ptr->IsDataType<int64_t>()) {
+      auto status = check_bounds(batch_indices_ptr->Data<int64_t>());
+      if (!status.IsOK()) return status;
+    } else if (batch_indices_ptr->IsDataType<int32_t>()) {
+      auto status = check_bounds(batch_indices_ptr->Data<int32_t>());
+      if (!status.IsOK()) return status;
+    } else {
+      return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
+                    "batch_indices must be of type int64_t or int32_t");
+    }
+  }
+
+  return Status::OK();
+}
+#endif
 
 enum struct RoiAlignMode {
   avg = 0,
@@ -17,10 +91,10 @@ enum struct RoiAlignMode {
 
 class RoiAlignBase {
  public:
-  explicit RoiAlignBase(const OpKernelInfo& info) {
-    // mode
+  template <typename TKernelInfo>
+  explicit RoiAlignBase(const TKernelInfo& info) {
     std::string mode;
-    if (info.GetAttr<std::string>("mode", &mode).IsOK()) {
+    if (info.template GetAttr<std::string>("mode", &mode).IsOK()) {
       std::transform(mode.begin(), mode.end(), mode.begin(), [](char i) { return static_cast<char>(::tolower(i)); });
       if (mode == "avg") {
         mode_ = RoiAlignMode::avg;
@@ -31,41 +105,33 @@ class RoiAlignBase {
       }
     }
 
-    // output_height
     int64_t output_height_tmp;
-    if (info.GetAttr<int64_t>("output_height", &output_height_tmp).IsOK()) {
+    if (info.template GetAttr<int64_t>("output_height", &output_height_tmp).IsOK()) {
       output_height_ = output_height_tmp;
     }
 
-    // output_width
     int64_t output_width_tmp;
-    if (info.GetAttr<int64_t>("output_width", &output_width_tmp).IsOK()) {
+    if (info.template GetAttr<int64_t>("output_width", &output_width_tmp).IsOK()) {
       output_width_ = output_width_tmp;
     }
 
-    // sampling_ratio
     int64_t sampling_ratio_tmp;
-    if (info.GetAttr<int64_t>("sampling_ratio", &sampling_ratio_tmp).IsOK()) {
+    if (info.template GetAttr<int64_t>("sampling_ratio", &sampling_ratio_tmp).IsOK()) {
       sampling_ratio_ = sampling_ratio_tmp;
       ORT_ENFORCE(sampling_ratio_ >= 0, "Sampling ratio should be >=0, but it was ", sampling_ratio_);
     }
 
-    // spatial_scale
     float spatial_scale_tmp;
-    if (info.GetAttr<float>("spatial_scale", &spatial_scale_tmp).IsOK()) {
+    if (info.template GetAttr<float>("spatial_scale", &spatial_scale_tmp).IsOK()) {
       spatial_scale_ = spatial_scale_tmp;
     }
 
     std::string coordinate_transformation_mode;
-    if (info.GetAttr<std::string>("coordinate_transformation_mode", &coordinate_transformation_mode).IsOK()) {
-      if (coordinate_transformation_mode == "half_pixel")
-        half_pixel_ = true;
-      else
-        half_pixel_ = false;
+    if (info.template GetAttr<std::string>("coordinate_transformation_mode", &coordinate_transformation_mode).IsOK()) {
+      half_pixel_ = coordinate_transformation_mode == "half_pixel";
     }
 
     if (mode_ == RoiAlignMode::max && sampling_ratio_ != 1) {
-      // TODO(fdwr): Issue #6146. ORT 1.13 will correct the incorrect summation of max mode with PR #7354.
       LOGS_DEFAULT(WARNING) << "The existing summation for max mode and sampling ratios besides 1 is incorrect "
                             << "and will be fixed in the next ORT 1.13 release. Thus the results of RoiAlign "
                             << "will be different.";
diff --git a/onnxruntime/core/providers/cpu/tensor/concat.cc b/onnxruntime/core/providers/cpu/tensor/concat.cc
index e3d5c0600420f..98d61ed1d3127 100644
--- a/onnxruntime/core/providers/cpu/tensor/concat.cc
+++ b/onnxruntime/core/providers/cpu/tensor/concat.cc
@@ -49,14 +49,6 @@ using EnabledDataTypes = ORT_OP_KERNEL_ARG_ENABLED_TYPE_LIST_ALL_OPSETS(kCpuExec
                                                                         Concat, Input, 0);
 }  // namespace
 
-// this method will be shared between 'Concat' (CPU and GPU) and
-// 'ConcatFromSequence' ('concat' and 'stack' modes) to validate inputs
-Status ConcatBase::PrepareForCompute(OpKernelContext* ctx,
-                                     const InlinedTensorsVector& input_tensors,
-                                     Prepare& p) const {
-  return PrepareForComputeImpl(ctx, input_tensors, p);
-}
-
 namespace {
 TensorShapeVector StridesForStack(const TensorShapeVector& full_strides, uint64_t axis) {
   // if we are stacking, skip the dimension that will be stacked along in the output strides
diff --git a/onnxruntime/core/providers/cpu/tensor/concatbase.h b/onnxruntime/core/providers/cpu/tensor/concatbase.h
index b9085b2a9318b..df2eb78c61180 100644
--- a/onnxruntime/core/providers/cpu/tensor/concatbase.h
+++ b/onnxruntime/core/providers/cpu/tensor/concatbase.h
@@ -209,8 +209,16 @@ class ConcatBase {
     return Status::OK();
   }
 
+#ifdef SHARED_PROVIDER
   Status PrepareForCompute(OpKernelContext* ctx, const InlinedTensorsVector& input_tensors,
                            Prepare& p) const;
+#else
+  template <typename KernelContextType>
+  inline Status PrepareForCompute(KernelContextType* ctx, const InlinedTensorsVector& input_tensors,
+                                  Prepare& p) const {
+    return PrepareForComputeImpl(ctx, input_tensors, p);
+  }
+#endif
 
  protected:
   template <typename KernelInfoType>
diff --git a/onnxruntime/core/providers/cpu/tensor/gather.cc b/onnxruntime/core/providers/cpu/tensor/gather.cc
index f171b33ee5f4f..3b3c67e7d818b 100644
--- a/onnxruntime/core/providers/cpu/tensor/gather.cc
+++ b/onnxruntime/core/providers/cpu/tensor/gather.cc
@@ -56,10 +56,6 @@ ONNX_CPU_OPERATOR_KERNEL(
         .TypeConstraint("Tind", BuildKernelDefConstraintsFromTypeList<EnabledIndexTypes>()),
     Gather);
 
-Status GatherBase::PrepareForCompute(OpKernelContext* context, Prepare& p) const {
-  return PrepareForComputeImpl(context, p);
-}
-
 template <typename Tin>
 Status GatherCopyData(const Tensor* indices_tensor, const uint8_t* src_base, uint8_t* dst_base, bool is_string_type,
                       const size_t element_bytes, const int64_t block_size, const int64_t M,
diff --git a/onnxruntime/core/providers/cpu/tensor/gatherbase.h b/onnxruntime/core/providers/cpu/tensor/gatherbase.h
index 1f5e85c554a78..fc29c04290883 100644
--- a/onnxruntime/core/providers/cpu/tensor/gatherbase.h
+++ b/onnxruntime/core/providers/cpu/tensor/gatherbase.h
@@ -46,7 +46,14 @@ class GatherBase {
     return Status::OK();
   }
 
+#ifdef SHARED_PROVIDER
   Status PrepareForCompute(OpKernelContext* context, Prepare& p) const;
+#else
+  template <typename KernelContextType>
+  inline Status PrepareForCompute(KernelContextType* context, Prepare& p) const {
+    return PrepareForComputeImpl(context, p);
+  }
+#endif
 
  protected:
   template <typename KernelInfoType>
diff --git a/onnxruntime/core/providers/cpu/tensor/space_depth_ops.h b/onnxruntime/core/providers/cpu/tensor/space_depth_ops.h
index 3218c8952d6ec..14a22fa7be0af 100644
--- a/onnxruntime/core/providers/cpu/tensor/space_depth_ops.h
+++ b/onnxruntime/core/providers/cpu/tensor/space_depth_ops.h
@@ -9,8 +9,9 @@ namespace onnxruntime {
 
 class SpaceDepthBase {
  protected:
-  explicit SpaceDepthBase(const OpKernelInfo& info) {
-    ORT_ENFORCE(info.GetAttr("blocksize", &blocksize_).IsOK(),
+  template <typename KernelInfoType>
+  explicit SpaceDepthBase(const KernelInfoType& info) {
+    ORT_ENFORCE(info.template GetAttr<int64_t>("blocksize", &blocksize_).IsOK(),
                 "Attribute blocksize is not set.");
   }
 
diff --git a/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc b/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc
index 1b6ee02061d34..5fdb57b1a5e35 100644
--- a/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc
+++ b/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc
@@ -77,57 +77,6 @@ ONNX_CPU_OPERATOR_KERNEL(
         .TypeConstraint("T", DataTypeImpl::AllTensorTypes()),
     Unsqueeze);
 
-Status UnsqueezeBase::PrepareCompute(OpKernelContext* ctx, Prepare& p) const {
-  const auto* X = ctx->Input<Tensor>(0);
-  ORT_ENFORCE(X != nullptr);
-  auto& input_tensor = *X;
-
-  TensorShapeVector axes;
-  size_t num_inputs = ctx->InputCount();
-  if (num_inputs == 2) {  // axes is an input
-    const Tensor* axes_tensor = ctx->Input<Tensor>(1);
-    ORT_ENFORCE(axes_tensor != nullptr, "Axes input is null");
-    ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 0 ||
-                    axes_tensor->Shape().NumDimensions() == 1,
-                "An axes tensor must be a scalar or a 1-D tensor.");
-    auto data_span = axes_tensor->template DataAsSpan<int64_t>();
-    axes.assign(data_span.begin(), data_span.end());
-  } else {
-    axes.assign(axes_.begin(), axes_.end());
-  }
-
-  // New dimension count is the current dimensions + the number of entries in axes
-  // Initialize output_dims to 0 in each axis initially
-  TensorShapeVector output_dims(axes.size() + input_tensor.Shape().NumDimensions(), 0);
-
-  // Set all axes indices to 1 in output_dims and check for duplicates
-  for (int64_t axis : axes) {
-    // Valid axis range is [0, output_rank - 1]
-    axis = HandleNegativeAxis(axis, onnxruntime::narrow<int64_t>(output_dims.size()));
-    if (axis < 0 || axis >= static_cast<int64_t>(output_dims.size()))
-      return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'axes' has an out of range axis");
-    if (output_dims[onnxruntime::narrow<size_t>(axis)] != 0)
-      return Status(ONNXRUNTIME, INVALID_ARGUMENT, "'axes' has a duplicate axis");
-    output_dims[onnxruntime::narrow<size_t>(axis)] = 1;
-  }
-
-  // Now fill in the zero entries with the existing shape
-  {
-    auto begin = input_tensor.Shape().GetDims().begin();
-    for (auto& axisSize : output_dims) {
-      if (axisSize == 0)
-        axisSize = *begin++;
-    }
-    assert(begin == input_tensor.Shape().GetDims().end());
-  }
-
-  TensorShape output_shape(output_dims);
-  p.output_tensor = ctx->Output(0, output_shape);
-  ORT_ENFORCE(nullptr != p.output_tensor);
-  p.input_tensor = &input_tensor;
-  return Status::OK();
-}
-
 Status Unsqueeze::Compute(OpKernelContext* ctx) const {
   Prepare p;
   ORT_RETURN_IF_ERROR(PrepareCompute(ctx, p));
diff --git a/onnxruntime/core/providers/cpu/tensor/unsqueeze.h b/onnxruntime/core/providers/cpu/tensor/unsqueeze.h
index 5a8a318923da5..09a77c113e022 100644
--- a/onnxruntime/core/providers/cpu/tensor/unsqueeze.h
+++ b/onnxruntime/core/providers/cpu/tensor/unsqueeze.h
@@ -19,7 +19,57 @@ class UnsqueezeBase {
     Tensor* output_tensor = nullptr;
   };
 
+#ifdef SHARED_PROVIDER
   Status PrepareCompute(OpKernelContext* context, Prepare& p) const;
+#else
+  template <typename KernelContextType>
+  inline Status PrepareCompute(KernelContextType* ctx, Prepare& p) const {
+    const auto* X = ctx->template Input<Tensor>(0);
+    ORT_ENFORCE(X != nullptr);
+    auto& input_tensor = *X;
+
+    TensorShapeVector axes;
+    size_t num_inputs = ctx->InputCount();
+    if (num_inputs == 2) {
+      const Tensor* axes_tensor = ctx->template Input<Tensor>(1);
+      ORT_ENFORCE(axes_tensor != nullptr, "Axes input is null");
+      ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 0 ||
+                      axes_tensor->Shape().NumDimensions() == 1,
+                  "An axes tensor must be a scalar or a 1-D tensor.");
+      auto data_span = axes_tensor->template DataAsSpan<int64_t>();
+      axes.assign(data_span.begin(), data_span.end());
+    } else {
+      axes.assign(axes_.begin(), axes_.end());
+    }
+
+    TensorShapeVector output_dims(axes.size() + input_tensor.Shape().NumDimensions(), 0);
+
+    for (int64_t axis : axes) {
+      axis = HandleNegativeAxis(axis, onnxruntime::narrow<int64_t>(output_dims.size()));
+      if (axis < 0 || axis >= static_cast<int64_t>(output_dims.size()))
+        return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "'axes' has an out of range axis");
+      if (output_dims[onnxruntime::narrow<size_t>(axis)] != 0)
+        return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "'axes' has a duplicate axis");
+      output_dims[onnxruntime::narrow<size_t>(axis)] = 1;
+    }
+
+    {
+      auto begin = input_tensor.Shape().GetDims().begin();
+      for (auto& axis_size : output_dims) {
+        if (axis_size == 0)
+          axis_size = *begin++;
+      }
+      assert(begin == input_tensor.Shape().GetDims().end());
+    }
+
+    TensorShape output_shape(output_dims);
+    p.output_tensor = ctx->Output(0, output_shape);
+    ORT_ENFORCE(nullptr != p.output_tensor);
+    p.input_tensor = &input_tensor;
+    return Status::OK();
+  }
+#endif
+
   static TensorShapeVector ComputeOutputShape(
       const TensorShape& input_shape,
       const TensorShapeVector& axes) {
diff --git a/onnxruntime/core/providers/cpu/tensor/upsample.cc b/onnxruntime/core/providers/cpu/tensor/upsample.cc
index b533f1b7dc80b..87ba56bc45dad 100644
--- a/onnxruntime/core/providers/cpu/tensor/upsample.cc
+++ b/onnxruntime/core/providers/cpu/tensor/upsample.cc
@@ -2,10 +2,6 @@
 // Licensed under the MIT License.
 
 #include "core/providers/cpu/tensor/upsample.h"
-
-#include <limits>
-
-#include "core/common/inlined_containers.h"
 #include "core/common/safeint.h"
 #include "core/platform/threadpool.h"
 #include "core/providers/cpu/tensor/upsample_antialias.h"
@@ -35,46 +31,6 @@ REGISTER_VERSIONED_TYPED_KERNEL(int32_t, 9, 9);
 REGISTER_VERSIONED_TYPED_KERNEL(int8_t, 9, 9);
 REGISTER_VERSIONED_TYPED_KERNEL(uint8_t, 9, 9);
 
-void UpsampleBase::AdjustOutputSizeAsPolicy(TensorShapeVector& output_dims, gsl::span<const int64_t> input_dims,
-                                            InlinedVector<float>& scales) const {
-  // AspectRatioPolicy::STRETCH is default policy when opset < 18
-  if (keep_aspect_ratio_policy_ == AspectRatioPolicy::STRETCH) {
-    return;
-  }
-
-  InlinedHashSet<int64_t> axes_set(axes_.begin(), axes_.end());
-
-  float scale_in_policy = 0.0f;
-  if (keep_aspect_ratio_policy_ == AspectRatioPolicy ::NOT_LARGER) {
-    scale_in_policy = std::numeric_limits<float>::max();
-
-    for (size_t i = 0; i < scales.size(); i++) {
-      if (axes_set.empty() || axes_set.count(i) > 0) {
-        scale_in_policy = std::min(scale_in_policy, scales[i]);
-      }
-    }
-  } else if (keep_aspect_ratio_policy_ == AspectRatioPolicy ::NOT_SMALLER) {
-    scale_in_policy = std::numeric_limits<float>::min();
-
-    for (size_t i = 0; i < scales.size(); i++) {
-      if (axes_set.empty() || axes_set.count(i) > 0) {
-        scale_in_policy = std::max(scale_in_policy, scales[i]);
-      }
-    }
-  }
-
-  for (size_t i = 0; i < scales.size(); i++) {
-    // if axes is not specified (AKA axes_set.empty()), we apply the policy to all axes
-    if (axes_set.empty() || axes_set.count(i) > 0) {
-      scales[i] = scale_in_policy;
-      output_dims[i] = static_cast<int64_t>(std::round(scales[i] * input_dims[i]));
-    } else {
-      scales[i] = 1.0f;
-      output_dims[i] = input_dims[i];
-    }
-  }
-}
-
 template <typename T>
 void UpsampleNearest2x(int64_t batch_size,
                        int64_t num_channels,
diff --git a/onnxruntime/core/providers/cpu/tensor/upsamplebase.h b/onnxruntime/core/providers/cpu/tensor/upsamplebase.h
index b0e309a70444f..7dcf88133e967 100644
--- a/onnxruntime/core/providers/cpu/tensor/upsamplebase.h
+++ b/onnxruntime/core/providers/cpu/tensor/upsamplebase.h
@@ -4,9 +4,12 @@
 #pragma once
 
 #include <algorithm>
+#include <cmath>
+#include <limits>
 #include <string>
 #include <string_view>
 #include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
 #include <core/common/inlined_containers_fwd.h>
@@ -120,6 +123,49 @@ void PrintAntiAliasBuffers(std::ostream& os, gsl::span<int64_t> bounds, gsl::spa
   os << std::endl;
 }
 
+namespace upsamplebase_helper {
+
+inline void AdjustOutputSizeAsPolicy(TensorShapeVector& output_dims, gsl::span<const int64_t> input_dims,
+                                     InlinedVector<float>& scales, AspectRatioPolicy keep_aspect_ratio_policy,
+                                     const TensorShapeVector& axes) {
+  if (keep_aspect_ratio_policy == AspectRatioPolicy::STRETCH) {
+    return;
+  }
+
+  std::unordered_set<int64_t> axes_set(axes.begin(), axes.end());
+
+  float scale_in_policy = 0.0f;
+  if (keep_aspect_ratio_policy == AspectRatioPolicy::NOT_LARGER) {
+    scale_in_policy = std::numeric_limits<float>::max();
+
+    for (size_t i = 0; i < scales.size(); ++i) {
+      if (axes_set.empty() || axes_set.count(static_cast<int64_t>(i)) > 0) {
+        scale_in_policy = std::min(scale_in_policy, scales[i]);
+      }
+    }
+  } else if (keep_aspect_ratio_policy == AspectRatioPolicy::NOT_SMALLER) {
+    scale_in_policy = std::numeric_limits<float>::min();
+
+    for (size_t i = 0; i < scales.size(); ++i) {
+      if (axes_set.empty() || axes_set.count(static_cast<int64_t>(i)) > 0) {
+        scale_in_policy = std::max(scale_in_policy, scales[i]);
+      }
+    }
+  }
+
+  for (size_t i = 0; i < scales.size(); ++i) {
+    if (axes_set.empty() || axes_set.count(static_cast<int64_t>(i)) > 0) {
+      scales[i] = scale_in_policy;
+      output_dims[i] = static_cast<int64_t>(std::round(scales[i] * input_dims[i]));
+    } else {
+      scales[i] = 1.0f;
+      output_dims[i] = input_dims[i];
+    }
+  }
+}
+
+}  // namespace upsamplebase_helper
+
 class UpsampleBase {
  public:
   // Make this available in other EP via provider bridge
@@ -597,6 +643,13 @@ class UpsampleBase {
   }
 };  // UpsampleBase
 
+#ifndef SHARED_PROVIDER
+inline void UpsampleBase::AdjustOutputSizeAsPolicy(TensorShapeVector& output_dims, gsl::span<const int64_t> input_dims,
+                                                   InlinedVector<float>& scales) const {
+  upsamplebase_helper::AdjustOutputSizeAsPolicy(output_dims, input_dims, scales, keep_aspect_ratio_policy_, axes_);
+}
+#endif
+
 }  // namespace onnxruntime
 #if defined(_MSC_VER) && !defined(__clang__)
 #pragma warning(pop)
diff --git a/onnxruntime/core/providers/cuda/tensor/grid_sample.cc b/onnxruntime/core/providers/cuda/tensor/grid_sample.cc
index b9d47a27e8e83..d97d5fcbb0b5b 100755
--- a/onnxruntime/core/providers/cuda/tensor/grid_sample.cc
+++ b/onnxruntime/core/providers/cuda/tensor/grid_sample.cc
@@ -51,7 +51,6 @@ template <typename T, bool IsNHWC>
 GridSample<T, IsNHWC>::GridSample(const OpKernelInfo& info) : CudaKernel(info) {
   opset_start_version_ = info.node().SinceVersion();
 
-  std::string mode_str = info.GetAttrOrDefault<std::string>("mode", "bilinear");
   std::string padding_mode_str = info.GetAttrOrDefault<std::string>("padding_mode", "zeros");
   align_corners_ = static_cast<bool>(info.GetAttrOrDefault<int64_t>("align_corners", 0));
 
diff --git a/onnxruntime/core/providers/cuda/tensor/upsample.cc b/onnxruntime/core/providers/cuda/tensor/upsample.cc
index e7032d5880581..e2c08618264dd 100644
--- a/onnxruntime/core/providers/cuda/tensor/upsample.cc
+++ b/onnxruntime/core/providers/cuda/tensor/upsample.cc
@@ -380,5 +380,11 @@ Status Upsample<T>::ComputeInternal(OpKernelContext* context) const {
   return BaseCompute(context, roi_array, scales_array, output_dims);
 }
 
+template class Upsample<float>;
+template class Upsample<double>;
+template class Upsample<MLFloat16>;
+template class Upsample<int32_t>;
+template class Upsample<uint8_t>;
+
 }  // namespace cuda
 }  // namespace onnxruntime
diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_utils.h b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_utils.h
index 93f0e47005050..c57322e46bfe8 100644
--- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_utils.h
+++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_utils.h
@@ -45,332 +45,6 @@ static int GetNumProfiles(std::unordered_map<std::string, std::vector<std::vecto
   return num_profile;
 }
 
-/*
- * Seralize engine profile
- * The profile contains min/max shape ranges of dynamic shape dimensions of each input tensor
- * For example, assume tensor_a has two dynamic shape dimensions: dim_0 and dim_2, and tensor_b
- * has one dynamic shape dimension: dim_1. The data in profile will be,
- * key: tensor_a, value: dim_0 min_shape max_shape dim_2 min_shape max_shape
- * key: tensor_b, value: dim_1 min_shape max_shape
- *
- * [Deprecated] Use SerializeProfileV2
- */
-static void SerializeProfile(const std::string& file_name, std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>>& shape_ranges) {
-  // Serialize profile
-  flexbuffers::Builder builder;
-  auto profile_start = builder.StartMap();
-  for (auto outer_it = shape_ranges.begin(); outer_it != shape_ranges.end(); ++outer_it) {
-    builder.TypedVector(outer_it->first.c_str(), [&] {
-      for (auto inner_it = outer_it->second.begin(); inner_it != outer_it->second.end(); ++inner_it) {
-        builder.Int(inner_it->first);
-        builder.Int(inner_it->second.first);
-        builder.Int(inner_it->second.second);
-      }
-    });
-  }
-  builder.EndMap(profile_start);
-  builder.Finish();
-
-  // Save flexbuffer
-  std::ofstream file(file_name, std::ios::binary | std::ios::out);
-  auto buf = builder.GetBuffer();
-  size_t size = builder.GetSize();
-  file.write(reinterpret_cast<const char*>(&buf[0]), size);
-  file.close();
-}
-
-// Deserialize engine profile
-// [Deprecated] Use DeserializeProfileV2
-static std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>> DeserializeProfile(std::ifstream& infile) {
-  // Load flexbuffer
-  infile.seekg(0, std::ios::end);
-  size_t length = infile.tellg();
-  infile.seekg(0, std::ios::beg);
-  std::unique_ptr<char[]> data{new char[length]};
-  infile.read((char*)data.get(), length);
-  infile.close();
-
-  // Deserialize profile
-  std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>> shape_ranges;
-  auto tensors_range_entries = flexbuffers::GetRoot((const uint8_t*)data.get(), length).AsMap();
-  auto keys = tensors_range_entries.Keys();
-  auto values = tensors_range_entries.Values();
-  for (size_t i = 0, i_end = keys.size(); i < i_end; ++i) {
-    auto dim_range_vectors = values[i].AsTypedVector();
-    std::unordered_map<size_t, std::pair<int64_t, int64_t>> inner_map;
-    for (size_t j = 0, j_end = dim_range_vectors.size() / 3; j < j_end; ++j) {
-      size_t idx = 3 * j;
-      inner_map[dim_range_vectors[idx].AsInt64()] = std::make_pair(dim_range_vectors[idx + 1].AsInt64(), dim_range_vectors[idx + 2].AsInt64());
-    }
-    shape_ranges[keys[i].AsString().c_str()] = inner_map;
-  }
-  return shape_ranges;
-}
-
-/*
- * Seralize engine profile. (This function starts from ORT 1.15)
- *
- *
- * (1) Single profile case:
- * Assume tensor_a has two dynamic shape dimensions: dim_0 and dim_2,
- * and tensor_b has one dynamic shape dimension: dim_1.
- *
- * The data before serialization will be:
- * {
- *   tensor_a: {
- *     dim_0: [[min_shape_0, max_shape_0, opt_shape_0]],
- *     dim_2: [[min_shape_2, max_shape_2, opt_shape_2]]
- *   },
- *   tensor_b: {
- *     dim_1: [[min_shape_1, max_shape_1, opt_shape_1]]
- *   }
- * }
- *
- * The data after serialization will be:
- * {
- *   tensor_a: [dim_0, min_shape_0, max_shape_0, opt_shape_0, dim_2, min_shape_2, max_shape_2, opt_shape_2]
- *   tensor_b: [dim_1, min_shape_1, max_shape_1, opt_shape_1]
- * }
- *
- *
- * (2) Multiple profiles case:
- * For example, if the data before serialization is:
- * {
- *   tensor_a: {
- *     dim_0: [[min_shape_0, max_shape_0, opt_shape_0], [min_shape_1, max_shape_1, opt_shape_1]]
- *   },
- *   tensor_b: {
- *     dim_1: [[min_shape_2, max_shape_2, opt_shape_2], [min_shape_3, max_shape_3, opt_shape_3]]
- *   }
- * }
- *
- * The data after serialization will be:
- * {
- *   tensor_a: [dim_0, min_shape_0, max_shape_0, opt_shape_0, dim_0, min_shape_1, max_shape_1, opt_shape_1]
- *              |                                          |  |                                          |
- *              ---------------- profile 0 -----------------  ---------------- profile 1 -----------------
- *
- *   tensor_b: [dim_1, min_shape_2, max_shape_2, opt_shape_2, dim_1, min_shape_3, max_shape_3, opt_shape_3]
- *              |                                          |  |                                          |
- *              ---------------- profile 0 -----------------  ---------------- profile 1 -----------------
- * }
- *
- */
-static void SerializeProfileV2(const std::string& file_name, std::unordered_map<std::string, std::unordered_map<size_t, std::vector<std::vector<int64_t>>>>& shape_ranges) {
-  LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] In SerializeProfileV2()";
-  // Serialize profile
-  flexbuffers::Builder builder;
-  auto tensor_map_start = builder.StartMap();
-  for (auto tensor_it = shape_ranges.begin(); tensor_it != shape_ranges.end(); tensor_it++) {  // iterate tensors
-    LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] input tensor is '" << tensor_it->first.c_str() << "'";
-    builder.TypedVector(tensor_it->first.c_str(), [&] {
-      for (auto dim_it = tensor_it->second.begin(); dim_it != tensor_it->second.end(); dim_it++) {
-        size_t num_profiles = dim_it->second.size();
-        for (size_t i = 0; i < num_profiles; i++) {
-          LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] profile #" << i << ", dim is " << dim_it->first;
-          builder.Int(dim_it->first);
-          builder.Int(dim_it->second[i][0]);
-          builder.Int(dim_it->second[i][1]);
-          builder.Int(dim_it->second[i][2]);
-          LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] " << dim_it->first << ", " << dim_it->second[i][0] << ", " << dim_it->second[i][1] << ", " << dim_it->second[i][2];
-        }
-      }
-    });
-  }
-  builder.EndMap(tensor_map_start);
-  builder.Finish();
-
-  // Save flexbuffer
-  std::ofstream file(file_name, std::ios::binary | std::ios::out);
-  auto buf = builder.GetBuffer();
-  size_t size = builder.GetSize();
-  file.write(reinterpret_cast<const char*>(&buf[0]), size);
-  file.close();
-}
-
-/*
- * Deserialize engine profile. (This function starts from ORT 1.15)
- *
- *
- * (1) Single profile case:
- * Assume tensor_a has two dynamic shape dimensions: dim_0 and dim_2,
- * and tensor_b has one dynamic shape dimension: dim_1.
- *
- * The data in profile file will be:
- * {
- *   tensor_a: [dim_0, min_shape_0, max_shape_0, opt_shape_0, dim_2, min_shape_2, max_shape_2, opt_shape_2]
- *   tensor_b: [dim_1, min_shape_1, max_shape_1, opt_shape_1]
- * }
- *
- * The data after deserialization will be:
- * {
- *   tensor_a: {
- *     dim_0: [[min_shape_0, max_shape_0, opt_shape_0]],
- *     dim_2: [[min_shape_2, max_shape_2, opt_shape_2]]
- *   },
- *   tensor_b: {
- *     dim_1: [[min_shape_1, max_shape_1, opt_shape_1]]
- *   }
- * }
- *
- *
- * (2) Multiple profiles case:
- * For example, if the data in profile file is:
- * {
- *   tensor_a: [dim_0, min_shape_0, max_shape_0, opt_shape_0, dim_0, min_shape_1, max_shape_1, opt_shape_1]
- *              |                                          |  |                                          |
- *              ---------------- profile 0 -----------------  ---------------- profile 1 -----------------
- *
- *   tensor_b: [dim_1, min_shape_2, max_shape_2, opt_shape_2, dim_1, min_shape_3, max_shape_3, opt_shape_3]
- *              |                                          |  |                                          |
- *              ---------------- profile 0 -----------------  ---------------- profile 1 -----------------
- * }
- *
- * The data after deserialization will be:
- * {
- *   tensor_a: {
- *     dim_0: [[min_shape_0, max_shape_0, opt_shape_0], [min_shape_1, max_shape_1, opt_shape_1]]
- *   },
- *   tensor_b: {
- *     dim_1: [[min_shape_2, max_shape_2, opt_shape_2], [min_shape_3, max_shape_3, opt_shape_3]]
- *   }
- * }
- */
-static std::unordered_map<std::string, std::unordered_map<size_t, std::vector<std::vector<int64_t>>>> DeserializeProfileV2(std::ifstream& infile) {
-  LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] In DeserializeProfileV2()";
-  // Load flexbuffer
-  infile.seekg(0, std::ios::end);
-  size_t length = infile.tellg();
-  infile.seekg(0, std::ios::beg);
-  std::unique_ptr<char[]> data{new char[length]};
-  infile.read((char*)data.get(), length);
-  infile.close();
-
-  // Deserialize profile
-  std::unordered_map<std::string, std::unordered_map<size_t, std::vector<std::vector<int64_t>>>> shape_ranges;
-  auto tensors_range_entries = flexbuffers::GetRoot((const uint8_t*)data.get(), length).AsMap();
-  auto keys = tensors_range_entries.Keys();
-  auto values = tensors_range_entries.Values();
-  for (size_t i = 0, end = keys.size(); i < end; ++i) {  // iterate tensors
-    LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] input tensor is '" << keys[i].AsString().c_str() << "'";
-    auto dim_range_vector = values[i].AsTypedVector();
-    std::unordered_map<size_t, std::vector<std::vector<int64_t>>> inner_map;
-    std::vector<std::vector<int64_t>> profile_vector;
-
-    for (size_t k = 0; k < (dim_range_vector.size() / 4); k++) {  // iterate dim, min, max, opt for all profiles
-      std::vector<int64_t> shape_vector;
-      auto idx = 4 * k;
-      auto dim = dim_range_vector[idx].AsInt64();
-      shape_vector.push_back(dim_range_vector[idx + 1].AsInt64());  // min shape
-      shape_vector.push_back(dim_range_vector[idx + 2].AsInt64());  // max shape
-      shape_vector.push_back(dim_range_vector[idx + 3].AsInt64());  // opt shape
-
-      if (inner_map.find(dim) == inner_map.end()) {
-        inner_map[dim] = profile_vector;
-      }
-      inner_map[dim].push_back(shape_vector);
-      LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] " << dim << ", " << shape_vector[0] << ", " << shape_vector[1] << ", " << shape_vector[2];
-    }
-    shape_ranges[keys[i].AsString().c_str()] = inner_map;
-  }
-  return shape_ranges;
-}
-
-/*
- * Compare profile shapes from profile file (.profile) with explicit profile min/max/opt shapes.
- * Return false meaning no need to rebuild engine if everything is same.
- * Otherwise return true and engine needs to be rebuilt.
- */
-static bool CompareProfiles(const std::string& file_name,
-                            std::unordered_map<std::string, std::vector<std::vector<int64_t>>>& profile_min_shapes,
-                            std::unordered_map<std::string, std::vector<std::vector<int64_t>>>& profile_max_shapes,
-                            std::unordered_map<std::string, std::vector<std::vector<int64_t>>>& profile_opt_shapes) {
-  std::ifstream profile_file(file_name, std::ios::binary | std::ios::in);
-  if (!profile_file) {
-    LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] " << file_name << " doesn't exist.";
-    return true;
-  }
-
-  std::unordered_map<std::string, std::unordered_map<size_t, std::vector<std::vector<int64_t>>>> shape_ranges;
-  shape_ranges = DeserializeProfileV2(profile_file);
-
-  /* The format of the two data structures are below, for example:
-   *
-   * shape_ranges:
-   * {
-   *   tensor_a: {
-   *     dim_0: [[min_shape, max_shape, opt_shape]],
-   *     dim_2: [[min_shape, max_shape, opt_shape]]
-   *   },
-   *   tensor_b: {
-   *     dim_1: [[min_shape, max_shape, opt_shape]]
-   *   }
-   * }
-   *
-   * profile_min_shapes:
-   * {
-   *   tensor_a: [[dim_0_value_0, dim_1_value_1, dim_2_value_2]],
-   *   tensor_b: [[dim_0_value_3, dim_1_value_4, dim_2_value_5]]
-   * }
-   *
-   */
-
-  // Check number of dynamic shape inputs
-  if (profile_min_shapes.size() != shape_ranges.size()) {
-    LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] Numbers of dynamic shape inputs are not the same.";
-    return true;
-  }
-
-  // Iterate through shape_ranges map
-  for (auto tensor_it = shape_ranges.begin(); tensor_it != shape_ranges.end(); tensor_it++) {  // iterate tensors
-    auto tensor_name = tensor_it->first;
-    if (profile_min_shapes.find(tensor_name) == profile_min_shapes.end()) {
-      LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] Tensor name '" << tensor_name << "' doesn't exist in trt_profile_min_shapes.";
-      return true;
-    }
-
-    for (auto dim_it = tensor_it->second.begin(); dim_it != tensor_it->second.end(); dim_it++) {  // iterate dimensions
-      auto dim = dim_it->first;
-      auto num_profiles = GetNumProfiles(profile_min_shapes);
-
-      if (dim_it->second.size() != static_cast<size_t>(num_profiles)) {
-        LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] Numbers of profiles are not the same.";
-        return true;
-      }
-
-      for (size_t i = 0; i < dim_it->second.size(); i++) {  // iterate (multiple) profile(s)
-        auto shape_values = dim_it->second[i];
-        if (dim > (profile_min_shapes[tensor_name][i].size() - 1)) {
-          LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] dimension " << dim << " of '" << tensor_name << "' in " << file_name << " exceeds the total dimension of trt_profile_min_shapes.";
-          return true;
-        }
-
-        LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] min shape value of dimension " << dim << " of '" << tensor_name << "' is " << profile_min_shapes[tensor_name][i][dim];
-        LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] min shape value of dimension " << dim << " of '" << tensor_name << "' is " << shape_values[0] << " in " << file_name;
-        if (profile_min_shapes[tensor_name][i][dim] != shape_values[0]) {
-          LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] min shape values of dimension " << dim << " of '" << tensor_name << "' are not the same";
-          return true;
-        }
-
-        LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] max shape value of dimension " << dim << " of '" << tensor_name << "' is " << profile_max_shapes[tensor_name][i][dim];
-        LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] max shape value of dimension " << dim << " of '" << tensor_name << "' is " << shape_values[1] << " in " << file_name;
-        if (profile_max_shapes[tensor_name][i][dim] != shape_values[1]) {
-          LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] max shape values of dimension " << dim << " of '" << tensor_name << "' are not the same";
-          return true;
-        }
-
-        LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] opt shape value of dimension " << dim << " of '" << tensor_name << "' is " << profile_opt_shapes[tensor_name][i][dim];
-        LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] opt shape value of dimension " << dim << " of '" << tensor_name << "' is " << shape_values[2] << " in " << file_name;
-        if (profile_opt_shapes[tensor_name][i][dim] != shape_values[2]) {
-          LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] opt shape values of dimension " << dim << " of '" << tensor_name << "' are not the same";
-          return true;
-        }
-      }
-    }
-  }
-  return false;
-}
-
 /*
  * Get cache by name
  *
@@ -394,37 +68,6 @@ static std::string GetComputeCapability(const cudaDeviceProp& prop) {
   return compute_capability;
 }
 
-/*
- * Get cache by type
- *
- * \param root root path of the cache
- * \param file_extension It could be ".engine", ".profile" or ".timing"
- */
-static std::vector<fs::path> GetCachesByType(const std::string& root, std::string file_extension) {
-  std::vector<fs::path> cache_files;
-  for (const auto& entry : fs::directory_iterator(root)) {
-    if (fs::path(file_extension) == fs::path(entry).extension()) {
-      cache_files.push_back(fs::path(entry));
-    }
-  }
-  return cache_files;
-}
-
-static bool IsCacheExistedByType(const std::string& root, std::string file_extension) {
-  auto cache_files = GetCachesByType(root, file_extension);
-  if (cache_files.size() == 0) {
-    return false;
-  }
-  return true;
-}
-
-static void RemoveCachesByType(const std::string& root, std::string file_extension) {
-  auto cache_files = GetCachesByType(root, file_extension);
-  for (const auto& entry : cache_files) {
-    fs::remove(entry);
-  }
-}
-
 /**
  * <summary>
  * Helper class to generate engine id via model name/model content/env metadata
@@ -631,51 +274,6 @@ static bool ParseProfileShapes(std::string profile_shapes_string, std::unordered
   return true;
 }
 
-static std::vector<std::string> split(const std::string& str, char delimiter) {
-  std::vector<std::string> tokens;
-  std::string token;
-  std::istringstream tokenStream(str);
-  while (std::getline(tokenStream, token, delimiter)) {
-    tokens.push_back(token);
-  }
-  return tokens;
-}
-
-static std::string join(const std::vector<std::string>& vec, const std::string& delimiter) {
-  std::string result;
-  for (size_t i = 0; i < vec.size(); ++i) {
-    result += vec[i];
-    if (i < vec.size() - 1) {
-      result += delimiter;
-    }
-  }
-  return result;
-}
-
-/*
- * Parse engine cache name suffix when user customizes prefix for engine cache name
- *
- * For example:
- * When default subgraph name is "NvExecutionProvider_TRTKernel_graph_torch-jit-export_2068723788287043730_189_189_fp16"
- * This func will generate the suffix "2068723788287043730_189_fp16"
- *
- */
-static std::string GetCacheSuffix(const std::string& fused_node_name, const std::string& trt_node_name_with_precision) {
-  std::vector<std::string> split_fused_node_name = split(fused_node_name, '_');
-  if (split_fused_node_name.size() >= 3) {
-    // Get index of model hash from fused_node_name
-    std::string model_hash = split_fused_node_name[split_fused_node_name.size() - 3];
-    size_t index = fused_node_name.find(model_hash);
-    // Parse suffix from trt_node_name_with_precision, as it has additional precision info
-    std::vector<std::string> suffix_group = split(trt_node_name_with_precision.substr(index), '_');
-    if (suffix_group.size() > 2) {
-      suffix_group.erase(suffix_group.begin() + 2);
-    }
-    return join(suffix_group, "_");
-  }
-  return "";
-}
-
 /*
  * Checks if there is a an element with value `-1` in nvinfer1::Dims
  */
@@ -700,37 +298,4 @@ static bool checkTrtTensorIsDynamic(nvinfer1::ITensor* tensor) {
     return checkTrtDimIsDynamic(tensor->getDimensions());
   }
 }
-
-struct ScopedContext {
-  explicit ScopedContext(int device_id) : pushed_(true) {
-    CUcontext cu_context = 0;
-    CU_CALL_THROW(cuCtxGetCurrent(&cu_context));
-    if (!cu_context) {
-      // cuCtxGetCurrent succeeded but returned nullptr, which indicates that no CUDA context
-      // is currently set for this thread. This implicates that there is not user created context.
-      // We use runtime API to initialize a context for the specified device.
-      CUDA_CALL_THROW(cudaSetDevice(device_id));
-      CU_CALL_THROW(cuCtxGetCurrent(&cu_context));
-    }
-    CU_CALL_THROW(cuCtxPushCurrent(cu_context));
-  }
-
-  /** \brief Push an existing context (e.g. CIG context); pop on destruction. */
-  explicit ScopedContext(CUcontext ctx) : pushed_(ctx != nullptr) {
-    if (ctx != nullptr) {
-      CU_CALL_THROW(cuCtxPushCurrent(ctx));
-    }
-  }
-
-  ScopedContext(const ScopedContext&) = delete;
-
-  ~ScopedContext() {
-    if (pushed_) {
-      cuCtxPopCurrent(nullptr);
-    }
-  }
-
- private:
-  bool pushed_ = true;
-};
 }  // namespace onnxruntime
diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc
index 4d5c5b45f65dd..31ff17f241371 100644
--- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc
+++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc
@@ -16,7 +16,6 @@
 #include "core/framework/plugin_ep_stream.h"
 #include "core/providers/nv_tensorrt_rtx/nv_provider_options.h"
 #include "core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.h"
-#include "core/providers/nv_tensorrt_rtx/nv_execution_provider_utils.h"
 #include "core/providers/cuda/cuda_stream_handle.h"
 
 // D3D12 headers for graphics interop on Windows
@@ -30,6 +29,7 @@
 #include "nv_provider_factory_creator.h"
 #include "nv_data_transfer.h"
 #include "nv_allocator.h"
+#include "nv_scoped_context.h"
 
 using namespace onnxruntime;
 
diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_scoped_context.h b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_scoped_context.h
new file mode 100644
index 0000000000000..8a16533b01c7b
--- /dev/null
+++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_scoped_context.h
@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Licensed under the MIT License.
+
+#include "nv_includes.h"
+#include "core/providers/cuda/cuda_pch.h"
+#include "core/providers/cuda/shared_inc/cuda_call.h"
+
+namespace onnxruntime {
+struct ScopedContext {
+  explicit ScopedContext(int device_id) : pushed_(true) {
+    CUcontext cu_context = 0;
+    CU_CALL_THROW(cuCtxGetCurrent(&cu_context));
+    if (!cu_context) {
+      // cuCtxGetCurrent succeeded but returned nullptr, which indicates that no CUDA context
+      // is currently set for this thread. This implicates that there is not user created context.
+      // We use runtime API to initialize a context for the specified device.
+      CUDA_CALL_THROW(cudaSetDevice(device_id));
+      CU_CALL_THROW(cuCtxGetCurrent(&cu_context));
+    }
+    CU_CALL_THROW(cuCtxPushCurrent(cu_context));
+  }
+
+  /** \brief Push an existing context (e.g. CIG context); pop on destruction. */
+  explicit ScopedContext(CUcontext ctx) : pushed_(ctx != nullptr) {
+    if (ctx != nullptr) {
+      CU_CALL_THROW(cuCtxPushCurrent(ctx));
+    }
+  }
+
+  ScopedContext(const ScopedContext&) = delete;
+
+  ~ScopedContext() {
+    if (pushed_) {
+      cuCtxPopCurrent(nullptr);
+    }
+  }
+
+ private:
+  bool pushed_ = true;
+};
+}  // namespace onnxruntime
diff --git a/onnxruntime/core/providers/vitisai/onnxruntime_providers_vitisai.rc b/onnxruntime/core/providers/vitisai/onnxruntime_providers_vitisai.rc
new file mode 100644
index 0000000000000..968086ebd2613
--- /dev/null
+++ b/onnxruntime/core/providers/vitisai/onnxruntime_providers_vitisai.rc
@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// This file REQUIRES the following external definitions:
+// FILE_NAME, VER_MAJOR, VER_MINOR, VER_BUILD, VER_PRIVATE, and VER_STRING
+
+#include <Winver.h>
+
+#if defined(DEBUG) || defined(_DEBUG)
+#define VER_DEBUG VS_FF_DEBUG
+#else
+#define VER_DEBUG 0
+#endif
+
+// -----------------------------------------------------------------------------
+
+VS_VERSION_INFO VERSIONINFO
+FILEVERSION     VER_MAJOR, VER_MINOR, VER_BUILD, VER_PRIVATE
+PRODUCTVERSION  VER_MAJOR, VER_MINOR, VER_BUILD, VER_PRIVATE
+FILEFLAGSMASK   VS_FFI_FILEFLAGSMASK
+FILEFLAGS       VER_DEBUG
+FILEOS          VOS__WINDOWS32
+FILETYPE        VFT_DLL
+FILESUBTYPE     VFT2_UNKNOWN
+
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904E4"
+        BEGIN
+            VALUE "CompanyName",      "Microsoft Corporation"
+            VALUE "FileDescription",  "ONNX Runtime VitisAI Provider"
+            VALUE "FileVersion",      VER_STRING
+            VALUE "InternalName",     "ONNX Runtime VitisAI Provider"
+            VALUE "LegalCopyright",   "\251 Microsoft Corporation. All rights reserved."
+            VALUE "OriginalFilename", FILE_NAME
+            VALUE "ProductName",      "Microsoft\256 Windows\256 Operating System"
+            VALUE "ProductVersion",   VER_STRING
+        END
+    END
+
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1252
+    END
+END
diff --git a/onnxruntime/python/tools/transformers/fusion_skiplayernorm.py b/onnxruntime/python/tools/transformers/fusion_skiplayernorm.py
index ed067a1362663..743bf50f6c608 100644
--- a/onnxruntime/python/tools/transformers/fusion_skiplayernorm.py
+++ b/onnxruntime/python/tools/transformers/fusion_skiplayernorm.py
@@ -110,9 +110,8 @@ def fuse(self, node, input_name_to_nodes, output_name_to_node):
                     )
                     return
             else:
-                # Shape inference failed. Use default skip_index=1 (no broadcasting) since both
-                # Add inputs have already been verified as non-initializer dynamic tensors above.
-                logger.debug("symbolic shape inference failed, using default skip_index for SkipLayerNormalization")
+                logger.debug("skip SkipLayerNormalization fusion since symbolic shape inference failed")
+                return
 
         gather_path = self.model.match_parent_path(add, ["Gather"], [None])
         if gather_path is not None and self.model.find_graph_input(gather_path[0].input[1]) is None:
diff --git a/onnxruntime/test/providers/cpu/tensor/grid_sample_test_custom.inc b/onnxruntime/test/providers/cpu/tensor/grid_sample_test_custom.inc
index 7fd22cc59745f..2423d7f120b20 100644
--- a/onnxruntime/test/providers/cpu/tensor/grid_sample_test_custom.inc
+++ b/onnxruntime/test/providers/cpu/tensor/grid_sample_test_custom.inc
@@ -37,7 +37,7 @@ TYPED_TEST(GridSampleCustomTest, test_grid_sample_20_4D_linear_zeros_mixed_bound
   test.AddAttribute("padding_mode", padding_mode);
   test.AddAttribute("align_corners", align_corners);
   test.AddOutput<TypeParam>("Y", Y_shape, Y_data);
-  RunTests(test, GetExecutionProviders(20));
+  RunTests(test, GetExecutionProviders());
 }
 
 TYPED_TEST(GridSampleCustomTest, test_grid_sample_20_4D_linear_zeros_mixed_bounds_left_top) {
@@ -69,6 +69,5 @@ TYPED_TEST(GridSampleCustomTest, test_grid_sample_20_4D_linear_zeros_mixed_bound
   test.AddAttribute("padding_mode", padding_mode);
   test.AddAttribute("align_corners", align_corners);
   test.AddOutput<TypeParam>("Y", Y_shape, Y_data);
-  RunTests(test, GetExecutionProviders(20));
+  RunTests(test, GetExecutionProviders());
 }
-
diff --git a/onnxruntime/test/providers/nv_tensorrt_rtx/nv_external_resource_importer_test.cc b/onnxruntime/test/providers/nv_tensorrt_rtx/nv_external_resource_importer_test.cc
index c7fccb086ea19..232477c96b4f4 100644
--- a/onnxruntime/test/providers/nv_tensorrt_rtx/nv_external_resource_importer_test.cc
+++ b/onnxruntime/test/providers/nv_tensorrt_rtx/nv_external_resource_importer_test.cc
@@ -1082,7 +1082,7 @@ TEST_F(NvExecutionProviderExternalResourceImporterTest, FullInferenceWithExterna
     // Configure to use our CUDA stream
     char stream_address[32];
     size_t stream_addr_val = reinterpret_cast<size_t>(ort_api_->SyncStream_GetHandle(ort_stream));
-    sprintf(stream_address, "%llu", static_cast<uint64_t>(stream_addr_val));
+    sprintf_s(stream_address, "%llu", static_cast<uint64_t>(stream_addr_val));
     const char* option_keys[] = {
         // TODO we should no longer require to set the compute stream at this point but there are too many cudaSetDevice calls from allocators and stream handling (NVBUG 5822116)
         onnxruntime::nv::provider_option_names::kUserComputeStream,
@@ -1095,7 +1095,7 @@ TEST_F(NvExecutionProviderExternalResourceImporterTest, FullInferenceWithExterna
     };
     char aux_stream_address[32];
     size_t aux_streams[] = {stream_addr_val};
-    sprintf(aux_stream_address, "%llu", reinterpret_cast<uint64_t>(aux_streams));
+    sprintf_s(aux_stream_address, "%llu", reinterpret_cast<uint64_t>(aux_streams));
     std::string max_shared_mem_size = std::to_string(1024 * 28);  // 28 KiB
     const char* option_values[] = {
         stream_address,
diff --git a/onnxruntime/test/python/transformers/test_attention_fusion.py b/onnxruntime/test/python/transformers/test_attention_fusion.py
index d25432173a8f0..caaaa1aa628cf 100644
--- a/onnxruntime/test/python/transformers/test_attention_fusion.py
+++ b/onnxruntime/test/python/transformers/test_attention_fusion.py
@@ -395,17 +395,17 @@ def test_qwen3_normalization_fusion(self):
         ssln_count = sum(1 for n in nodes if n.op_type == "SkipSimplifiedLayerNormalization")
 
         # 4 RMSNorm patterns: pre-attn, Q-norm, K-norm, post-attn.
-        # Post-attn RMSNorm has an Add parent (residual) → fused as SkipSimplifiedLayerNormalization.
-        # Remaining 3 stay as SimplifiedLayerNormalization.
+        # Fallback for SkipLayerNormalization is disabled, so post-attn RMSNorm does not fuse.
+        # All 4 stay as SimplifiedLayerNormalization.
         self.assertEqual(
             sln_count,
-            3,
-            f"Expected 3 SimplifiedLayerNormalization (pre-attn + Q-norm + K-norm), got {sln_count}",
+            4,
+            f"Expected 4 SimplifiedLayerNormalization (pre-attn + Q-norm + K-norm + post-attn), got {sln_count}",
         )
         self.assertEqual(
             ssln_count,
-            1,
-            f"Expected 1 SkipSimplifiedLayerNormalization (residual + post-attn RMSNorm), got {ssln_count}",
+            0,
+            f"Expected 0 SkipSimplifiedLayerNormalization (residual + post-attn RMSNorm failed to fuse), got {ssln_count}",
         )