[QNN-EP] Implement file mapped weights feature

onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc

@@ -94,6 +94,7 @@ Status GetEpContextFromMainNode(const onnxruntime::Node& main_context_node,
     const std::string& context_binary = node_helper.Get(EP_CACHE_CONTEXT, "");
     return qnn_backend_manager->LoadCachedQnnContextFromBuffer(const_cast<char*>(context_binary.c_str()),
                                                                static_cast<uint64_t>(context_binary.length()),
+                                                               "",
                                                                main_context_node.Name(),
                                                                qnn_models,
                                                                max_spill_fill_size);
@@ -127,6 +128,18 @@ Status GetEpContextFromMainNode(const onnxruntime::Node& main_context_node,
     return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_GRAPH, "The file path in ep_cache_context does not exist or is not accessible.");
   }
 
+  std::string context_binary_path_str = context_binary_path.string();
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+  if (qnn_backend_manager->FileMappingIsEnabled()) {
+    return qnn_backend_manager->LoadCachedQnnContextFromBuffer(nullptr,
+                                                               0,
+                                                               context_binary_path_str,
+                                                               main_context_node.Name(),
+                                                               qnn_models,
+                                                               max_spill_fill_size);
+  }
+#endif
+
   size_t buffer_size{0};
   std::ifstream cache_file(context_binary_path.string().c_str(), std::ifstream::binary);
   ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to open cache file.");
@@ -144,6 +157,7 @@ Status GetEpContextFromMainNode(const onnxruntime::Node& main_context_node,
   cache_file.close();
   return qnn_backend_manager->LoadCachedQnnContextFromBuffer(buffer.get(),
                                                              static_cast<uint64_t>(buffer_size),
+                                                             context_binary_path_str,
                                                              main_context_node.Name(),
                                                              qnn_models,
                                                              max_spill_fill_size);

onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h

@@ -25,13 +25,18 @@
 #include "System/QnnSystemInterface.h"
 
 #include "core/providers/qnn/ort_api.h"
+#include "core/providers/qnn/rpcmem_library.h"
 #include "core/providers/qnn/builder/op_builder_factory.h"
 #include "core/providers/qnn/builder/qnn_context_mem_handle_manager.h"
 #include "core/providers/qnn/builder/qnn_def.h"
 #include "core/providers/qnn/builder/qnn_htp_power_config_manager.h"
 #include "core/providers/qnn/builder/qnn_profile_serializer.h"
 #include "core/providers/qnn/builder/qnn_node_group/qnn_node_group.h"
 
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+#include "core/providers/qnn/builder/qnn_file_mapping_interface.h"
+#endif
+
 namespace onnxruntime {
 namespace qnn {
 
@@ -154,6 +159,7 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   std::unique_ptr<unsigned char[]> GetContextBinaryBuffer(uint64_t& written_buffer_size);
 
   Status LoadCachedQnnContextFromBuffer(char* buffer, uint64_t buffer_length,
+                                        const std::string& context_bin_filepath,
                                         std::string node_name,
                                         std::unordered_map<std::string, std::unique_ptr<qnn::QnnModel>>& qnn_models,
                                         int64_t max_spill_fill_size);
@@ -163,6 +169,8 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   Status SetupBackend(const logging::Logger& logger, bool load_from_cached_context,
                       bool need_load_system_lib, bool share_ep_contexts,
                       bool enable_vtcm_backup_buffer_sharing,
+                      bool enable_file_mapped_weights,
+                      std::shared_ptr<qnn::RpcMemLibrary> rpcmem_library,
                       std::unordered_map<std::string, std::unique_ptr<std::vector<std::string>>>& context_bin_map);
 
   Status CreateHtpPowerCfgId(uint32_t deviceId, uint32_t coreId, uint32_t& htp_power_config_id);
@@ -248,9 +256,34 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   bool ProfilingEnabled() { return profiling_enabled_; }
 #endif
 
+  bool FileMappingIsEnabled() {
+    return file_mapped_weights_enabled_;
+  }
+
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+  Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request,
+                               Qnn_ContextBinaryDmaDataResponse_t* response,
+                               void* const mapped_base_ptr,
+                               const size_t file_size);
+
+  Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, void* mapped_base_ptr);
+#endif
+
   QnnLog_Level_t MapOrtSeverityToQNNLogLevel(logging::Severity ort_log_level);
   static logging::Severity MapQNNLogLevelToOrtSeverity(QnnLog_Level_t qnn_log_level);
 
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+  typedef struct FileMappingCallbackInfo {
+    void* const mapped_file_ptr;
+    const size_t file_size;
+    QnnBackendManager* const backend_manager;
+
+    FileMappingCallbackInfo(void* ptr, size_t size, QnnBackendManager* manager)
+        : mapped_file_ptr(ptr), file_size(size), backend_manager(manager) {}
+
+  } FileMappingCallbackInfo_t;
+#endif
+
  private:
   Status LoadBackend();
 
@@ -268,9 +301,24 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
 
   Status CreateContext(bool enable_htp_weight_sharing);
 
+  Status GetFileSizeIfValid(const std::string& filepath, size_t& file_size);
+
+  Status ReadContextBinIfValid(const std::string& context_bin_filepath,
+                               std::vector<char>& buffer);
+
   Status CreateContextVtcmBackupBufferSharingEnabled(std::unordered_map<std::string,
                                                                         std::unique_ptr<std::vector<std::string>>>& context_bin_map);
 
+  Status CreateContextFromListAsync(const QnnContext_Config_t** configs,
+                                    std::unordered_map<std::string,
+                                                       std::unique_ptr<std::vector<std::string>>>& context_bin_map);
+
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+  Status CreateContextFromListAsyncWithCallback(const QnnContext_Config_t** configs,
+                                                std::unordered_map<std::string,
+                                                                   std::unique_ptr<std::vector<std::string>>>& context_bin_map);
+#endif
+
   Status ReleaseContext();
 
   // Sets the ORT logger and creates a corresponding QNN logger with the same log level.
@@ -455,6 +503,15 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   bool context_created_ = false;
   bool backend_setup_completed_ = false;
   bool vtcm_backup_buffer_sharing_enabled_ = false;
+  bool file_mapped_weights_enabled_ = false;
+
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+  std::unique_ptr<FileMappingInterface> file_mapper_ = nullptr;
+  // Notify params for file mapping must persist throughout lifetime of
+  // QnnBackendManager for release of DMA data callback on destruction
+  std::vector<std::unique_ptr<FileMappingCallbackInfo_t>> file_mapping_notify_params_;
+#endif
+
   // NPU backend requires quantized model
   QnnBackendType qnn_backend_type_ = QnnBackendType::CPU;
   Qnn_ProfileHandle_t profile_backend_handle_ = nullptr;
@@ -473,6 +530,8 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   // Mapping of thread id to on-run-start/end power configs
   std::mutex per_thread_power_configs_mutex_;
   std::unordered_map<std::thread::id, PerThreadHtpPowerConfigs_t> per_thread_power_configs_;
+
+  std::shared_ptr<qnn::RpcMemLibrary> rpcmem_library_ = nullptr;
 };
 
 }  // namespace qnn

onnxruntime/core/providers/qnn/builder/qnn_def.h

@@ -19,6 +19,12 @@ namespace qnn {
 #define QNN_SYSTEM_PROFILE_API_ENABLED
 #endif
 
+#if defined(_WIN32) && (defined(__aarch64__) || defined(_M_ARM64))
+#if QNN_API_VERSION_MAJOR > 2 || ((QNN_API_VERSION_MAJOR) == 2 && (QNN_API_VERSION_MINOR >= 32))
+#define QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+#endif
+#endif
+
 // QNN only support subset of POSIX of dlopen/dlsym/dladdr/dlerror/dlclose
 // except the following flags for dlopen, others should be done only
 // when we really need them

onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <string>
+
+#include <QnnContext.h>
+
+#include "core/providers/qnn/ort_api.h"
+#include "core/providers/qnn/builder/qnn_def.h"
+
+namespace onnxruntime {
+namespace qnn {
+
+class FileMappingInterface {
+ public:
+  virtual ~FileMappingInterface() = default;
+
+  virtual Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath,
+                                              void** mapped_data_ptr) = 0;
+};
+
+}  // namespace qnn
+}  // namespace onnxruntime

onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc

@@ -0,0 +1,113 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/providers/qnn/builder/qnn_windows_file_mapper.h"
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+
+#include <wil/filesystem.h>
+
+#include <utility>
+
+#include <QnnContext.h>
+
+#include "core/providers/qnn/ort_api.h"
+
+namespace onnxruntime {
+namespace qnn {
+
+WindowsFileMapper::WindowsFileMapper(const logging::Logger& logger)
+    : logger_(&logger) {
+}
+
+WindowsFileMapper::~WindowsFileMapper() {
+}
+
+static void UnmapFile(void* addr) noexcept {
+  bool successful = UnmapViewOfFile(addr);
+  if (!successful) {
+    const auto error_code = GetLastError();
+    LOGS_DEFAULT(ERROR) << "Failed to unmap view of file with ptr: " << addr
+                        << ", Error code: " << error_code << ", \""
+                        << std::system_category().message(error_code) << "\"";
+  }
+}
+
+Status WindowsFileMapper::GetContextBinMappedMemoryPtr(const std::string& bin_filepath,
+                                                       void** mapped_data_ptr) {
+  LOGS(*logger_, INFO) << "Creating context bin file mapping for "
+                       << bin_filepath;
+
+  ORT_RETURN_IF(bin_filepath.empty(), "Context bin file path is empty");
+
+  std::lock_guard<std::mutex> lock(map_mutex_);
+  auto map_it = mapped_memory_ptrs_.find(bin_filepath);
+  if (map_it != mapped_memory_ptrs_.end()) {
+    *mapped_data_ptr = map_it->second.get();
+    LOGS(*logger_, INFO) << "Found existing mapview memory pointer (" << mapped_data_ptr
+                         << ") for context bin file: " << bin_filepath;
+    return Status::OK();
+  }
+
+  std::wstring bin_filepath_wstr(bin_filepath.begin(), bin_filepath.end());
+  wil::unique_hfile file_handle{CreateFile2(bin_filepath_wstr.c_str(),
+                                            GENERIC_READ,
+                                            FILE_SHARE_READ,
+                                            OPEN_EXISTING,
+                                            NULL)};
+  if (file_handle.get() == INVALID_HANDLE_VALUE) {
+    const auto error_code = GetLastError();
+    return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
+                           "Failed to create file handle for context bin", bin_filepath,
+                           ". Error code: ", error_code, ", \"",
+                           std::system_category().message(error_code), "\"");
+  }
+
+  LOGS(*logger_, VERBOSE) << "Created file handle (" << file_handle.get() << ") for context bin: "
+                          << bin_filepath;
+
+  wil::unique_hfile file_mapping_handle{CreateFileMappingW(file_handle.get(),
+                                                           nullptr,
+                                                           PAGE_READONLY,
+                                                           0x00,
+                                                           0x00,
+                                                           nullptr)};
+  if (file_mapping_handle.get() == INVALID_HANDLE_VALUE) {
+    const auto error_code = GetLastError();
+    return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
+                           "Failed to create file mapping handle for context bin",
+                           bin_filepath, ". Error code: ", error_code, ", \"",
+                           std::system_category().message(error_code), "\"");
+  }
+
+  LOGS(*logger_, VERBOSE) << "Created file mapping with handle (" << file_mapping_handle.get()
+                          << ") for context bin:" << bin_filepath;
+
+  void* const mapped_base_ptr = MapViewOfFile(file_mapping_handle.get(),
+                                              FILE_MAP_READ,
+                                              0, 0, 0);
+
+  if (mapped_base_ptr == nullptr) {
+    const auto error_code = GetLastError();
+    return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
+                           "Failed to retrieve mapview pointer for context bin",
+                           bin_filepath, ". Error code: ", error_code, ", \"",
+                           std::system_category().message(error_code), "\"");
+  }
+
+  LOGS(*logger_, INFO) << "Created mapview pointer with address " << mapped_base_ptr
+                       << " for context bin " << bin_filepath;
+
+  onnxruntime::Env::MappedMemoryPtr mapped_memory_ptr{reinterpret_cast<char*>(mapped_base_ptr),
+                                                      [mapped_base_ptr](void*) {
+                                                        UnmapFile(mapped_base_ptr);
+                                                      }};
+
+  *mapped_data_ptr = mapped_memory_ptr.get();
+  mapped_memory_ptrs_.emplace(bin_filepath, std::move(mapped_memory_ptr));
+
+  return Status::OK();
+}
+}  // namespace qnn
+}  // namespace onnxruntime
+
+#endif  // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE

onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.h

@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/providers/qnn/builder/qnn_file_mapping_interface.h"
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+
+#include <memory>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+
+#include <QnnContext.h>
+
+#include "core/providers/qnn/ort_api.h"
+
+namespace onnxruntime {
+namespace qnn {
+
+class WindowsFileMapper : public FileMappingInterface {
+ public:
+  explicit WindowsFileMapper(const logging::Logger& logger);
+  ~WindowsFileMapper() override;
+
+  // Creates a file mapping of the context binary and returns the
+  // mapview pointer of the file mapping
+  Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath,
+                                      void** mapped_data_ptr) override;
+
+ private:
+  // A container of smart pointers of mapview memory pointers to mapped context bins
+  // key: filepath to context bin, value: smart pointer of mapview memory pointers
+  std::mutex map_mutex_;
+  std::unordered_map<std::string, onnxruntime::Env::MappedMemoryPtr> mapped_memory_ptrs_;
+  const logging::Logger* logger_;
+};
+
+}  // namespace qnn
+}  // namespace onnxruntime
+
+#endif  // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE