[QNN-EP] Implement file mapped weights feature (#26952)

Description
Enables the file mapping of weights as well as the overall context bin.
This feature is currently only enabled for ARM64 WIN devices

Motivation and Context
Currently, when reading the context bin, ORT allocates a large buffer on
the heap. Assuming the same model is used, each ORT session will
allocate a buffer for the context bin. This is incredibly wasteful when
large models are used. Instead, WIN file mapping can be leveraged to map
the context bin, then every time a context needs to be created with the
context bin, the pointer to the context bin can be retrieved and used
instead of some pre-allocated buffer, thus making QNN EP more
memory-efficient. In the case of multiple ORT sessions, the context bin
will only be loaded once for all sessions, increasing memory efficiency
and overall initialization performance. This is very useful regarding
the use of LLMs going forward.

---------

Co-authored-by: quic_calvnguy <quic_calvnguy@quic_inc.com>

Author: quic-calvnguy

onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc

@@ -94,6 +94,7 @@ Status GetEpContextFromMainNode(const onnxruntime::Node& main_context_node,
     const std::string& context_binary = node_helper.Get(EP_CACHE_CONTEXT, "");
     return qnn_backend_manager->LoadCachedQnnContextFromBuffer(const_cast<char*>(context_binary.c_str()),
                                                                static_cast<uint64_t>(context_binary.length()),
+                                                               "",
                                                                main_context_node.Name(),
                                                                qnn_models,
                                                                max_spill_fill_size);
@@ -127,6 +128,18 @@ Status GetEpContextFromMainNode(const onnxruntime::Node& main_context_node,
     return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_GRAPH, "The file path in ep_cache_context does not exist or is not accessible.");
   }
 
+  std::string context_binary_path_str = context_binary_path.string();
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+  if (qnn_backend_manager->FileMappingIsEnabled()) {
+    return qnn_backend_manager->LoadCachedQnnContextFromBuffer(nullptr,
+                                                               0,
+                                                               context_binary_path_str,
+                                                               main_context_node.Name(),
+                                                               qnn_models,
+                                                               max_spill_fill_size);
+  }
+#endif
+
   size_t buffer_size{0};
   std::ifstream cache_file(context_binary_path.string().c_str(), std::ifstream::binary);
   ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to open cache file.");
@@ -144,6 +157,7 @@ Status GetEpContextFromMainNode(const onnxruntime::Node& main_context_node,
   cache_file.close();
   return qnn_backend_manager->LoadCachedQnnContextFromBuffer(buffer.get(),
                                                              static_cast<uint64_t>(buffer_size),
+                                                             context_binary_path_str,
                                                              main_context_node.Name(),
                                                              qnn_models,
                                                              max_spill_fill_size);

onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc

@@ -25,13 +25,18 @@
 #include "System/QnnSystemInterface.h"
 
 #include "core/providers/qnn/ort_api.h"
+#include "core/providers/qnn/rpcmem_library.h"
 #include "core/providers/qnn/builder/op_builder_factory.h"
 #include "core/providers/qnn/builder/qnn_context_mem_handle_manager.h"
 #include "core/providers/qnn/builder/qnn_def.h"
 #include "core/providers/qnn/builder/qnn_htp_power_config_manager.h"
 #include "core/providers/qnn/builder/qnn_profile_serializer.h"
 #include "core/providers/qnn/builder/qnn_node_group/qnn_node_group.h"
 
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+#include "core/providers/qnn/builder/qnn_file_mapping_interface.h"
+#endif
+
 namespace onnxruntime {
 namespace qnn {
 
@@ -154,6 +159,7 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   std::unique_ptr<unsigned char[]> GetContextBinaryBuffer(uint64_t& written_buffer_size);
 
   Status LoadCachedQnnContextFromBuffer(char* buffer, uint64_t buffer_length,
+                                        const std::string& context_bin_filepath,
                                         std::string node_name,
                                         std::unordered_map<std::string, std::unique_ptr<qnn::QnnModel>>& qnn_models,
                                         int64_t max_spill_fill_size);
@@ -163,6 +169,8 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   Status SetupBackend(const logging::Logger& logger, bool load_from_cached_context,
                       bool need_load_system_lib, bool share_ep_contexts,
                       bool enable_vtcm_backup_buffer_sharing,
+                      bool enable_file_mapped_weights,
+                      std::shared_ptr<qnn::RpcMemLibrary> rpcmem_library,
                       std::unordered_map<std::string, std::unique_ptr<std::vector<std::string>>>& context_bin_map);
 
   Status CreateHtpPowerCfgId(uint32_t deviceId, uint32_t coreId, uint32_t& htp_power_config_id);
@@ -248,9 +256,34 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   bool ProfilingEnabled() { return profiling_enabled_; }
 #endif
 
+  bool FileMappingIsEnabled() {
+    return file_mapped_weights_enabled_;
+  }
+
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+  Qnn_ErrorHandle_t MapDmaData(Qnn_ContextBinaryDataRequest_t request,
+                               Qnn_ContextBinaryDmaDataResponse_t* response,
+                               void* const mapped_base_ptr,
+                               const size_t file_size);
+
+  Qnn_ErrorHandle_t ReleaseDmaData(Qnn_ContextBinaryDmaDataMem_t data_mem, void* mapped_base_ptr);
+#endif
+
   QnnLog_Level_t MapOrtSeverityToQNNLogLevel(logging::Severity ort_log_level);
   static logging::Severity MapQNNLogLevelToOrtSeverity(QnnLog_Level_t qnn_log_level);
 
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+  typedef struct FileMappingCallbackInfo {
+    void* const mapped_file_ptr;
+    const size_t file_size;
+    QnnBackendManager* const backend_manager;
+
+    FileMappingCallbackInfo(void* ptr, size_t size, QnnBackendManager* manager)
+        : mapped_file_ptr(ptr), file_size(size), backend_manager(manager) {}
+
+  } FileMappingCallbackInfo_t;
+#endif
+
  private:
   Status LoadBackend();
 
@@ -268,9 +301,24 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
 
   Status CreateContext(bool enable_htp_weight_sharing);
 
+  Status GetFileSizeIfValid(const std::string& filepath, size_t& file_size);
+
+  Status ReadContextBinIfValid(const std::string& context_bin_filepath,
+                               std::vector<char>& buffer);
+
   Status CreateContextVtcmBackupBufferSharingEnabled(std::unordered_map<std::string,
                                                                         std::unique_ptr<std::vector<std::string>>>& context_bin_map);
 
+  Status CreateContextFromListAsync(const QnnContext_Config_t** configs,
+                                    std::unordered_map<std::string,
+                                                       std::unique_ptr<std::vector<std::string>>>& context_bin_map);
+
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+  Status CreateContextFromListAsyncWithCallback(const QnnContext_Config_t** configs,
+                                                std::unordered_map<std::string,
+                                                                   std::unique_ptr<std::vector<std::string>>>& context_bin_map);
+#endif
+
   Status ReleaseContext();
 
   // Sets the ORT logger and creates a corresponding QNN logger with the same log level.
@@ -455,6 +503,15 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   bool context_created_ = false;
   bool backend_setup_completed_ = false;
   bool vtcm_backup_buffer_sharing_enabled_ = false;
+  bool file_mapped_weights_enabled_ = false;
+
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+  std::unique_ptr<FileMappingInterface> file_mapper_ = nullptr;
+  // Notify params for file mapping must persist throughout lifetime of
+  // QnnBackendManager for release of DMA data callback on destruction
+  std::vector<std::unique_ptr<FileMappingCallbackInfo_t>> file_mapping_notify_params_;
+#endif
+
   // NPU backend requires quantized model
   QnnBackendType qnn_backend_type_ = QnnBackendType::CPU;
   Qnn_ProfileHandle_t profile_backend_handle_ = nullptr;
@@ -473,6 +530,8 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   // Mapping of thread id to on-run-start/end power configs
   std::mutex per_thread_power_configs_mutex_;
   std::unordered_map<std::thread::id, PerThreadHtpPowerConfigs_t> per_thread_power_configs_;
+
+  std::shared_ptr<qnn::RpcMemLibrary> rpcmem_library_ = nullptr;
 };
 
 }  // namespace qnn

onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h

@@ -19,6 +19,12 @@ namespace qnn {
 #define QNN_SYSTEM_PROFILE_API_ENABLED
 #endif
 
+#if defined(_WIN32) && (defined(__aarch64__) || defined(_M_ARM64))
+#if QNN_API_VERSION_MAJOR > 2 || ((QNN_API_VERSION_MAJOR) == 2 && (QNN_API_VERSION_MINOR >= 32))
+#define QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+#endif
+#endif
+
 // QNN only support subset of POSIX of dlopen/dlsym/dladdr/dlerror/dlclose
 // except the following flags for dlopen, others should be done only
 // when we really need them

onnxruntime/core/providers/qnn/builder/qnn_def.h

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <string>
+
+#include <QnnContext.h>
+
+#include "core/providers/qnn/ort_api.h"
+#include "core/providers/qnn/builder/qnn_def.h"
+
+namespace onnxruntime {
+namespace qnn {
+
+class FileMappingInterface {
+ public:
+  virtual ~FileMappingInterface() = default;
+
+  virtual Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath,
+                                              void** mapped_data_ptr) = 0;
+};
+
+}  // namespace qnn
+}  // namespace onnxruntime

onnxruntime/core/providers/qnn/builder/qnn_file_mapping_interface.h

@@ -0,0 +1,113 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/providers/qnn/builder/qnn_windows_file_mapper.h"
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+
+#include <wil/filesystem.h>
+
+#include <utility>
+
+#include <QnnContext.h>
+
+#include "core/providers/qnn/ort_api.h"
+
+namespace onnxruntime {
+namespace qnn {
+
+WindowsFileMapper::WindowsFileMapper(const logging::Logger& logger)
+    : logger_(&logger) {
+}
+
+WindowsFileMapper::~WindowsFileMapper() {
+}
+
+static void UnmapFile(void* addr) noexcept {
+  bool successful = UnmapViewOfFile(addr);
+  if (!successful) {
+    const auto error_code = GetLastError();
+    LOGS_DEFAULT(ERROR) << "Failed to unmap view of file with ptr: " << addr
+                        << ", Error code: " << error_code << ", \""
+                        << std::system_category().message(error_code) << "\"";
+  }
+}
+
+Status WindowsFileMapper::GetContextBinMappedMemoryPtr(const std::string& bin_filepath,
+                                                       void** mapped_data_ptr) {
+  LOGS(*logger_, INFO) << "Creating context bin file mapping for "
+                       << bin_filepath;
+
+  ORT_RETURN_IF(bin_filepath.empty(), "Context bin file path is empty");
+
+  std::lock_guard<std::mutex> lock(map_mutex_);
+  auto map_it = mapped_memory_ptrs_.find(bin_filepath);
+  if (map_it != mapped_memory_ptrs_.end()) {
+    *mapped_data_ptr = map_it->second.get();
+    LOGS(*logger_, INFO) << "Found existing mapview memory pointer (" << mapped_data_ptr
+                         << ") for context bin file: " << bin_filepath;
+    return Status::OK();
+  }
+
+  std::wstring bin_filepath_wstr(bin_filepath.begin(), bin_filepath.end());
+  wil::unique_hfile file_handle{CreateFile2(bin_filepath_wstr.c_str(),
+                                            GENERIC_READ,
+                                            FILE_SHARE_READ,
+                                            OPEN_EXISTING,
+                                            NULL)};
+  if (file_handle.get() == INVALID_HANDLE_VALUE) {
+    const auto error_code = GetLastError();
+    return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
+                           "Failed to create file handle for context bin", bin_filepath,
+                           ". Error code: ", error_code, ", \"",
+                           std::system_category().message(error_code), "\"");
+  }
+
+  LOGS(*logger_, VERBOSE) << "Created file handle (" << file_handle.get() << ") for context bin: "
+                          << bin_filepath;
+
+  wil::unique_hfile file_mapping_handle{CreateFileMappingW(file_handle.get(),
+                                                           nullptr,
+                                                           PAGE_READONLY,
+                                                           0x00,
+                                                           0x00,
+                                                           nullptr)};
+  if (file_mapping_handle.get() == INVALID_HANDLE_VALUE) {
+    const auto error_code = GetLastError();
+    return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
+                           "Failed to create file mapping handle for context bin",
+                           bin_filepath, ". Error code: ", error_code, ", \"",
+                           std::system_category().message(error_code), "\"");
+  }
+
+  LOGS(*logger_, VERBOSE) << "Created file mapping with handle (" << file_mapping_handle.get()
+                          << ") for context bin:" << bin_filepath;
+
+  void* const mapped_base_ptr = MapViewOfFile(file_mapping_handle.get(),
+                                              FILE_MAP_READ,
+                                              0, 0, 0);
+
+  if (mapped_base_ptr == nullptr) {
+    const auto error_code = GetLastError();
+    return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
+                           "Failed to retrieve mapview pointer for context bin",
+                           bin_filepath, ". Error code: ", error_code, ", \"",
+                           std::system_category().message(error_code), "\"");
+  }
+
+  LOGS(*logger_, INFO) << "Created mapview pointer with address " << mapped_base_ptr
+                       << " for context bin " << bin_filepath;
+
+  onnxruntime::Env::MappedMemoryPtr mapped_memory_ptr{reinterpret_cast<char*>(mapped_base_ptr),
+                                                      [mapped_base_ptr](void*) {
+                                                        UnmapFile(mapped_base_ptr);
+                                                      }};
+
+  *mapped_data_ptr = mapped_memory_ptr.get();
+  mapped_memory_ptrs_.emplace(bin_filepath, std::move(mapped_memory_ptr));
+
+  return Status::OK();
+}
+}  // namespace qnn
+}  // namespace onnxruntime
+
+#endif  // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE

onnxruntime/core/providers/qnn/builder/qnn_windows_file_mapper.cc

@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/providers/qnn/builder/qnn_file_mapping_interface.h"
+#ifdef QNN_FILE_MAPPED_WEIGHTS_AVAILABLE
+
+#include <memory>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+
+#include <QnnContext.h>
+
+#include "core/providers/qnn/ort_api.h"
+
+namespace onnxruntime {
+namespace qnn {
+
+class WindowsFileMapper : public FileMappingInterface {
+ public:
+  explicit WindowsFileMapper(const logging::Logger& logger);
+  ~WindowsFileMapper() override;
+
+  // Creates a file mapping of the context binary and returns the
+  // mapview pointer of the file mapping
+  Status GetContextBinMappedMemoryPtr(const std::string& bin_filepath,
+                                      void** mapped_data_ptr) override;
+
+ private:
+  // A container of smart pointers of mapview memory pointers to mapped context bins
+  // key: filepath to context bin, value: smart pointer of mapview memory pointers
+  std::mutex map_mutex_;
+  std::unordered_map<std::string, onnxruntime::Env::MappedMemoryPtr> mapped_memory_ptrs_;
+  const logging::Logger* logger_;
+};
+
+}  // namespace qnn
+}  // namespace onnxruntime
+
+#endif  // QNN_FILE_MAPPED_WEIGHTS_AVAILABLE