ORT perf test support for plugin EP

onnxruntime/test/perftest/command_args_parser.cc

@@ -39,8 +39,8 @@ namespace perftest {
       "\t-A: Disable memory arena\n"
       "\t-I: Generate tensor input binding. Free dimensions are treated as 1 unless overridden using -f.\n"
       "\t-c [parallel runs]: Specifies the (max) number of runs to invoke simultaneously. Default:1.\n"
-      "\t-e [cpu|cuda|dnnl|tensorrt|openvino|dml|acl|nnapi|coreml|qnn|snpe|rocm|migraphx|xnnpack|vitisai|webgpu]: Specifies the provider 'cpu','cuda','dnnl','tensorrt', "
-      "'nvtensorrtrtx', 'openvino', 'dml', 'acl', 'nnapi', 'coreml', 'qnn', 'snpe', 'rocm', 'migraphx', 'xnnpack', 'vitisai' or 'webgpu'. "
+      "\t-e [cpu|cuda|dnnl|tensorrt|openvino|dml|acl|nnapi|coreml|qnn|snpe|rocm|migraphx|xnnpack|vitisai|webgpu|plugin_ep]: Specifies the provider 'cpu','cuda','dnnl','tensorrt', "
+      "'nvtensorrtrtx', 'openvino', 'dml', 'acl', 'nnapi', 'coreml', 'qnn', 'snpe', 'rocm', 'migraphx', 'xnnpack', 'vitisai', 'webgpu' or plugin execution provider that provided via ep library. "
       "Default:'cpu'.\n"
       "\t-b [tf|ort]: backend to use. Default:ort\n"
       "\t-r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000.\n"
@@ -195,7 +195,7 @@ static bool ParseDimensionOverride(std::basic_string<ORTCHAR_T>& dim_identifier,
 
 /*static*/ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]) {
   int ch;
-  while ((ch = getopt(argc, argv, ORT_TSTR("m:e:r:t:p:x:y:c:d:o:u:i:f:F:S:T:C:AMPIDZvhsqznlgR:X"))) != -1) {
+  while ((ch = getopt(argc, argv, ORT_TSTR("m:e:r:t:p:x:y:c:d:o:u:i:f:F:S:T:C:AMPIDZvhsqznlgR:XL:"))) != -1) {
     switch (ch) {
       case 'f': {
         std::basic_string<ORTCHAR_T> dim_name;
@@ -273,7 +273,8 @@ static bool ParseDimensionOverride(std::basic_string<ORTCHAR_T>& dim_identifier,
         } else if (!CompareCString(optarg, ORT_TSTR("nvtensorrtrtx"))) {
           test_config.machine_config.provider_type_name = onnxruntime::kNvTensorRTRTXExecutionProvider;
         } else {
-          return false;
+          // Could be plugin EP, save it first and handle later.
+          test_config.machine_config.provider_type_name = ToUTF8String(optarg);
         }
         break;
       case 'r':
@@ -404,6 +405,9 @@ static bool ParseDimensionOverride(std::basic_string<ORTCHAR_T>& dim_identifier,
       case 'X':
         test_config.run_config.use_extensions = true;
         break;
+      case 'L':
+        test_config.plugin_ep_names_and_libs = optarg;
+        break;
       case '?':
       case 'h':
       default:

onnxruntime/test/perftest/main.cc

@@ -6,6 +6,7 @@
 #include <random>
 #include "command_args_parser.h"
 #include "performance_runner.h"
+#include "strings_helper.h"
 #include <google/protobuf/stubs/common.h>
 
 using namespace onnxruntime;
@@ -41,23 +42,50 @@ int real_main(int argc, char* argv[]) {
     if (failed)
       return -1;
   }
-  std::random_device rd;
-  perftest::PerformanceRunner perf_runner(env, test_config, rd);
 
-  // Exit if user enabled -n option so that user can measure session creation time
-  if (test_config.run_config.exit_after_session_creation) {
-    perf_runner.LogSessionCreationTime();
-    return 0;
+  auto status = Status::OK();
+
+  {
+    std::random_device rd;
+    perftest::PerformanceRunner perf_runner(env, test_config, rd);
+
+    // Exit if user enabled -n option so that user can measure session creation time
+    if (test_config.run_config.exit_after_session_creation) {
+      perf_runner.LogSessionCreationTime();
+      return 0;
+    }
+
+    status = perf_runner.Run();
+
+    if (!status.IsOK()) {
+      printf("Run failed:%s\n", status.ErrorMessage().c_str());
+    } else {
+      perf_runner.SerializeResult();
+    }
+  }
+
+  // Unregister all registered plugin EP libraries before program exits.
+  //
+  // This is necessary because unregistering the plugin EP also unregisters any associated shared allocators.
+  // If we don't do this first and program returns, the factories stored inside the environment will be destroyed when the environment goes out of scope.
+  // Later, when the shared allocator's deleter runs, it may cause a segmentation fault because it attempts to use the already-destroyed factory to call ReleaseAllocator.
+  //
+  // See "ep_device.ep_factory->ReleaseAllocator" in Environment::CreateSharedAllocatorImpl.
+  std::unordered_map<std::string, std::string> ep_names_to_libs;
+#ifdef _MSC_VER
+  std::string ep_names_and_libs_string = ToUTF8String(test_config.plugin_ep_names_and_libs);
+#else
+  std::string ep_names_and_libs_string = test_config.plugin_ep_names_and_libs;
+#endif
+  onnxruntime::perftest::ParseSessionConfigs(ep_names_and_libs_string, ep_names_to_libs);
+  for (auto& pair : ep_names_to_libs) {
+    env.UnregisterExecutionProviderLibrary(pair.first.c_str());
   }
 
-  auto status = perf_runner.Run();
   if (!status.IsOK()) {
-    printf("Run failed:%s\n", status.ErrorMessage().c_str());
     return -1;
   }
 
-  perf_runner.SerializeResult();
-
   return 0;
 }
 

onnxruntime/test/perftest/ort_test_session.cc

@@ -62,6 +62,56 @@
     : rand_engine_(rd()), input_names_(m.GetInputCount()), input_names_str_(m.GetInputCount()), input_length_(m.GetInputCount()) {
   Ort::SessionOptions session_options;
 
+#ifdef _MSC_VER
+  std::string ep_names_and_libs_string = ToUTF8String(performance_test_config.plugin_ep_names_and_libs);
+#else
+  std::string ep_names_and_libs_string = performance_test_config.plugin_ep_names_and_libs;
+#endif
+  std::unordered_map<std::string, std::string> ep_names_to_libs;
+  ParseSessionConfigs(ep_names_and_libs_string, ep_names_to_libs);
+  bool is_plugin_ep_avaiable = false;
+
+  if (ep_names_to_libs.size() > 0) {
+    // Register plugin EP libraries if provided via "-L" argument.
+    for (auto& pair : ep_names_to_libs) {
+      const std::filesystem::path library_path = pair.second;
+      const std::string registration_name = pair.first;
+      env.RegisterExecutionProviderLibrary(registration_name.c_str(), library_path.c_str());
+      registered_plugin_ep_names_.push_back(registration_name);
+    }
+
+    std::vector<Ort::ConstEpDevice> ep_devices = env.GetEpDevices();
+    std::vector<Ort::ConstEpDevice> added_ep_devices;
+
+    // All OrtEpDevice instances must be from the same execution provider.
+    // Find the OrtEpDevice associated with the execution provider provided via "-e" argument.
+    Ort::ConstEpDevice plugin_ep_device;
+    for (Ort::ConstEpDevice& device : ep_devices) {
+      if (std::string(device.EpName()) == performance_test_config.machine_config.provider_type_name) {
+        plugin_ep_device = device;
+        added_ep_devices.push_back(plugin_ep_device);
+      }
+    }
+
+    if (added_ep_devices.empty()) {
+      for (auto ep_name : registered_plugin_ep_names_) {
+        env.UnregisterExecutionProviderLibrary(ep_name.c_str());
+      }
+      ORT_THROW(
+          "[ERROR] [plugin EP] No matching execution provider name found in EP library's factory.");
+    }
+
+#if defined(_MSC_VER)
+    std::string provider_option_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string);
+#else
+    std::string provider_option_string = performance_test_config.run_config.ep_runtime_config_string;
+#endif
+    std::unordered_map<std::string, std::string> provider_options;
+    ParseSessionConfigs(provider_option_string, provider_options);
+    session_options.AppendExecutionProvider_V2(env, added_ep_devices, provider_options);
+    is_plugin_ep_avaiable = true;
+  }
+
   provider_name_ = performance_test_config.machine_config.provider_type_name;
   std::unordered_map<std::string, std::string> provider_options;
   if (provider_name_ == onnxruntime::kDnnlExecutionProvider) {
@@ -574,7 +624,8 @@
 #endif
   } else if (!provider_name_.empty() &&
              provider_name_ != onnxruntime::kCpuExecutionProvider &&
-             provider_name_ != onnxruntime::kOpenVINOExecutionProvider) {
+             provider_name_ != onnxruntime::kOpenVINOExecutionProvider &&
+             !is_plugin_ep_avaiable) {
     ORT_THROW("This backend is not included in perf test runner.\n");
   }
 

onnxruntime/test/perftest/ort_test_session.h

@@ -51,6 +51,7 @@ class OnnxRuntimeTestSession : public TestSession {
   const int input_length_;
   std::string provider_name_;
   std::string device_memory_name_;  // Device memory type name to use from the list in allocator.h
+  std::vector<std::string> registered_plugin_ep_names_;
 };
 
 }  // namespace perftest

onnxruntime/test/perftest/test_configuration.h

@@ -74,6 +74,7 @@ struct PerformanceTestConfig {
   ModelInfo model_info;
   MachineConfig machine_config;
   RunConfig run_config;
+  std::basic_string<ORTCHAR_T> plugin_ep_names_and_libs;
 };
 
 }  // namespace perftest