From 2ef72d9c66400dbede5bb6def233d58832333365 Mon Sep 17 00:00:00 2001 From: n1harika Date: Tue, 23 Jun 2026 20:56:24 -0700 Subject: [PATCH 1/3] [CVS-187745] Add multi-shape profiling support to perftest tool (--data_shape) (#1132) Usage: .\onnxruntime_perf_test.exe -v -e openvino -m times -r 10 -I --data_shape "data:[1,3,60,40][1,3,60,60]" -i "device_type|NPU" "PSO2_ctx.onnx" or, using testdata- .\onnxruntime_perf_test.exe -v -e openvino -m times -r 10 --data_shape "data:[1,3,60,40][1,3,60,60]" -i "device_type|NPU" "PSO2_ctx.onnx" Addresses the feature request: microsoft#28628 --- .../test/perftest/command_args_parser.cc | 19 ++ onnxruntime/test/perftest/ort_test_session.cc | 105 +++++++++- onnxruntime/test/perftest/ort_test_session.h | 10 + .../test/perftest/performance_runner.cc | 182 +++++++++++++++++- .../test/perftest/performance_runner.h | 12 ++ onnxruntime/test/perftest/strings_helper.cc | 151 +++++++++++++++ onnxruntime/test/perftest/strings_helper.h | 4 + .../test/perftest/test_configuration.h | 2 + onnxruntime/test/perftest/test_session.h | 1 + .../test/testdata/dynamic_shape_add.onnx | Bin 0 -> 162 bytes .../test/testdata/dynamic_shape_add.py | 44 +++++ 11 files changed, 520 insertions(+), 10 deletions(-) create mode 100644 onnxruntime/test/testdata/dynamic_shape_add.onnx create mode 100644 onnxruntime/test/testdata/dynamic_shape_add.py diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 3f878ac796ee3..c8698140a8731 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -214,6 +214,15 @@ ABSL_FLAG(bool, compile_ep_context, DefaultPerformanceTestConfig().run_config.co ABSL_FLAG(std::string, compile_model_path, "model_ctx.onnx", "The compiled model path for saving EP context model. Overwrites if already exists"); ABSL_FLAG(bool, compile_binary_embed, DefaultPerformanceTestConfig().run_config.compile_binary_embed, "Embed binary blob within EP context node"); ABSL_FLAG(bool, compile_only, DefaultPerformanceTestConfig().run_config.compile_only, "Only compile EP context model without running it"); +ABSL_FLAG(std::string, data_shape, "", + "Specifies input shapes for multi-shape profiling within a single session.\n" + "The model is compiled once and run with each shape group in round-robin order.\n" + "[Usage]: --data_shape \"input_name:[d0,d1,...][d0,d1,...] ...\"\n" + "[Example]: --data_shape \"input:[1,3,224,224][1,3,448,448][1,3,112,112]\"\n" + "With -I: generates random input of the specified shapes.\n" + "Without -I: selects test data folders whose tensor shapes match.\n" + "All inputs must have the same number of shape groups.\n" + "All dimension values must be positive integers."); ABSL_FLAG(bool, h, false, "Print program usage."); namespace onnxruntime { @@ -481,6 +490,16 @@ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int a // -I test_config.run_config.generate_model_input_binding = absl::GetFlag(FLAGS_I); + // --data_shape + { + const auto& data_shape_str = absl::GetFlag(FLAGS_data_shape); + if (!data_shape_str.empty()) { + if (!ParseDataShapeGroups(data_shape_str, test_config.run_config.data_shape_groups)) { + return false; + } + } + } + // -d if (absl::GetFlag(FLAGS_d) < 0) return false; test_config.run_config.cudnn_conv_algo = absl::GetFlag(FLAGS_d); diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 679be75fa586e..4e2dd1b59feed 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -42,9 +42,14 @@ namespace onnxruntime { namespace perftest { RunTiming OnnxRuntimeTestSession::Run() { - // Randomly pick one OrtValueArray from test_inputs_. (NOT ThreadSafe) - const std::uniform_int_distribution::param_type p(0, static_cast(test_inputs_.size() - 1)); - const size_t id = static_cast(dist_(rand_engine_, p)); + // Select input set: round-robin for multi-shape mode, random otherwise. + size_t id; + if (use_round_robin_ && test_inputs_.size() > 1) { + id = shape_group_counter_.fetch_add(1, std::memory_order_relaxed) % test_inputs_.size(); + } else { + const std::uniform_int_distribution::param_type p(0, static_cast(test_inputs_.size() - 1)); + id = static_cast(dist_(rand_engine_, p)); + } auto& input = test_inputs_.at(id); auto start = std::chrono::high_resolution_clock::now(); @@ -88,6 +93,7 @@ RunTiming OnnxRuntimeTestSession::Run() { timing.submit_timing = std::chrono::high_resolution_clock::now() - start; timing.total_timing = timing.submit_timing; } + timing.shape_group_index = id; return timing; } @@ -1131,6 +1137,99 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { } return true; } + +bool OnnxRuntimeTestSession::PopulateMultiShapeInputTestData( + int32_t seed, + const std::map>>& data_shape_groups) { + // Validate that all input names in data_shape_groups exist in the model + for (const auto& [name, groups] : data_shape_groups) { + bool found = false; + for (int i = 0; i < input_length_; i++) { + if (input_names_str_[i] == name) { + found = true; + break; + } + } + if (!found) { + std::cerr << "Error: --data_shape specifies unknown input '" << name << "'." << std::endl; + return false; + } + } + + const size_t num_groups = data_shape_groups.begin()->second.size(); + Ort::AllocatorWithDefaultOptions default_allocator; + + for (size_t g = 0; g < num_groups; g++) { + for (size_t i = 0; i < static_cast(input_length_); i++) { + Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); + if (type_info.GetONNXType() != ONNX_TYPE_TENSOR) { + continue; + } + + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + std::vector input_node_dim; + + // Use user-specified shape if available, otherwise fall back to model metadata + auto it = data_shape_groups.find(input_names_str_[i]); + if (it != data_shape_groups.end()) { + input_node_dim = it->second[g]; + const auto model_shape = tensor_info.GetShape(); + if (!model_shape.empty() && input_node_dim.size() != model_shape.size()) { + std::cerr << "Error: --data_shape rank mismatch for input '" << input_names_str_[i] + << "': expected " << model_shape.size() << " dims but got " << input_node_dim.size() << "." << std::endl; + return false; + } + } else { + input_node_dim = tensor_info.GetShape(); + auto transform_fcn = [](int64_t input) { return (input == -1) ? -input : input; }; + std::transform(input_node_dim.begin(), input_node_dim.end(), input_node_dim.begin(), transform_fcn); + } + + if (device_memory_name_ != CUDA) { + Ort::Value input_tensor = Ort::Value::CreateTensor(allocator_, (const int64_t*)input_node_dim.data(), + input_node_dim.size(), tensor_info.GetElementType()); + InitializeTensorWithSeed(seed, input_tensor); + PreLoadTestData(g, i, std::move(input_tensor)); + } +#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_NV) + else { + Ort::Value default_tensor = Ort::Value::CreateTensor(default_allocator, (const int64_t*)input_node_dim.data(), + input_node_dim.size(), tensor_info.GetElementType()); + InitializeTensorWithSeed(seed, default_tensor); + + const void* default_ptr = default_tensor.GetTensorRawData(); + size_t total_bytes = default_tensor.GetTensorSizeInBytes(); + + Ort::Value cuda_tensor = Ort::Value::CreateTensor(allocator_, input_node_dim.data(), + input_node_dim.size(), tensor_info.GetElementType()); + void* cuda_ptr = cuda_tensor.GetTensorMutableData(); + + cudaError_t cuda_err = cudaMemcpy(cuda_ptr, default_ptr, total_bytes, cudaMemcpyHostToDevice); + if (cuda_err != cudaSuccess) { + ORT_THROW("Failed to copy tensor data from CPU to CUDA device. CUDA Error: ", cudaGetErrorString(cuda_err)); + } + PreLoadTestData(g, i, std::move(cuda_tensor)); + } +#endif + } + } + use_round_robin_ = true; + return true; +} + +std::vector OnnxRuntimeTestSession::GetLoadedInputShape(size_t test_data_id, size_t input_id) const { + return test_inputs_.at(test_data_id).at(input_id).GetTensorTypeAndShapeInfo().GetShape(); +} + +void OnnxRuntimeTestSession::SelectTestDataSets(const std::vector& selected_ids) { + std::vector> filtered; + filtered.reserve(selected_ids.size()); + for (size_t id : selected_ids) { + filtered.push_back(std::move(test_inputs_.at(id))); + } + test_inputs_ = std::move(filtered); +} + OnnxRuntimeTestSession::~OnnxRuntimeTestSession() { #ifdef USE_CUDA if (device_memory_name_ == CUDA && stream_ != nullptr) { diff --git a/onnxruntime/test/perftest/ort_test_session.h b/onnxruntime/test/perftest/ort_test_session.h index 0a2c683428bf2..438b67f5a4117 100644 --- a/onnxruntime/test/perftest/ort_test_session.h +++ b/onnxruntime/test/perftest/ort_test_session.h @@ -2,6 +2,7 @@ // Licensed under the MIT License. #pragma once +#include #include #include #include "test_configuration.h" @@ -31,6 +32,13 @@ class OnnxRuntimeTestSession : public TestSession { } bool PopulateGeneratedInputTestData(int32_t seed); + bool PopulateMultiShapeInputTestData( + int32_t seed, + const std::map>>& data_shape_groups); + + std::vector GetLoadedInputShape(size_t test_data_id, size_t input_id) const; + void SelectTestDataSets(const std::vector& selected_ids); + void SetUseRoundRobin(bool v) { use_round_robin_ = v; } ~OnnxRuntimeTestSession(); @@ -60,6 +68,8 @@ class OnnxRuntimeTestSession : public TestSession { std::string device_memory_name_; // Device memory type name to use from the list in allocator.h const std::unordered_map& run_config_entries_; bool has_dynamic_output_shapes_ = false; + std::atomic shape_group_counter_{0}; + bool use_round_robin_{false}; #if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_NV) cudaStream_t stream_; // Device stream if required by IO bindings #endif diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index a14e6437c0e01..e252819d95b83 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -109,6 +109,38 @@ void PerformanceResult::DumpToFile(const std::basic_string& path, boo output_stats(std::cout); } + + // Per-shape statistics (when --data_shape is used) + if (!per_shape_time_costs_total.empty() && f_include_statistics) { + auto output_per_shape = [&](std::ostream& ostream) { + for (size_t g = 0; g < per_shape_time_costs_total.size(); g++) { + const auto& shape_costs = per_shape_time_costs_total[g]; + if (shape_costs.empty()) continue; + + std::vector sorted = shape_costs; + std::sort(sorted.begin(), sorted.end()); + size_t count = sorted.size(); + size_t s50 = static_cast(count * 0.5); + size_t s90 = static_cast(count * 0.9); + size_t s95 = static_cast(count * 0.95); + size_t s99 = static_cast(count * 0.99); + + ostream << "\nShape group " << (g + 1) << " (" << count << " iterations):\n"; + ostream << " Min Latency: " << sorted[0] << " s\n"; + ostream << " Max Latency: " << sorted[count - 1] << " s\n"; + ostream << " P50 Latency: " << sorted[s50] << " s\n"; + ostream << " P90 Latency: " << sorted[s90] << " s\n"; + ostream << " P95 Latency: " << sorted[s95] << " s\n"; + ostream << " P99 Latency: " << sorted[s99] << " s" << std::endl; + } + }; + + if (have_file) { + output_per_shape(outfile); + } + + output_per_shape(std::cout); + } } void PerformanceRunner::LogSessionCreationTime() { @@ -116,14 +148,71 @@ void PerformanceRunner::LogSessionCreationTime() { std::cout << "\nSession creation time cost: " << session_create_duration.count() << " s\n"; } +void PerformanceRunner::PrintPerShapeStats() const { + const auto& shape_groups = performance_test_config_.run_config.data_shape_groups; + const auto& per_shape_costs = performance_result_.per_shape_time_costs_total; + size_t num_groups = per_shape_costs.size(); + + std::cout << "\nLatency per shape group:" << std::endl; + + for (size_t g = 0; g < num_groups; g++) { + // Build label: "input_name : [d0,d1,...]" + std::string label; + for (const auto& [input_name, groups] : shape_groups) { + if (!label.empty()) label += ", "; + label += input_name + " : ["; + const auto& dims = groups[g]; + for (size_t d = 0; d < dims.size(); d++) { + if (d > 0) label += ","; + label += std::to_string(dims[d]); + } + label += "]"; + } + + std::cout << " " << (g + 1) << ". " << label << std::endl; + + const auto& time_costs = per_shape_costs[g]; + if (time_costs.empty()) { + std::cout << " (no data)" << std::endl; + continue; + } + + std::vector sorted_time = time_costs; + std::sort(sorted_time.begin(), sorted_time.end()); + size_t total = sorted_time.size(); + size_t n50 = static_cast(total * 0.5); + size_t n90 = static_cast(total * 0.9); + size_t n95 = static_cast(total * 0.95); + size_t n99 = static_cast(total * 0.99); + + double avg = std::accumulate(sorted_time.begin(), sorted_time.end(), 0.0) / static_cast(total); + + std::cout << " Iterations: " << total << "\n" + << " Average Latency: " << avg * 1000.0 << " ms\n" + << " Min Latency: " << sorted_time[0] * 1000.0 << " ms\n" + << " Max Latency: " << sorted_time[total - 1] * 1000.0 << " ms\n" + << " P50 Latency: " << sorted_time[n50] * 1000.0 << " ms\n" + << " P90 Latency: " << sorted_time[n90] * 1000.0 << " ms\n" + << " P95 Latency: " << sorted_time[n95] * 1000.0 << " ms\n" + << " P99 Latency: " << sorted_time[n99] * 1000.0 << " ms" + << std::endl; + } +} + Status PerformanceRunner::Run() { if (!Initialize()) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "failed to initialize."); } - // warm up + // warm up — run one iteration per shape group to ensure all shapes are warmed + size_t warmup_count = 1; + if (!performance_test_config_.run_config.data_shape_groups.empty()) { + warmup_count = performance_test_config_.run_config.data_shape_groups.begin()->second.size(); + } initial_inference_result_.start = std::chrono::high_resolution_clock::now(); - ORT_RETURN_IF_ERROR(RunOneIteration()); + for (size_t w = 0; w < warmup_count; w++) { + ORT_RETURN_IF_ERROR(RunOneIteration()); + } initial_inference_result_.end = std::chrono::high_resolution_clock::now(); // TODO: start profiling @@ -171,6 +260,10 @@ Status PerformanceRunner::Run() { << "Peak working set size: " << performance_result_.peak_workingset_size << " bytes" << std::endl; + if (!performance_result_.per_shape_time_costs_total.empty()) { + PrintPerShapeStats(); + } + return Status::OK(); } @@ -306,9 +399,20 @@ bool PerformanceRunner::Initialize() { test_case_ = CreateOnnxTestCase(narrow_model_name, std::move(test_model_info_), 0.0, 0.0); if (performance_test_config_.run_config.generate_model_input_binding) { - return static_cast( - session_.get()) - ->PopulateGeneratedInputTestData(performance_test_config_.run_config.random_seed_for_input_data); + auto* ort_session = static_cast(session_.get()); + if (!performance_test_config_.run_config.data_shape_groups.empty()) { + if (!ort_session->PopulateMultiShapeInputTestData( + performance_test_config_.run_config.random_seed_for_input_data, + performance_test_config_.run_config.data_shape_groups)) { + return false; + } + // Pre-size per-shape timing vectors + size_t num_groups = performance_test_config_.run_config.data_shape_groups.begin()->second.size(); + performance_result_.per_shape_time_costs_total.resize(num_groups); + return true; + } + return ort_session->PopulateGeneratedInputTestData( + performance_test_config_.run_config.random_seed_for_input_data); } // TODO: Place input tensor on cpu memory if dnnl provider type to avoid CopyTensor logic in CopyInputAcrossDevices @@ -317,11 +421,10 @@ bool PerformanceRunner::Initialize() { std::cout << "there is no test data for model " << test_case_->GetTestCaseName() << std::endl; return false; } + int input_count = test_model_info->GetInputCount(); for (size_t test_data_id = 0; test_data_id != test_data_count; ++test_data_id) { std::unordered_map feeds; test_case_->LoadTestData(test_data_id /* id */, b_, feeds, true); - // Discard the names in feeds - int input_count = test_model_info->GetInputCount(); for (int i = 0; i != input_count; ++i) { auto iter = feeds.find(test_model_info->GetInputName(i)); if (iter == feeds.end()) { @@ -333,6 +436,71 @@ bool PerformanceRunner::Initialize() { } } + // When --data_shape is specified without -I, select only the test data sets + // whose tensor shapes match the requested shape groups. + const auto& data_shape_groups = performance_test_config_.run_config.data_shape_groups; + if (!data_shape_groups.empty()) { + auto* ort_session = static_cast(session_.get()); + size_t num_groups = data_shape_groups.begin()->second.size(); + + // Build input name -> index map + std::unordered_map input_name_to_idx; + for (int i = 0; i < input_count; ++i) { + input_name_to_idx[test_model_info->GetInputName(i)] = static_cast(i); + } + + // Validate that all names in data_shape_groups exist in the model + for (const auto& [name, groups] : data_shape_groups) { + if (input_name_to_idx.find(name) == input_name_to_idx.end()) { + std::cerr << "Error: --data_shape specifies unknown input '" << name << "'." << std::endl; + return false; + } + } + + // For each shape group, find the matching test data set + std::vector selected_ids; + for (size_t g = 0; g < num_groups; ++g) { + bool found = false; + for (size_t test_data_id = 0; test_data_id < test_data_count; ++test_data_id) { + bool match = true; + for (const auto& [input_name, shape_list] : data_shape_groups) { + size_t input_idx = input_name_to_idx[input_name]; + auto loaded_shape = ort_session->GetLoadedInputShape(test_data_id, input_idx); + if (loaded_shape != shape_list[g]) { + match = false; + break; + } + } + if (match) { + if (std::find(selected_ids.begin(), selected_ids.end(), test_data_id) != selected_ids.end()) { + std::cerr << "Error: --data_shape shape group " << (g + 1) + << " matches a test data set already selected. " + << "Duplicate shape groups are not supported when selecting from test data." << std::endl; + return false; + } + selected_ids.push_back(test_data_id); + found = true; + break; + } + } + if (!found) { + std::cerr << "No test data found matching shape ["; + const auto& first_input = data_shape_groups.begin(); + const auto& dims = first_input->second[g]; + for (size_t d = 0; d < dims.size(); ++d) { + if (d > 0) std::cerr << ","; + std::cerr << dims[d]; + } + std::cerr << "] for input '" << first_input->first << "'." << std::endl; + return false; + } + } + + ort_session->SelectTestDataSets(selected_ids); + ort_session->SetUseRoundRobin(true); + performance_result_.per_shape_time_costs_total.resize(num_groups); + } + return true; } diff --git a/onnxruntime/test/perftest/performance_runner.h b/onnxruntime/test/perftest/performance_runner.h index f67ab9f5c2287..4e95066a26687 100644 --- a/onnxruntime/test/perftest/performance_runner.h +++ b/onnxruntime/test/perftest/performance_runner.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,8 @@ struct PerformanceResult { double total_time_cost{0}; std::vector time_costs_submission; std::vector time_costs_total; + // Per-shape-group timing (only populated when --data_shape is used) + std::vector> per_shape_time_costs_total; std::string model_name; void DumpToFile(const std::basic_string& path, bool f_include_statistics = false) const; @@ -59,6 +62,7 @@ class PerformanceRunner { private: bool Initialize(); + void PrintPerShapeStats() const; template Status RunOneIteration() { @@ -80,6 +84,14 @@ class PerformanceRunner { performance_result_.time_costs_total.emplace_back(duration_seconds.total_timing.count()); // if the test did not run async the device timing will be equal to the CPU timing performance_result_.total_time_cost += duration_seconds.total_timing.count(); + // Record per-shape timing when multi-shape mode is active + if (!performance_result_.per_shape_time_costs_total.empty()) { + size_t shape_idx = duration_seconds.shape_group_index; + if (shape_idx < performance_result_.per_shape_time_costs_total.size()) { + performance_result_.per_shape_time_costs_total[shape_idx].emplace_back( + duration_seconds.total_timing.count()); + } + } if (performance_test_config_.run_config.f_verbose) { std::cout << "iteration:" << performance_result_.time_costs_submission.size() << "," << "time_cost_submission:" << performance_result_.time_costs_submission.back() << "," diff --git a/onnxruntime/test/perftest/strings_helper.cc b/onnxruntime/test/perftest/strings_helper.cc index d9fd2a2a55c09..dd77c1c08ab59 100644 --- a/onnxruntime/test/perftest/strings_helper.cc +++ b/onnxruntime/test/perftest/strings_helper.cc @@ -154,5 +154,156 @@ void ParseEpDeviceFilterKeyValuePairs(const std::string& input, std::vector>>& data_shape_groups) { + // Parse format: "input_name:[d0,d1,...][d0,d1,...] input_name2:[d0,d1][d0,d1]" + // Split on whitespace that is outside brackets to allow spaces inside shape specs. + std::vector input_specs; + std::string current; + int bracket_depth = 0; + for (char c : input) { + if (c == '[') { + bracket_depth++; + current += c; + } else if (c == ']') { + bracket_depth--; + current += c; + } else if ((c == ' ' || c == '\t') && bracket_depth == 0) { + if (!current.empty()) { + input_specs.push_back(current); + current.clear(); + } + } else { + current += c; + } + } + if (!current.empty()) { + input_specs.push_back(current); + } + + if (bracket_depth != 0) { + std::cerr << "Error parsing --data_shape: mismatched brackets (unbalanced '[' and ']')." << std::endl; + return false; + } + + for (const auto& input_spec : input_specs) { + // Split on first ':' to get input_name and shapes string + size_t colon_pos = input_spec.find(':'); + if (colon_pos == std::string::npos || colon_pos == 0) { + std::cerr << "Error parsing --data_shape: expected 'name:[shape]...' format, got '" << input_spec << "'." << std::endl; + return false; + } + + std::string input_name = input_spec.substr(0, colon_pos); + std::string shapes_str = input_spec.substr(colon_pos + 1); + + if (shapes_str.empty()) { + std::cerr << "Error parsing --data_shape: no shape groups provided for input '" << input_name << "'." << std::endl; + return false; + } + + std::vector> shape_groups; + + // Extract bracket-delimited shape groups: [d0,d1,...][d0,d1,...] + size_t pos = 0; + while (pos < shapes_str.size()) { + while (pos < shapes_str.size() && (shapes_str[pos] == ' ' || shapes_str[pos] == '\t')) { + ++pos; + } + if (pos >= shapes_str.size()) break; + if (shapes_str[pos] != '[') { + std::cerr << "Error parsing --data_shape: expected '[' at position " << pos + << " in shapes for input '" << input_name << "'." << std::endl; + return false; + } + + size_t close_bracket = shapes_str.find(']', pos); + if (close_bracket == std::string::npos) { + std::cerr << "Error parsing --data_shape: unmatched bracket for input '" << input_name << "'." << std::endl; + return false; + } + + std::string dims_str = shapes_str.substr(pos + 1, close_bracket - pos - 1); + if (dims_str.empty()) { + std::cerr << "Error parsing --data_shape: empty shape group for input '" << input_name << "'." << std::endl; + return false; + } + + // Parse comma-separated dimensions + std::vector dims; + std::stringstream dims_ss(dims_str); + std::string dim_token; + while (std::getline(dims_ss, dim_token, ',')) { + // Trim whitespace from dimension token + size_t start = dim_token.find_first_not_of(" \t"); + size_t end = dim_token.find_last_not_of(" \t"); + if (start == std::string::npos) { + std::cerr << "Error parsing --data_shape: empty dimension token for input '" << input_name << "'." << std::endl; + return false; + } + dim_token = dim_token.substr(start, end - start + 1); + + ORT_TRY { + size_t chars_parsed = 0; + int64_t dim_val = std::stoll(dim_token, &chars_parsed); + if (chars_parsed != dim_token.length()) { + std::cerr << "Error parsing --data_shape: invalid characters in dimension '" + << dim_token << "' for input '" << input_name << "'." << std::endl; + return false; + } + if (dim_val <= 0) { + std::cerr << "Error parsing --data_shape: dimensions must be positive integers, got '" + << dim_token << "' for input '" << input_name << "'." << std::endl; + return false; + } + dims.push_back(dim_val); + } + ORT_CATCH(const std::exception& ex) { + ORT_HANDLE_EXCEPTION([&]() { + std::cerr << "Error parsing --data_shape: invalid dimension value '" << dim_token + << "' for input '" << input_name << "': " << ex.what() << std::endl; + }); + return false; + } + } + + shape_groups.push_back(std::move(dims)); + pos = close_bracket + 1; + } + + if (shape_groups.empty()) { + std::cerr << "Error parsing --data_shape: no shape groups found for input '" << input_name << "'." << std::endl; + return false; + } + + if (data_shape_groups.count(input_name) > 0) { + std::cerr << "Error parsing --data_shape: duplicate input name '" << input_name << "'." << std::endl; + return false; + } + + data_shape_groups[input_name] = std::move(shape_groups); + } + + if (data_shape_groups.empty()) { + std::cerr << "Error parsing --data_shape: no input specifications found." << std::endl; + return false; + } + + // Validate all inputs have the same number of shape groups + if (data_shape_groups.size() > 1) { + size_t expected_count = data_shape_groups.begin()->second.size(); + for (const auto& [name, groups] : data_shape_groups) { + if (groups.size() != expected_count) { + std::cerr << "Error parsing --data_shape: all inputs must have the same number of shape groups. " + << "Input '" << data_shape_groups.begin()->first << "' has " << expected_count + << " groups but input '" << name << "' has " << groups.size() << "." << std::endl; + return false; + } + } + } + + return true; +} } // namespace perftest } // namespace onnxruntime diff --git a/onnxruntime/test/perftest/strings_helper.h b/onnxruntime/test/perftest/strings_helper.h index d6c6f6112ab6c..cfd56a9dab40f 100644 --- a/onnxruntime/test/perftest/strings_helper.h +++ b/onnxruntime/test/perftest/strings_helper.h @@ -2,6 +2,7 @@ // Copyright (c) 2023 NVIDIA Corporation. // SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates // Licensed under the MIT License. +#include #include #include #include @@ -26,5 +27,8 @@ void ParseEpOptions(const std::string& input, std::vector& result); void ParseEpDeviceFilterKeyValuePairs(const std::string& input, std::vector>& result); + +bool ParseDataShapeGroups(const std::string& input, + std::map>>& data_shape_groups); } // namespace perftest } // namespace onnxruntime diff --git a/onnxruntime/test/perftest/test_configuration.h b/onnxruntime/test/perftest/test_configuration.h index dea0b196f6bcb..b4e525902954a 100644 --- a/onnxruntime/test/perftest/test_configuration.h +++ b/onnxruntime/test/perftest/test_configuration.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "core/graph/constants.h" #include "core/framework/session_options.h" @@ -67,6 +68,7 @@ struct RunConfig { std::unordered_map run_config_entries; std::map free_dim_name_overrides; std::map free_dim_denotation_overrides; + std::map>> data_shape_groups; std::string intra_op_thread_affinities; bool disable_spinning = false; bool disable_spinning_between_run = false; diff --git a/onnxruntime/test/perftest/test_session.h b/onnxruntime/test/perftest/test_session.h index 71cb6062a93f4..f1f6b0db5bccf 100644 --- a/onnxruntime/test/perftest/test_session.h +++ b/onnxruntime/test/perftest/test_session.h @@ -12,6 +12,7 @@ namespace perftest { struct RunTiming { std::chrono::duration submit_timing = std::chrono::seconds(0); std::chrono::duration total_timing = std::chrono::seconds(0); + size_t shape_group_index = 0; // which shape group (test_inputs_ index) was used }; class TestSession { diff --git a/onnxruntime/test/testdata/dynamic_shape_add.onnx b/onnxruntime/test/testdata/dynamic_shape_add.onnx new file mode 100644 index 0000000000000000000000000000000000000000..31e4cdbf9200f21b1999e1a4560a449852840ce0 GIT binary patch literal 162 zcmd Date: Mon, 6 Jul 2026 21:43:31 -0700 Subject: [PATCH 2/3] Address review feedback, remove manual test files, add warnings for clarity --- onnxruntime/test/perftest/ort_test_session.cc | 130 ++++++++---------- onnxruntime/test/perftest/ort_test_session.h | 6 +- .../test/perftest/performance_runner.cc | 30 +--- .../test/perftest/performance_runner.h | 2 +- onnxruntime/test/perftest/strings_helper.cc | 48 ++++--- onnxruntime/test/perftest/strings_helper.h | 3 + onnxruntime/test/perftest/test_session.h | 2 +- .../test/testdata/dynamic_shape_add.onnx | Bin 162 -> 0 bytes .../test/testdata/dynamic_shape_add.py | 44 ------ 9 files changed, 105 insertions(+), 160 deletions(-) delete mode 100644 onnxruntime/test/testdata/dynamic_shape_add.onnx delete mode 100644 onnxruntime/test/testdata/dynamic_shape_add.py diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 4e2dd1b59feed..e534be056e912 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -93,7 +93,7 @@ RunTiming OnnxRuntimeTestSession::Run() { timing.submit_timing = std::chrono::high_resolution_clock::now() - start; timing.total_timing = timing.submit_timing; } - timing.shape_group_index = id; + timing.test_input_index = id; return timing; } @@ -1089,56 +1089,58 @@ static void InitializeTensorWithSeed(int32_t seed, Ort::Value& tensor) { #undef CASE_FOR_TYPE } +void OnnxRuntimeTestSession::CreateAndStoreGeneratedInput(size_t test_data_id, size_t input_idx, + const std::vector& dims, + ONNXTensorElementDataType element_type, int32_t seed) { + if (device_memory_name_ != CUDA) { + Ort::Value input_tensor = Ort::Value::CreateTensor(allocator_, (const int64_t*)dims.data(), + dims.size(), element_type); + InitializeTensorWithSeed(seed, input_tensor); + PreLoadTestData(test_data_id, input_idx, std::move(input_tensor)); + } +#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_NV) + else { + Ort::AllocatorWithDefaultOptions default_allocator; + Ort::Value default_tensor = Ort::Value::CreateTensor(default_allocator, (const int64_t*)dims.data(), + dims.size(), element_type); + InitializeTensorWithSeed(seed, default_tensor); + + const void* default_ptr = default_tensor.GetTensorRawData(); + size_t total_bytes = default_tensor.GetTensorSizeInBytes(); + + Ort::Value cuda_tensor = Ort::Value::CreateTensor(allocator_, dims.data(), + dims.size(), element_type); + void* cuda_ptr = cuda_tensor.GetTensorMutableData(); + + cudaError_t cuda_err = cudaMemcpy(cuda_ptr, default_ptr, total_bytes, cudaMemcpyHostToDevice); + if (cuda_err != cudaSuccess) { + ORT_THROW("Failed to copy tensor data from CPU to CUDA device. CUDA Error: ", cudaGetErrorString(cuda_err)); + } + PreLoadTestData(test_data_id, input_idx, std::move(cuda_tensor)); + } +#endif +} + bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { - Ort::AllocatorWithDefaultOptions default_allocator; - // iterate over all input nodes for (size_t i = 0; i < static_cast(input_length_); i++) { Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); - if (type_info.GetONNXType() == ONNX_TYPE_TENSOR) { - auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - std::vector input_node_dim = tensor_info.GetShape(); - - // free dimensions are treated as 1 if not overridden - auto transform_fcn = [](int64_t input) { return (input == -1) ? -input : input; }; - std::transform(input_node_dim.begin(), input_node_dim.end(), input_node_dim.begin(), transform_fcn); - - if (device_memory_name_ != CUDA) { - Ort::Value input_tensor = Ort::Value::CreateTensor(allocator_, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); - InitializeTensorWithSeed(seed, input_tensor); - PreLoadTestData(0, i, std::move(input_tensor)); - } -// Create tensor on CPU, initialize and copy to CUDA tensor -#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_NV) - else { - Ort::Value default_tensor = Ort::Value::CreateTensor(default_allocator, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); - InitializeTensorWithSeed(seed, default_tensor); - - // Get pointer to CPU tensor data - const void* default_ptr = default_tensor.GetTensorRawData(); - - size_t total_bytes = default_tensor.GetTensorSizeInBytes(); + if (type_info.GetONNXType() != ONNX_TYPE_TENSOR) { + continue; + } - Ort::Value cuda_tensor = Ort::Value::CreateTensor(allocator_, input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + std::vector input_node_dim = tensor_info.GetShape(); - void* cuda_ptr = cuda_tensor.GetTensorMutableData(); + // free dimensions are treated as 1 if not overridden + auto transform_fcn = [](int64_t input) { return (input == -1) ? -input : input; }; + std::transform(input_node_dim.begin(), input_node_dim.end(), input_node_dim.begin(), transform_fcn); - // Copy the initialized data from CPU to GPU - cudaError_t cuda_err = cudaMemcpy(cuda_ptr, default_ptr, total_bytes, cudaMemcpyHostToDevice); - if (cuda_err != cudaSuccess) { - ORT_THROW("Failed to copy tensor data from CPU to CUDA device. CUDA Error: ", cudaGetErrorString(cuda_err)); - } - PreLoadTestData(0, i, std::move(cuda_tensor)); - } -#endif - } + CreateAndStoreGeneratedInput(0, i, input_node_dim, tensor_info.GetElementType(), seed); } return true; } -bool OnnxRuntimeTestSession::PopulateMultiShapeInputTestData( +bool OnnxRuntimeTestSession::PopulateGeneratedMultiShapeInputTestData( int32_t seed, const std::map>>& data_shape_groups) { // Validate that all input names in data_shape_groups exist in the model @@ -1157,7 +1159,6 @@ bool OnnxRuntimeTestSession::PopulateMultiShapeInputTestData( } const size_t num_groups = data_shape_groups.begin()->second.size(); - Ort::AllocatorWithDefaultOptions default_allocator; for (size_t g = 0; g < num_groups; g++) { for (size_t i = 0; i < static_cast(input_length_); i++) { @@ -1181,36 +1182,22 @@ bool OnnxRuntimeTestSession::PopulateMultiShapeInputTestData( } } else { input_node_dim = tensor_info.GetShape(); + bool has_dynamic_dim = std::any_of(input_node_dim.begin(), input_node_dim.end(), + [](int64_t d) { return d == -1; }); auto transform_fcn = [](int64_t input) { return (input == -1) ? -input : input; }; std::transform(input_node_dim.begin(), input_node_dim.end(), input_node_dim.begin(), transform_fcn); - } - - if (device_memory_name_ != CUDA) { - Ort::Value input_tensor = Ort::Value::CreateTensor(allocator_, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); - InitializeTensorWithSeed(seed, input_tensor); - PreLoadTestData(g, i, std::move(input_tensor)); - } -#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_NV) - else { - Ort::Value default_tensor = Ort::Value::CreateTensor(default_allocator, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); - InitializeTensorWithSeed(seed, default_tensor); - - const void* default_ptr = default_tensor.GetTensorRawData(); - size_t total_bytes = default_tensor.GetTensorSizeInBytes(); - - Ort::Value cuda_tensor = Ort::Value::CreateTensor(allocator_, input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); - void* cuda_ptr = cuda_tensor.GetTensorMutableData(); - - cudaError_t cuda_err = cudaMemcpy(cuda_ptr, default_ptr, total_bytes, cudaMemcpyHostToDevice); - if (cuda_err != cudaSuccess) { - ORT_THROW("Failed to copy tensor data from CPU to CUDA device. CUDA Error: ", cudaGetErrorString(cuda_err)); + if (g == 0 && has_dynamic_dim) { + std::cerr << "Warning: input '" << input_names_str_[i] + << "' not specified in --data_shape; using inferred shape ["; + for (size_t d = 0; d < input_node_dim.size(); d++) { + if (d > 0) std::cerr << ","; + std::cerr << input_node_dim[d]; + } + std::cerr << "] (dynamic dims defaulted to 1)." << std::endl; } - PreLoadTestData(g, i, std::move(cuda_tensor)); } -#endif + + CreateAndStoreGeneratedInput(g, i, input_node_dim, tensor_info.GetElementType(), seed); } } use_round_robin_ = true; @@ -1218,7 +1205,12 @@ bool OnnxRuntimeTestSession::PopulateMultiShapeInputTestData( } std::vector OnnxRuntimeTestSession::GetLoadedInputShape(size_t test_data_id, size_t input_id) const { - return test_inputs_.at(test_data_id).at(input_id).GetTensorTypeAndShapeInfo().GetShape(); + const auto& v = test_inputs_.at(test_data_id).at(input_id); + if (!v.IsTensor()) { + ORT_THROW("--data_shape only supports tensor inputs; input_id=", input_id, " in test_data_id=", test_data_id, + " is not a tensor."); + } + return v.GetTensorTypeAndShapeInfo().GetShape(); } void OnnxRuntimeTestSession::SelectTestDataSets(const std::vector& selected_ids) { diff --git a/onnxruntime/test/perftest/ort_test_session.h b/onnxruntime/test/perftest/ort_test_session.h index 438b67f5a4117..0512d32a2471e 100644 --- a/onnxruntime/test/perftest/ort_test_session.h +++ b/onnxruntime/test/perftest/ort_test_session.h @@ -32,7 +32,7 @@ class OnnxRuntimeTestSession : public TestSession { } bool PopulateGeneratedInputTestData(int32_t seed); - bool PopulateMultiShapeInputTestData( + bool PopulateGeneratedMultiShapeInputTestData( int32_t seed, const std::map>>& data_shape_groups); @@ -47,6 +47,10 @@ class OnnxRuntimeTestSession : public TestSession { ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OnnxRuntimeTestSession); private: + void CreateAndStoreGeneratedInput(size_t test_data_id, size_t input_idx, + const std::vector& dims, + ONNXTensorElementDataType element_type, int32_t seed); + Ort::Session session_{nullptr}; std::mt19937 rand_engine_; std::uniform_int_distribution dist_; diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index e252819d95b83..025f674c40cb7 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -10,6 +10,7 @@ #include #include "TestCase.h" +#include "strings_helper.h" #include "utils.h" #include "ort_test_session.h" using onnxruntime::Status; @@ -138,8 +139,6 @@ void PerformanceResult::DumpToFile(const std::basic_string& path, boo if (have_file) { output_per_shape(outfile); } - - output_per_shape(std::cout); } } @@ -156,20 +155,7 @@ void PerformanceRunner::PrintPerShapeStats() const { std::cout << "\nLatency per shape group:" << std::endl; for (size_t g = 0; g < num_groups; g++) { - // Build label: "input_name : [d0,d1,...]" - std::string label; - for (const auto& [input_name, groups] : shape_groups) { - if (!label.empty()) label += ", "; - label += input_name + " : ["; - const auto& dims = groups[g]; - for (size_t d = 0; d < dims.size(); d++) { - if (d > 0) label += ","; - label += std::to_string(dims[d]); - } - label += "]"; - } - - std::cout << " " << (g + 1) << ". " << label << std::endl; + std::cout << " " << (g + 1) << ". " << FormatShapeGroup(shape_groups, g) << std::endl; const auto& time_costs = per_shape_costs[g]; if (time_costs.empty()) { @@ -401,7 +387,7 @@ bool PerformanceRunner::Initialize() { if (performance_test_config_.run_config.generate_model_input_binding) { auto* ort_session = static_cast(session_.get()); if (!performance_test_config_.run_config.data_shape_groups.empty()) { - if (!ort_session->PopulateMultiShapeInputTestData( + if (!ort_session->PopulateGeneratedMultiShapeInputTestData( performance_test_config_.run_config.random_seed_for_input_data, performance_test_config_.run_config.data_shape_groups)) { return false; @@ -484,14 +470,8 @@ bool PerformanceRunner::Initialize() { } } if (!found) { - std::cerr << "No test data found matching shape ["; - const auto& first_input = data_shape_groups.begin(); - const auto& dims = first_input->second[g]; - for (size_t d = 0; d < dims.size(); ++d) { - if (d > 0) std::cerr << ","; - std::cerr << dims[d]; - } - std::cerr << "] for input '" << first_input->first << "'." << std::endl; + std::cerr << "No test data found matching shape group " << (g + 1) << ": " + << FormatShapeGroup(data_shape_groups, g) << "." << std::endl; return false; } } diff --git a/onnxruntime/test/perftest/performance_runner.h b/onnxruntime/test/perftest/performance_runner.h index 4e95066a26687..d1f99044f42c1 100644 --- a/onnxruntime/test/perftest/performance_runner.h +++ b/onnxruntime/test/perftest/performance_runner.h @@ -86,7 +86,7 @@ class PerformanceRunner { performance_result_.total_time_cost += duration_seconds.total_timing.count(); // Record per-shape timing when multi-shape mode is active if (!performance_result_.per_shape_time_costs_total.empty()) { - size_t shape_idx = duration_seconds.shape_group_index; + size_t shape_idx = duration_seconds.test_input_index; if (shape_idx < performance_result_.per_shape_time_costs_total.size()) { performance_result_.per_shape_time_costs_total[shape_idx].emplace_back( duration_seconds.total_timing.count()); diff --git a/onnxruntime/test/perftest/strings_helper.cc b/onnxruntime/test/perftest/strings_helper.cc index dd77c1c08ab59..5e9fd8cb649b8 100644 --- a/onnxruntime/test/perftest/strings_helper.cc +++ b/onnxruntime/test/perftest/strings_helper.cc @@ -3,6 +3,7 @@ // SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates // Licensed under the MIT License. +#include #include #include @@ -244,28 +245,21 @@ bool ParseDataShapeGroups(const std::string& input, } dim_token = dim_token.substr(start, end - start + 1); - ORT_TRY { - size_t chars_parsed = 0; - int64_t dim_val = std::stoll(dim_token, &chars_parsed); - if (chars_parsed != dim_token.length()) { - std::cerr << "Error parsing --data_shape: invalid characters in dimension '" - << dim_token << "' for input '" << input_name << "'." << std::endl; - return false; - } - if (dim_val <= 0) { - std::cerr << "Error parsing --data_shape: dimensions must be positive integers, got '" - << dim_token << "' for input '" << input_name << "'." << std::endl; - return false; - } - dims.push_back(dim_val); + int64_t dim_val = 0; + const char* begin = dim_token.data(); + const char* end_ptr = begin + dim_token.size(); + auto [ptr, ec] = std::from_chars(begin, end_ptr, dim_val); + if (ec != std::errc{} || ptr != end_ptr) { + std::cerr << "Error parsing --data_shape: invalid dimension value '" + << dim_token << "' for input '" << input_name << "'." << std::endl; + return false; } - ORT_CATCH(const std::exception& ex) { - ORT_HANDLE_EXCEPTION([&]() { - std::cerr << "Error parsing --data_shape: invalid dimension value '" << dim_token - << "' for input '" << input_name << "': " << ex.what() << std::endl; - }); + if (dim_val <= 0) { + std::cerr << "Error parsing --data_shape: dimensions must be positive integers, got '" + << dim_token << "' for input '" << input_name << "'." << std::endl; return false; } + dims.push_back(dim_val); } shape_groups.push_back(std::move(dims)); @@ -305,5 +299,21 @@ bool ParseDataShapeGroups(const std::string& input, return true; } + +std::string FormatShapeGroup(const std::map>>& groups, size_t g) { + std::string result; + for (const auto& [name, shapes] : groups) { + if (!result.empty()) result += ", "; + result += name + ":["; + const auto& dims = shapes[g]; + for (size_t d = 0; d < dims.size(); d++) { + if (d > 0) result += ","; + result += std::to_string(dims[d]); + } + result += "]"; + } + return result; +} + } // namespace perftest } // namespace onnxruntime diff --git a/onnxruntime/test/perftest/strings_helper.h b/onnxruntime/test/perftest/strings_helper.h index cfd56a9dab40f..07dc3a9c96518 100644 --- a/onnxruntime/test/perftest/strings_helper.h +++ b/onnxruntime/test/perftest/strings_helper.h @@ -30,5 +30,8 @@ void ParseEpDeviceFilterKeyValuePairs(const std::string& input, std::vector>>& data_shape_groups); + +std::string FormatShapeGroup(const std::map>>& groups, size_t g); + } // namespace perftest } // namespace onnxruntime diff --git a/onnxruntime/test/perftest/test_session.h b/onnxruntime/test/perftest/test_session.h index f1f6b0db5bccf..d2b7cee97efb8 100644 --- a/onnxruntime/test/perftest/test_session.h +++ b/onnxruntime/test/perftest/test_session.h @@ -12,7 +12,7 @@ namespace perftest { struct RunTiming { std::chrono::duration submit_timing = std::chrono::seconds(0); std::chrono::duration total_timing = std::chrono::seconds(0); - size_t shape_group_index = 0; // which shape group (test_inputs_ index) was used + size_t test_input_index = 0; // which test_inputs_ entry was used }; class TestSession { diff --git a/onnxruntime/test/testdata/dynamic_shape_add.onnx b/onnxruntime/test/testdata/dynamic_shape_add.onnx deleted file mode 100644 index 31e4cdbf9200f21b1999e1a4560a449852840ce0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 162 zcmd Date: Wed, 8 Jul 2026 00:58:28 -0700 Subject: [PATCH 3/3] Address review feedback 2, adjust output formatting per shape group, add apt headers --- onnxruntime/test/perftest/ort_test_session.cc | 3 +- onnxruntime/test/perftest/ort_test_session.h | 2 +- .../test/perftest/performance_runner.cc | 75 ++++++------------- .../test/perftest/performance_runner.h | 7 +- onnxruntime/test/perftest/strings_helper.cc | 4 + onnxruntime/test/perftest/strings_helper.h | 6 +- 6 files changed, 39 insertions(+), 58 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index e534be056e912..ae9ed2a504320 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -45,8 +45,9 @@ RunTiming OnnxRuntimeTestSession::Run() { // Select input set: round-robin for multi-shape mode, random otherwise. size_t id; if (use_round_robin_ && test_inputs_.size() > 1) { - id = shape_group_counter_.fetch_add(1, std::memory_order_relaxed) % test_inputs_.size(); + id = round_robin_counter_.fetch_add(1, std::memory_order_relaxed) % test_inputs_.size(); } else { + // Random selection (not thread-safe). const std::uniform_int_distribution::param_type p(0, static_cast(test_inputs_.size() - 1)); id = static_cast(dist_(rand_engine_, p)); } diff --git a/onnxruntime/test/perftest/ort_test_session.h b/onnxruntime/test/perftest/ort_test_session.h index 0512d32a2471e..422e76ba0e72d 100644 --- a/onnxruntime/test/perftest/ort_test_session.h +++ b/onnxruntime/test/perftest/ort_test_session.h @@ -72,7 +72,7 @@ class OnnxRuntimeTestSession : public TestSession { std::string device_memory_name_; // Device memory type name to use from the list in allocator.h const std::unordered_map& run_config_entries_; bool has_dynamic_output_shapes_ = false; - std::atomic shape_group_counter_{0}; + std::atomic round_robin_counter_{0}; bool use_round_robin_{false}; #if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_NV) cudaStream_t stream_; // Device stream if required by IO bindings diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index 025f674c40cb7..91334ae00f81f 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -55,7 +55,8 @@ Eigen::ThreadPoolInterface* GetDefaultThreadPool(const onnxruntime::Env& env) { namespace onnxruntime { namespace perftest { -void PerformanceResult::DumpToFile(const std::basic_string& path, bool f_include_statistics) const { +void PerformanceResult::DumpToFile(const std::basic_string& path, bool f_include_statistics, + const std::map>>& shape_groups) const { bool have_file = !path.empty(); std::ofstream outfile; @@ -114,9 +115,17 @@ void PerformanceResult::DumpToFile(const std::basic_string& path, boo // Per-shape statistics (when --data_shape is used) if (!per_shape_time_costs_total.empty() && f_include_statistics) { auto output_per_shape = [&](std::ostream& ostream) { + ostream << "\nLatency per shape group:" << std::endl; for (size_t g = 0; g < per_shape_time_costs_total.size(); g++) { const auto& shape_costs = per_shape_time_costs_total[g]; - if (shape_costs.empty()) continue; + + std::string label = FormatShapeGroup(shape_groups, g); + ostream << " " << (g + 1) << ". " << (label.empty() ? "shape group" : label) << std::endl; + + if (shape_costs.empty()) { + ostream << " (no data)" << std::endl; + continue; + } std::vector sorted = shape_costs; std::sort(sorted.begin(), sorted.end()); @@ -125,20 +134,24 @@ void PerformanceResult::DumpToFile(const std::basic_string& path, boo size_t s90 = static_cast(count * 0.9); size_t s95 = static_cast(count * 0.95); size_t s99 = static_cast(count * 0.99); - - ostream << "\nShape group " << (g + 1) << " (" << count << " iterations):\n"; - ostream << " Min Latency: " << sorted[0] << " s\n"; - ostream << " Max Latency: " << sorted[count - 1] << " s\n"; - ostream << " P50 Latency: " << sorted[s50] << " s\n"; - ostream << " P90 Latency: " << sorted[s90] << " s\n"; - ostream << " P95 Latency: " << sorted[s95] << " s\n"; - ostream << " P99 Latency: " << sorted[s99] << " s" << std::endl; + double avg = std::accumulate(sorted.begin(), sorted.end(), 0.0) / static_cast(count); + + ostream << " Iterations: " << count << "\n" + << " Average Latency: " << avg * 1000.0 << " ms\n" + << " Min Latency: " << sorted[0] * 1000.0 << " ms\n" + << " Max Latency: " << sorted[count - 1] * 1000.0 << " ms\n" + << " P50 Latency: " << sorted[s50] * 1000.0 << " ms\n" + << " P90 Latency: " << sorted[s90] * 1000.0 << " ms\n" + << " P95 Latency: " << sorted[s95] * 1000.0 << " ms\n" + << " P99 Latency: " << sorted[s99] * 1000.0 << " ms" << std::endl; } }; if (have_file) { output_per_shape(outfile); } + + output_per_shape(std::cout); } } @@ -147,44 +160,6 @@ void PerformanceRunner::LogSessionCreationTime() { std::cout << "\nSession creation time cost: " << session_create_duration.count() << " s\n"; } -void PerformanceRunner::PrintPerShapeStats() const { - const auto& shape_groups = performance_test_config_.run_config.data_shape_groups; - const auto& per_shape_costs = performance_result_.per_shape_time_costs_total; - size_t num_groups = per_shape_costs.size(); - - std::cout << "\nLatency per shape group:" << std::endl; - - for (size_t g = 0; g < num_groups; g++) { - std::cout << " " << (g + 1) << ". " << FormatShapeGroup(shape_groups, g) << std::endl; - - const auto& time_costs = per_shape_costs[g]; - if (time_costs.empty()) { - std::cout << " (no data)" << std::endl; - continue; - } - - std::vector sorted_time = time_costs; - std::sort(sorted_time.begin(), sorted_time.end()); - size_t total = sorted_time.size(); - size_t n50 = static_cast(total * 0.5); - size_t n90 = static_cast(total * 0.9); - size_t n95 = static_cast(total * 0.95); - size_t n99 = static_cast(total * 0.99); - - double avg = std::accumulate(sorted_time.begin(), sorted_time.end(), 0.0) / static_cast(total); - - std::cout << " Iterations: " << total << "\n" - << " Average Latency: " << avg * 1000.0 << " ms\n" - << " Min Latency: " << sorted_time[0] * 1000.0 << " ms\n" - << " Max Latency: " << sorted_time[total - 1] * 1000.0 << " ms\n" - << " P50 Latency: " << sorted_time[n50] * 1000.0 << " ms\n" - << " P90 Latency: " << sorted_time[n90] * 1000.0 << " ms\n" - << " P95 Latency: " << sorted_time[n95] * 1000.0 << " ms\n" - << " P99 Latency: " << sorted_time[n99] * 1000.0 << " ms" - << std::endl; - } -} - Status PerformanceRunner::Run() { if (!Initialize()) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "failed to initialize."); @@ -246,10 +221,6 @@ Status PerformanceRunner::Run() { << "Peak working set size: " << performance_result_.peak_workingset_size << " bytes" << std::endl; - if (!performance_result_.per_shape_time_costs_total.empty()) { - PrintPerShapeStats(); - } - return Status::OK(); } diff --git a/onnxruntime/test/perftest/performance_runner.h b/onnxruntime/test/perftest/performance_runner.h index d1f99044f42c1..34b699573ddc4 100644 --- a/onnxruntime/test/perftest/performance_runner.h +++ b/onnxruntime/test/perftest/performance_runner.h @@ -40,7 +40,8 @@ struct PerformanceResult { std::vector> per_shape_time_costs_total; std::string model_name; - void DumpToFile(const std::basic_string& path, bool f_include_statistics = false) const; + void DumpToFile(const std::basic_string& path, bool f_include_statistics = false, + const std::map>>& shape_groups = {}) const; }; class PerformanceRunner { @@ -56,13 +57,13 @@ class PerformanceRunner { inline void SerializeResult() const { performance_result_.DumpToFile(performance_test_config_.model_info.result_file_path, - performance_test_config_.run_config.f_dump_statistics); + performance_test_config_.run_config.f_dump_statistics, + performance_test_config_.run_config.data_shape_groups); } ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(PerformanceRunner); private: bool Initialize(); - void PrintPerShapeStats() const; template Status RunOneIteration() { diff --git a/onnxruntime/test/perftest/strings_helper.cc b/onnxruntime/test/perftest/strings_helper.cc index 5e9fd8cb649b8..bb57b66303c0b 100644 --- a/onnxruntime/test/perftest/strings_helper.cc +++ b/onnxruntime/test/perftest/strings_helper.cc @@ -6,6 +6,7 @@ #include #include #include +#include #include "strings_helper.h" #include "core/common/common.h" @@ -301,6 +302,9 @@ bool ParseDataShapeGroups(const std::string& input, } std::string FormatShapeGroup(const std::map>>& groups, size_t g) { + if (groups.empty() || g >= groups.begin()->second.size()) { + return ""; + } std::string result; for (const auto& [name, shapes] : groups) { if (!result.empty()) result += ", "; diff --git a/onnxruntime/test/perftest/strings_helper.h b/onnxruntime/test/perftest/strings_helper.h index 07dc3a9c96518..38ad40cf1a92a 100644 --- a/onnxruntime/test/perftest/strings_helper.h +++ b/onnxruntime/test/perftest/strings_helper.h @@ -2,9 +2,13 @@ // Copyright (c) 2023 NVIDIA Corporation. // SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates // Licensed under the MIT License. + +#pragma once + #include -#include #include +#include +#include #include #include #include