diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 3f878ac796ee3..c8698140a8731 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -214,6 +214,15 @@ ABSL_FLAG(bool, compile_ep_context, DefaultPerformanceTestConfig().run_config.co ABSL_FLAG(std::string, compile_model_path, "model_ctx.onnx", "The compiled model path for saving EP context model. Overwrites if already exists"); ABSL_FLAG(bool, compile_binary_embed, DefaultPerformanceTestConfig().run_config.compile_binary_embed, "Embed binary blob within EP context node"); ABSL_FLAG(bool, compile_only, DefaultPerformanceTestConfig().run_config.compile_only, "Only compile EP context model without running it"); +ABSL_FLAG(std::string, data_shape, "", + "Specifies input shapes for multi-shape profiling within a single session.\n" + "The model is compiled once and run with each shape group in round-robin order.\n" + "[Usage]: --data_shape \"input_name:[d0,d1,...][d0,d1,...] ...\"\n" + "[Example]: --data_shape \"input:[1,3,224,224][1,3,448,448][1,3,112,112]\"\n" + "With -I: generates random input of the specified shapes.\n" + "Without -I: selects test data folders whose tensor shapes match.\n" + "All inputs must have the same number of shape groups.\n" + "All dimension values must be positive integers."); ABSL_FLAG(bool, h, false, "Print program usage."); namespace onnxruntime { @@ -481,6 +490,16 @@ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int a // -I test_config.run_config.generate_model_input_binding = absl::GetFlag(FLAGS_I); + // --data_shape + { + const auto& data_shape_str = absl::GetFlag(FLAGS_data_shape); + if (!data_shape_str.empty()) { + if (!ParseDataShapeGroups(data_shape_str, test_config.run_config.data_shape_groups)) { + return false; + } + } + } + // -d if (absl::GetFlag(FLAGS_d) < 0) return false; test_config.run_config.cudnn_conv_algo = absl::GetFlag(FLAGS_d); diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 679be75fa586e..ae9ed2a504320 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -42,9 +42,15 @@ namespace onnxruntime { namespace perftest { RunTiming OnnxRuntimeTestSession::Run() { - // Randomly pick one OrtValueArray from test_inputs_. (NOT ThreadSafe) - const std::uniform_int_distribution::param_type p(0, static_cast(test_inputs_.size() - 1)); - const size_t id = static_cast(dist_(rand_engine_, p)); + // Select input set: round-robin for multi-shape mode, random otherwise. + size_t id; + if (use_round_robin_ && test_inputs_.size() > 1) { + id = round_robin_counter_.fetch_add(1, std::memory_order_relaxed) % test_inputs_.size(); + } else { + // Random selection (not thread-safe). + const std::uniform_int_distribution::param_type p(0, static_cast(test_inputs_.size() - 1)); + id = static_cast(dist_(rand_engine_, p)); + } auto& input = test_inputs_.at(id); auto start = std::chrono::high_resolution_clock::now(); @@ -88,6 +94,7 @@ RunTiming OnnxRuntimeTestSession::Run() { timing.submit_timing = std::chrono::high_resolution_clock::now() - start; timing.total_timing = timing.submit_timing; } + timing.test_input_index = id; return timing; } @@ -1083,54 +1090,139 @@ static void InitializeTensorWithSeed(int32_t seed, Ort::Value& tensor) { #undef CASE_FOR_TYPE } +void OnnxRuntimeTestSession::CreateAndStoreGeneratedInput(size_t test_data_id, size_t input_idx, + const std::vector& dims, + ONNXTensorElementDataType element_type, int32_t seed) { + if (device_memory_name_ != CUDA) { + Ort::Value input_tensor = Ort::Value::CreateTensor(allocator_, (const int64_t*)dims.data(), + dims.size(), element_type); + InitializeTensorWithSeed(seed, input_tensor); + PreLoadTestData(test_data_id, input_idx, std::move(input_tensor)); + } +#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_NV) + else { + Ort::AllocatorWithDefaultOptions default_allocator; + Ort::Value default_tensor = Ort::Value::CreateTensor(default_allocator, (const int64_t*)dims.data(), + dims.size(), element_type); + InitializeTensorWithSeed(seed, default_tensor); + + const void* default_ptr = default_tensor.GetTensorRawData(); + size_t total_bytes = default_tensor.GetTensorSizeInBytes(); + + Ort::Value cuda_tensor = Ort::Value::CreateTensor(allocator_, dims.data(), + dims.size(), element_type); + void* cuda_ptr = cuda_tensor.GetTensorMutableData(); + + cudaError_t cuda_err = cudaMemcpy(cuda_ptr, default_ptr, total_bytes, cudaMemcpyHostToDevice); + if (cuda_err != cudaSuccess) { + ORT_THROW("Failed to copy tensor data from CPU to CUDA device. CUDA Error: ", cudaGetErrorString(cuda_err)); + } + PreLoadTestData(test_data_id, input_idx, std::move(cuda_tensor)); + } +#endif +} + bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { - Ort::AllocatorWithDefaultOptions default_allocator; - // iterate over all input nodes for (size_t i = 0; i < static_cast(input_length_); i++) { Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); - if (type_info.GetONNXType() == ONNX_TYPE_TENSOR) { - auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - std::vector input_node_dim = tensor_info.GetShape(); + if (type_info.GetONNXType() != ONNX_TYPE_TENSOR) { + continue; + } - // free dimensions are treated as 1 if not overridden - auto transform_fcn = [](int64_t input) { return (input == -1) ? -input : input; }; - std::transform(input_node_dim.begin(), input_node_dim.end(), input_node_dim.begin(), transform_fcn); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + std::vector input_node_dim = tensor_info.GetShape(); - if (device_memory_name_ != CUDA) { - Ort::Value input_tensor = Ort::Value::CreateTensor(allocator_, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); - InitializeTensorWithSeed(seed, input_tensor); - PreLoadTestData(0, i, std::move(input_tensor)); - } -// Create tensor on CPU, initialize and copy to CUDA tensor -#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_NV) - else { - Ort::Value default_tensor = Ort::Value::CreateTensor(default_allocator, (const int64_t*)input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); - InitializeTensorWithSeed(seed, default_tensor); + // free dimensions are treated as 1 if not overridden + auto transform_fcn = [](int64_t input) { return (input == -1) ? -input : input; }; + std::transform(input_node_dim.begin(), input_node_dim.end(), input_node_dim.begin(), transform_fcn); - // Get pointer to CPU tensor data - const void* default_ptr = default_tensor.GetTensorRawData(); + CreateAndStoreGeneratedInput(0, i, input_node_dim, tensor_info.GetElementType(), seed); + } + return true; +} - size_t total_bytes = default_tensor.GetTensorSizeInBytes(); +bool OnnxRuntimeTestSession::PopulateGeneratedMultiShapeInputTestData( + int32_t seed, + const std::map>>& data_shape_groups) { + // Validate that all input names in data_shape_groups exist in the model + for (const auto& [name, groups] : data_shape_groups) { + bool found = false; + for (int i = 0; i < input_length_; i++) { + if (input_names_str_[i] == name) { + found = true; + break; + } + } + if (!found) { + std::cerr << "Error: --data_shape specifies unknown input '" << name << "'." << std::endl; + return false; + } + } - Ort::Value cuda_tensor = Ort::Value::CreateTensor(allocator_, input_node_dim.data(), - input_node_dim.size(), tensor_info.GetElementType()); + const size_t num_groups = data_shape_groups.begin()->second.size(); - void* cuda_ptr = cuda_tensor.GetTensorMutableData(); + for (size_t g = 0; g < num_groups; g++) { + for (size_t i = 0; i < static_cast(input_length_); i++) { + Ort::TypeInfo type_info = session_.GetInputTypeInfo(i); + if (type_info.GetONNXType() != ONNX_TYPE_TENSOR) { + continue; + } - // Copy the initialized data from CPU to GPU - cudaError_t cuda_err = cudaMemcpy(cuda_ptr, default_ptr, total_bytes, cudaMemcpyHostToDevice); - if (cuda_err != cudaSuccess) { - ORT_THROW("Failed to copy tensor data from CPU to CUDA device. CUDA Error: ", cudaGetErrorString(cuda_err)); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + std::vector input_node_dim; + + // Use user-specified shape if available, otherwise fall back to model metadata + auto it = data_shape_groups.find(input_names_str_[i]); + if (it != data_shape_groups.end()) { + input_node_dim = it->second[g]; + const auto model_shape = tensor_info.GetShape(); + if (!model_shape.empty() && input_node_dim.size() != model_shape.size()) { + std::cerr << "Error: --data_shape rank mismatch for input '" << input_names_str_[i] + << "': expected " << model_shape.size() << " dims but got " << input_node_dim.size() << "." << std::endl; + return false; + } + } else { + input_node_dim = tensor_info.GetShape(); + bool has_dynamic_dim = std::any_of(input_node_dim.begin(), input_node_dim.end(), + [](int64_t d) { return d == -1; }); + auto transform_fcn = [](int64_t input) { return (input == -1) ? -input : input; }; + std::transform(input_node_dim.begin(), input_node_dim.end(), input_node_dim.begin(), transform_fcn); + if (g == 0 && has_dynamic_dim) { + std::cerr << "Warning: input '" << input_names_str_[i] + << "' not specified in --data_shape; using inferred shape ["; + for (size_t d = 0; d < input_node_dim.size(); d++) { + if (d > 0) std::cerr << ","; + std::cerr << input_node_dim[d]; + } + std::cerr << "] (dynamic dims defaulted to 1)." << std::endl; } - PreLoadTestData(0, i, std::move(cuda_tensor)); } -#endif + + CreateAndStoreGeneratedInput(g, i, input_node_dim, tensor_info.GetElementType(), seed); } } + use_round_robin_ = true; return true; } + +std::vector OnnxRuntimeTestSession::GetLoadedInputShape(size_t test_data_id, size_t input_id) const { + const auto& v = test_inputs_.at(test_data_id).at(input_id); + if (!v.IsTensor()) { + ORT_THROW("--data_shape only supports tensor inputs; input_id=", input_id, " in test_data_id=", test_data_id, + " is not a tensor."); + } + return v.GetTensorTypeAndShapeInfo().GetShape(); +} + +void OnnxRuntimeTestSession::SelectTestDataSets(const std::vector& selected_ids) { + std::vector> filtered; + filtered.reserve(selected_ids.size()); + for (size_t id : selected_ids) { + filtered.push_back(std::move(test_inputs_.at(id))); + } + test_inputs_ = std::move(filtered); +} + OnnxRuntimeTestSession::~OnnxRuntimeTestSession() { #ifdef USE_CUDA if (device_memory_name_ == CUDA && stream_ != nullptr) { diff --git a/onnxruntime/test/perftest/ort_test_session.h b/onnxruntime/test/perftest/ort_test_session.h index 0a2c683428bf2..422e76ba0e72d 100644 --- a/onnxruntime/test/perftest/ort_test_session.h +++ b/onnxruntime/test/perftest/ort_test_session.h @@ -2,6 +2,7 @@ // Licensed under the MIT License. #pragma once +#include #include #include #include "test_configuration.h" @@ -31,6 +32,13 @@ class OnnxRuntimeTestSession : public TestSession { } bool PopulateGeneratedInputTestData(int32_t seed); + bool PopulateGeneratedMultiShapeInputTestData( + int32_t seed, + const std::map>>& data_shape_groups); + + std::vector GetLoadedInputShape(size_t test_data_id, size_t input_id) const; + void SelectTestDataSets(const std::vector& selected_ids); + void SetUseRoundRobin(bool v) { use_round_robin_ = v; } ~OnnxRuntimeTestSession(); @@ -39,6 +47,10 @@ class OnnxRuntimeTestSession : public TestSession { ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OnnxRuntimeTestSession); private: + void CreateAndStoreGeneratedInput(size_t test_data_id, size_t input_idx, + const std::vector& dims, + ONNXTensorElementDataType element_type, int32_t seed); + Ort::Session session_{nullptr}; std::mt19937 rand_engine_; std::uniform_int_distribution dist_; @@ -60,6 +72,8 @@ class OnnxRuntimeTestSession : public TestSession { std::string device_memory_name_; // Device memory type name to use from the list in allocator.h const std::unordered_map& run_config_entries_; bool has_dynamic_output_shapes_ = false; + std::atomic round_robin_counter_{0}; + bool use_round_robin_{false}; #if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_NV) cudaStream_t stream_; // Device stream if required by IO bindings #endif diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index a14e6437c0e01..91334ae00f81f 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -10,6 +10,7 @@ #include #include "TestCase.h" +#include "strings_helper.h" #include "utils.h" #include "ort_test_session.h" using onnxruntime::Status; @@ -54,7 +55,8 @@ Eigen::ThreadPoolInterface* GetDefaultThreadPool(const onnxruntime::Env& env) { namespace onnxruntime { namespace perftest { -void PerformanceResult::DumpToFile(const std::basic_string& path, bool f_include_statistics) const { +void PerformanceResult::DumpToFile(const std::basic_string& path, bool f_include_statistics, + const std::map>>& shape_groups) const { bool have_file = !path.empty(); std::ofstream outfile; @@ -109,6 +111,48 @@ void PerformanceResult::DumpToFile(const std::basic_string& path, boo output_stats(std::cout); } + + // Per-shape statistics (when --data_shape is used) + if (!per_shape_time_costs_total.empty() && f_include_statistics) { + auto output_per_shape = [&](std::ostream& ostream) { + ostream << "\nLatency per shape group:" << std::endl; + for (size_t g = 0; g < per_shape_time_costs_total.size(); g++) { + const auto& shape_costs = per_shape_time_costs_total[g]; + + std::string label = FormatShapeGroup(shape_groups, g); + ostream << " " << (g + 1) << ". " << (label.empty() ? "shape group" : label) << std::endl; + + if (shape_costs.empty()) { + ostream << " (no data)" << std::endl; + continue; + } + + std::vector sorted = shape_costs; + std::sort(sorted.begin(), sorted.end()); + size_t count = sorted.size(); + size_t s50 = static_cast(count * 0.5); + size_t s90 = static_cast(count * 0.9); + size_t s95 = static_cast(count * 0.95); + size_t s99 = static_cast(count * 0.99); + double avg = std::accumulate(sorted.begin(), sorted.end(), 0.0) / static_cast(count); + + ostream << " Iterations: " << count << "\n" + << " Average Latency: " << avg * 1000.0 << " ms\n" + << " Min Latency: " << sorted[0] * 1000.0 << " ms\n" + << " Max Latency: " << sorted[count - 1] * 1000.0 << " ms\n" + << " P50 Latency: " << sorted[s50] * 1000.0 << " ms\n" + << " P90 Latency: " << sorted[s90] * 1000.0 << " ms\n" + << " P95 Latency: " << sorted[s95] * 1000.0 << " ms\n" + << " P99 Latency: " << sorted[s99] * 1000.0 << " ms" << std::endl; + } + }; + + if (have_file) { + output_per_shape(outfile); + } + + output_per_shape(std::cout); + } } void PerformanceRunner::LogSessionCreationTime() { @@ -121,9 +165,15 @@ Status PerformanceRunner::Run() { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "failed to initialize."); } - // warm up + // warm up — run one iteration per shape group to ensure all shapes are warmed + size_t warmup_count = 1; + if (!performance_test_config_.run_config.data_shape_groups.empty()) { + warmup_count = performance_test_config_.run_config.data_shape_groups.begin()->second.size(); + } initial_inference_result_.start = std::chrono::high_resolution_clock::now(); - ORT_RETURN_IF_ERROR(RunOneIteration()); + for (size_t w = 0; w < warmup_count; w++) { + ORT_RETURN_IF_ERROR(RunOneIteration()); + } initial_inference_result_.end = std::chrono::high_resolution_clock::now(); // TODO: start profiling @@ -306,9 +356,20 @@ bool PerformanceRunner::Initialize() { test_case_ = CreateOnnxTestCase(narrow_model_name, std::move(test_model_info_), 0.0, 0.0); if (performance_test_config_.run_config.generate_model_input_binding) { - return static_cast( - session_.get()) - ->PopulateGeneratedInputTestData(performance_test_config_.run_config.random_seed_for_input_data); + auto* ort_session = static_cast(session_.get()); + if (!performance_test_config_.run_config.data_shape_groups.empty()) { + if (!ort_session->PopulateGeneratedMultiShapeInputTestData( + performance_test_config_.run_config.random_seed_for_input_data, + performance_test_config_.run_config.data_shape_groups)) { + return false; + } + // Pre-size per-shape timing vectors + size_t num_groups = performance_test_config_.run_config.data_shape_groups.begin()->second.size(); + performance_result_.per_shape_time_costs_total.resize(num_groups); + return true; + } + return ort_session->PopulateGeneratedInputTestData( + performance_test_config_.run_config.random_seed_for_input_data); } // TODO: Place input tensor on cpu memory if dnnl provider type to avoid CopyTensor logic in CopyInputAcrossDevices @@ -317,11 +378,10 @@ bool PerformanceRunner::Initialize() { std::cout << "there is no test data for model " << test_case_->GetTestCaseName() << std::endl; return false; } + int input_count = test_model_info->GetInputCount(); for (size_t test_data_id = 0; test_data_id != test_data_count; ++test_data_id) { std::unordered_map feeds; test_case_->LoadTestData(test_data_id /* id */, b_, feeds, true); - // Discard the names in feeds - int input_count = test_model_info->GetInputCount(); for (int i = 0; i != input_count; ++i) { auto iter = feeds.find(test_model_info->GetInputName(i)); if (iter == feeds.end()) { @@ -333,6 +393,65 @@ bool PerformanceRunner::Initialize() { } } + // When --data_shape is specified without -I, select only the test data sets + // whose tensor shapes match the requested shape groups. + const auto& data_shape_groups = performance_test_config_.run_config.data_shape_groups; + if (!data_shape_groups.empty()) { + auto* ort_session = static_cast(session_.get()); + size_t num_groups = data_shape_groups.begin()->second.size(); + + // Build input name -> index map + std::unordered_map input_name_to_idx; + for (int i = 0; i < input_count; ++i) { + input_name_to_idx[test_model_info->GetInputName(i)] = static_cast(i); + } + + // Validate that all names in data_shape_groups exist in the model + for (const auto& [name, groups] : data_shape_groups) { + if (input_name_to_idx.find(name) == input_name_to_idx.end()) { + std::cerr << "Error: --data_shape specifies unknown input '" << name << "'." << std::endl; + return false; + } + } + + // For each shape group, find the matching test data set + std::vector selected_ids; + for (size_t g = 0; g < num_groups; ++g) { + bool found = false; + for (size_t test_data_id = 0; test_data_id < test_data_count; ++test_data_id) { + bool match = true; + for (const auto& [input_name, shape_list] : data_shape_groups) { + size_t input_idx = input_name_to_idx[input_name]; + auto loaded_shape = ort_session->GetLoadedInputShape(test_data_id, input_idx); + if (loaded_shape != shape_list[g]) { + match = false; + break; + } + } + if (match) { + if (std::find(selected_ids.begin(), selected_ids.end(), test_data_id) != selected_ids.end()) { + std::cerr << "Error: --data_shape shape group " << (g + 1) + << " matches a test data set already selected. " + << "Duplicate shape groups are not supported when selecting from test data." << std::endl; + return false; + } + selected_ids.push_back(test_data_id); + found = true; + break; + } + } + if (!found) { + std::cerr << "No test data found matching shape group " << (g + 1) << ": " + << FormatShapeGroup(data_shape_groups, g) << "." << std::endl; + return false; + } + } + + ort_session->SelectTestDataSets(selected_ids); + ort_session->SetUseRoundRobin(true); + performance_result_.per_shape_time_costs_total.resize(num_groups); + } + return true; } diff --git a/onnxruntime/test/perftest/performance_runner.h b/onnxruntime/test/perftest/performance_runner.h index f67ab9f5c2287..34b699573ddc4 100644 --- a/onnxruntime/test/perftest/performance_runner.h +++ b/onnxruntime/test/perftest/performance_runner.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -35,9 +36,12 @@ struct PerformanceResult { double total_time_cost{0}; std::vector time_costs_submission; std::vector time_costs_total; + // Per-shape-group timing (only populated when --data_shape is used) + std::vector> per_shape_time_costs_total; std::string model_name; - void DumpToFile(const std::basic_string& path, bool f_include_statistics = false) const; + void DumpToFile(const std::basic_string& path, bool f_include_statistics = false, + const std::map>>& shape_groups = {}) const; }; class PerformanceRunner { @@ -53,7 +57,8 @@ class PerformanceRunner { inline void SerializeResult() const { performance_result_.DumpToFile(performance_test_config_.model_info.result_file_path, - performance_test_config_.run_config.f_dump_statistics); + performance_test_config_.run_config.f_dump_statistics, + performance_test_config_.run_config.data_shape_groups); } ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(PerformanceRunner); @@ -80,6 +85,14 @@ class PerformanceRunner { performance_result_.time_costs_total.emplace_back(duration_seconds.total_timing.count()); // if the test did not run async the device timing will be equal to the CPU timing performance_result_.total_time_cost += duration_seconds.total_timing.count(); + // Record per-shape timing when multi-shape mode is active + if (!performance_result_.per_shape_time_costs_total.empty()) { + size_t shape_idx = duration_seconds.test_input_index; + if (shape_idx < performance_result_.per_shape_time_costs_total.size()) { + performance_result_.per_shape_time_costs_total[shape_idx].emplace_back( + duration_seconds.total_timing.count()); + } + } if (performance_test_config_.run_config.f_verbose) { std::cout << "iteration:" << performance_result_.time_costs_submission.size() << "," << "time_cost_submission:" << performance_result_.time_costs_submission.back() << "," diff --git a/onnxruntime/test/perftest/strings_helper.cc b/onnxruntime/test/perftest/strings_helper.cc index d9fd2a2a55c09..bb57b66303c0b 100644 --- a/onnxruntime/test/perftest/strings_helper.cc +++ b/onnxruntime/test/perftest/strings_helper.cc @@ -3,8 +3,10 @@ // SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates // Licensed under the MIT License. +#include #include #include +#include #include "strings_helper.h" #include "core/common/common.h" @@ -154,5 +156,168 @@ void ParseEpDeviceFilterKeyValuePairs(const std::string& input, std::vector>>& data_shape_groups) { + // Parse format: "input_name:[d0,d1,...][d0,d1,...] input_name2:[d0,d1][d0,d1]" + // Split on whitespace that is outside brackets to allow spaces inside shape specs. + std::vector input_specs; + std::string current; + int bracket_depth = 0; + for (char c : input) { + if (c == '[') { + bracket_depth++; + current += c; + } else if (c == ']') { + bracket_depth--; + current += c; + } else if ((c == ' ' || c == '\t') && bracket_depth == 0) { + if (!current.empty()) { + input_specs.push_back(current); + current.clear(); + } + } else { + current += c; + } + } + if (!current.empty()) { + input_specs.push_back(current); + } + + if (bracket_depth != 0) { + std::cerr << "Error parsing --data_shape: mismatched brackets (unbalanced '[' and ']')." << std::endl; + return false; + } + + for (const auto& input_spec : input_specs) { + // Split on first ':' to get input_name and shapes string + size_t colon_pos = input_spec.find(':'); + if (colon_pos == std::string::npos || colon_pos == 0) { + std::cerr << "Error parsing --data_shape: expected 'name:[shape]...' format, got '" << input_spec << "'." << std::endl; + return false; + } + + std::string input_name = input_spec.substr(0, colon_pos); + std::string shapes_str = input_spec.substr(colon_pos + 1); + + if (shapes_str.empty()) { + std::cerr << "Error parsing --data_shape: no shape groups provided for input '" << input_name << "'." << std::endl; + return false; + } + + std::vector> shape_groups; + + // Extract bracket-delimited shape groups: [d0,d1,...][d0,d1,...] + size_t pos = 0; + while (pos < shapes_str.size()) { + while (pos < shapes_str.size() && (shapes_str[pos] == ' ' || shapes_str[pos] == '\t')) { + ++pos; + } + if (pos >= shapes_str.size()) break; + if (shapes_str[pos] != '[') { + std::cerr << "Error parsing --data_shape: expected '[' at position " << pos + << " in shapes for input '" << input_name << "'." << std::endl; + return false; + } + + size_t close_bracket = shapes_str.find(']', pos); + if (close_bracket == std::string::npos) { + std::cerr << "Error parsing --data_shape: unmatched bracket for input '" << input_name << "'." << std::endl; + return false; + } + + std::string dims_str = shapes_str.substr(pos + 1, close_bracket - pos - 1); + if (dims_str.empty()) { + std::cerr << "Error parsing --data_shape: empty shape group for input '" << input_name << "'." << std::endl; + return false; + } + + // Parse comma-separated dimensions + std::vector dims; + std::stringstream dims_ss(dims_str); + std::string dim_token; + while (std::getline(dims_ss, dim_token, ',')) { + // Trim whitespace from dimension token + size_t start = dim_token.find_first_not_of(" \t"); + size_t end = dim_token.find_last_not_of(" \t"); + if (start == std::string::npos) { + std::cerr << "Error parsing --data_shape: empty dimension token for input '" << input_name << "'." << std::endl; + return false; + } + dim_token = dim_token.substr(start, end - start + 1); + + int64_t dim_val = 0; + const char* begin = dim_token.data(); + const char* end_ptr = begin + dim_token.size(); + auto [ptr, ec] = std::from_chars(begin, end_ptr, dim_val); + if (ec != std::errc{} || ptr != end_ptr) { + std::cerr << "Error parsing --data_shape: invalid dimension value '" + << dim_token << "' for input '" << input_name << "'." << std::endl; + return false; + } + if (dim_val <= 0) { + std::cerr << "Error parsing --data_shape: dimensions must be positive integers, got '" + << dim_token << "' for input '" << input_name << "'." << std::endl; + return false; + } + dims.push_back(dim_val); + } + + shape_groups.push_back(std::move(dims)); + pos = close_bracket + 1; + } + + if (shape_groups.empty()) { + std::cerr << "Error parsing --data_shape: no shape groups found for input '" << input_name << "'." << std::endl; + return false; + } + + if (data_shape_groups.count(input_name) > 0) { + std::cerr << "Error parsing --data_shape: duplicate input name '" << input_name << "'." << std::endl; + return false; + } + + data_shape_groups[input_name] = std::move(shape_groups); + } + + if (data_shape_groups.empty()) { + std::cerr << "Error parsing --data_shape: no input specifications found." << std::endl; + return false; + } + + // Validate all inputs have the same number of shape groups + if (data_shape_groups.size() > 1) { + size_t expected_count = data_shape_groups.begin()->second.size(); + for (const auto& [name, groups] : data_shape_groups) { + if (groups.size() != expected_count) { + std::cerr << "Error parsing --data_shape: all inputs must have the same number of shape groups. " + << "Input '" << data_shape_groups.begin()->first << "' has " << expected_count + << " groups but input '" << name << "' has " << groups.size() << "." << std::endl; + return false; + } + } + } + + return true; +} + +std::string FormatShapeGroup(const std::map>>& groups, size_t g) { + if (groups.empty() || g >= groups.begin()->second.size()) { + return ""; + } + std::string result; + for (const auto& [name, shapes] : groups) { + if (!result.empty()) result += ", "; + result += name + ":["; + const auto& dims = shapes[g]; + for (size_t d = 0; d < dims.size(); d++) { + if (d > 0) result += ","; + result += std::to_string(dims[d]); + } + result += "]"; + } + return result; +} + } // namespace perftest } // namespace onnxruntime diff --git a/onnxruntime/test/perftest/strings_helper.h b/onnxruntime/test/perftest/strings_helper.h index d6c6f6112ab6c..38ad40cf1a92a 100644 --- a/onnxruntime/test/perftest/strings_helper.h +++ b/onnxruntime/test/perftest/strings_helper.h @@ -2,8 +2,13 @@ // Copyright (c) 2023 NVIDIA Corporation. // SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates // Licensed under the MIT License. -#include + +#pragma once + +#include #include +#include +#include #include #include #include @@ -26,5 +31,11 @@ void ParseEpOptions(const std::string& input, std::vector& result); void ParseEpDeviceFilterKeyValuePairs(const std::string& input, std::vector>& result); + +bool ParseDataShapeGroups(const std::string& input, + std::map>>& data_shape_groups); + +std::string FormatShapeGroup(const std::map>>& groups, size_t g); + } // namespace perftest } // namespace onnxruntime diff --git a/onnxruntime/test/perftest/test_configuration.h b/onnxruntime/test/perftest/test_configuration.h index dea0b196f6bcb..b4e525902954a 100644 --- a/onnxruntime/test/perftest/test_configuration.h +++ b/onnxruntime/test/perftest/test_configuration.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "core/graph/constants.h" #include "core/framework/session_options.h" @@ -67,6 +68,7 @@ struct RunConfig { std::unordered_map run_config_entries; std::map free_dim_name_overrides; std::map free_dim_denotation_overrides; + std::map>> data_shape_groups; std::string intra_op_thread_affinities; bool disable_spinning = false; bool disable_spinning_between_run = false; diff --git a/onnxruntime/test/perftest/test_session.h b/onnxruntime/test/perftest/test_session.h index 71cb6062a93f4..d2b7cee97efb8 100644 --- a/onnxruntime/test/perftest/test_session.h +++ b/onnxruntime/test/perftest/test_session.h @@ -12,6 +12,7 @@ namespace perftest { struct RunTiming { std::chrono::duration submit_timing = std::chrono::seconds(0); std::chrono::duration total_timing = std::chrono::seconds(0); + size_t test_input_index = 0; // which test_inputs_ entry was used }; class TestSession {