Skip to content

Recover the verbose and artifacts saving utility in test framework #147

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
qti-chuteng merged 2 commits into from
Mar 19, 2026
Merged

Recover the verbose and artifacts saving utility in test framework #147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9204a25
Ensure the verbose and artifacts saving utility in test framework with
qti-hungjuiw Mar 11, 2026
ccecd59
Remove unnecessary attributes
qti-hungjuiw Mar 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions onnxruntime/test/providers/qnn/qnn_test_utils.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,14 @@ void RunQnnModelTest(const GetTestModelFn& build_test_case, ProviderOptions prov
std::string model_data;
helper.model_.SerializeToString(&model_data);

if (QNNTestEnvironment::GetInstance().dump_onnx()) {
// TODO: Use public logger
// LOGS(logging_manager.DefaultLogger(), VERBOSE) << "Save onnx model at: " << dump_path;
Comment thread
qti-chuteng marked this conversation as resolved.
auto dump_path = output_dir / "dumped_f32_model.onnx";
std::ofstream ofs(dump_path, std::ios::binary);
ofs.write(model_data.data(), static_cast<std::streamsize>(model_data.size()));
}

if (QNNTestEnvironment::GetInstance().dump_dlc()) {
provider_options["dump_qnn_ir_dlc"] = "1";
provider_options["dump_qnn_ir_dlc_dir"] = output_dir.string();
Expand All @@ -322,8 +330,11 @@ void RunQnnModelTest(const GetTestModelFn& build_test_case, ProviderOptions prov
const std::string& registration_name = "QNNExecutionProvider";
Ort::SessionOptions session_options;

session_options.SetLogSeverityLevel(log_severity);
session_options.AddConfigEntry(kOrtSessionOptionsRecordEpGraphAssignmentInfo, "1");
session_options.SetLogSeverityLevel(log_severity);
if (QNNTestEnvironment::GetInstance().verbose()) {
session_options.SetLogSeverityLevel(OrtLoggingLevel::ORT_LOGGING_LEVEL_VERBOSE);
}

TryEnableQNNSaver(provider_options);
RegisterQnnEpLibrary(registered_ep_device, session_options, registration_name, provider_options);
Expand Down Expand Up @@ -391,6 +402,7 @@ void InferenceModel(const std::string& model_data,
ExpectedEPNodeAssignment expected_ep_assignment,
std::unordered_map<std::string, Ort::Value>& feeds,
std::vector<Ort::Value>& output_vals,
OrtLoggingLevel log_severity,
const std::unordered_map<std::string, std::string>& session_option_pairs,
std::optional<GraphOptimizationLevel> graph_optimization_level,
std::function<void(const Graph&)>* graph_checker [[maybe_unused]]) {
Expand All @@ -403,9 +415,10 @@ void InferenceModel(const std::string& model_data,
RegisterQnnEpLibrary(registered_ep_device, session_options, registration_name, provider_options);

session_options.SetLogId(log_id);

// Uncomment to dump verbose output to stdout.
// session_options.SetLogSeverityLevel(ORT_LOGGING_LEVEL_VERBOSE);
session_options.SetLogSeverityLevel(log_severity);
if (QNNTestEnvironment::GetInstance().verbose()) {
session_options.SetLogSeverityLevel(OrtLoggingLevel::ORT_LOGGING_LEVEL_VERBOSE);
}

for (auto key_value : session_option_pairs) {
session_options.AddConfigEntry(key_value.first.c_str(), key_value.second.c_str());
Expand Down
13 changes: 7 additions & 6 deletions onnxruntime/test/providers/qnn/qnn_test_utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -587,6 +587,7 @@ void InferenceModel(const std::string& model_data,
ExpectedEPNodeAssignment expected_ep_assignment,
std::unordered_map<std::string, Ort::Value>& feeds,
std::vector<Ort::Value>& output_vals,
OrtLoggingLevel log_severity = OrtLoggingLevel::ORT_LOGGING_LEVEL_ERROR,
const std::unordered_map<std::string, std::string>& session_option_pairs = {},
std::optional<GraphOptimizationLevel> graph_optimization_level = std::nullopt,
std::function<void(const Graph&)>* graph_checker = nullptr);
Expand Down Expand Up @@ -834,7 +835,7 @@ inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn,
ProviderOptions qnn_options, int opset_version,
ExpectedEPNodeAssignment expected_ep_assignment,
QDQTolerance tolerance = QDQTolerance(),
OrtLoggingLevel log_severity [[maybe_unused]] = OrtLoggingLevel::ORT_LOGGING_LEVEL_ERROR,
OrtLoggingLevel log_severity = OrtLoggingLevel::ORT_LOGGING_LEVEL_ERROR,
const std::string& qnn_ctx_model_path = "",
const std::unordered_map<std::string, std::string>& session_option_pairs = {},
std::optional<GraphOptimizationLevel> graph_optimization_level = std::nullopt,
Expand Down Expand Up @@ -936,8 +937,6 @@ inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn,
qnn_options["json_qnn_graph_dir"] = output_dir.string();
}

// TODO: Enable QNN_VERBOSE to SetLogSeverityLevel

TryEnableQNNSaver(qnn_options);

// Run with QNN.
Expand All @@ -954,6 +953,7 @@ inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn,
expected_ep_assignment,
qdq_helper.feeds_,
qnn_qdq_outputs,
log_severity,
session_option_pairs,
graph_optimization_level);
} else {
Expand All @@ -963,6 +963,7 @@ inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn,
expected_ep_assignment,
qdq_helper.feeds_,
qnn_qdq_outputs,
log_severity,
session_option_pairs,
graph_optimization_level,
qnn_ep_graph_checker);
Expand Down Expand Up @@ -1087,7 +1088,7 @@ inline void TestFp16ModelAccuracy(const GetTestModelFn& f32_model_fn,
int opset_version,
ExpectedEPNodeAssignment expected_ep_assignment,
float tolerance = 0.004,
OrtLoggingLevel log_severity [[maybe_unused]] = OrtLoggingLevel::ORT_LOGGING_LEVEL_ERROR,
OrtLoggingLevel log_severity = OrtLoggingLevel::ORT_LOGGING_LEVEL_ERROR,
const std::string& qnn_ctx_model_path = "",
const std::unordered_map<std::string, std::string>& session_option_pairs = {}) {
std::filesystem::path output_dir;
Expand Down Expand Up @@ -1179,8 +1180,6 @@ inline void TestFp16ModelAccuracy(const GetTestModelFn& f32_model_fn,
qnn_options["json_qnn_graph_dir"] = output_dir.string();
}

// TODO: Enable QNN_VERBOSE to SetLogSeverityLevel

TryEnableQNNSaver(qnn_options);

// Run with QNN.
Expand All @@ -1197,6 +1196,7 @@ inline void TestFp16ModelAccuracy(const GetTestModelFn& f32_model_fn,
expected_ep_assignment,
f16_helper.feeds_,
qnn_f16_outputs,
log_severity,
session_option_pairs);
} else {
InferenceModel(f16_model_data,
Expand All @@ -1205,6 +1205,7 @@ inline void TestFp16ModelAccuracy(const GetTestModelFn& f32_model_fn,
expected_ep_assignment,
f16_helper.feeds_,
qnn_f16_outputs,
log_severity,
session_option_pairs);
}

Expand Down
Loading