From bbc39fa6ef5a21cf6016b52f1efffc7aa5bdd1c0 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 20 Aug 2020 07:42:08 -0700 Subject: [PATCH] add signal handler for fpe, bus error (#18956) * add handler for fpe, bus error * fix bug * fix build * copy comments --- include/mxnet/c_api_error.h | 45 +----- include/mxnet/runtime/c_runtime_api.h | 47 ++++++ python/mxnet/error.py | 4 + src/c_api/c_api_common.h | 6 +- src/c_api/c_api_error.cc | 206 -------------------------- src/c_api/c_api_profile.cc | 35 ----- src/initialize.cc | 190 +++++++++++++++++++++--- src/runtime/c_runtime_api.cc | 183 ++++++++++++++++++++++- 8 files changed, 403 insertions(+), 313 deletions(-) delete mode 100644 src/c_api/c_api_error.cc diff --git a/include/mxnet/c_api_error.h b/include/mxnet/c_api_error.h index 48cabc39c056..6d26b1499847 100644 --- a/include/mxnet/c_api_error.h +++ b/include/mxnet/c_api_error.h @@ -22,52 +22,9 @@ * \file c_api_error.h * \brief Error handling for C API. */ -#include +#include #ifndef MXNET_C_API_ERROR_H_ #define MXNET_C_API_ERROR_H_ -/*! - * \brief Macros to guard beginning and end section of all functions - * every function starts with API_BEGIN() - * and finishes with API_END() or API_END_HANDLE_ERROR() - * The finally clause contains procedure to cleanup states when an error happens. - */ -#define MX_API_BEGIN() \ - try { \ - on_enter_api(__FUNCTION__); -#define MX_API_END() \ - } \ - catch (const std::exception &_except_) { \ - on_exit_api(); \ - return MXAPIHandleException(_except_); \ - } \ - on_exit_api(); \ - return 0; // NOLINT(*) -#define MX_API_END_HANDLE_ERROR(Finalize) \ - } \ - catch (const std::exception &_except_) { \ - Finalize; \ - on_exit_api(); \ - return MXAPIHandleException(_except_); \ - } \ - on_exit_api(); \ - return 0; // NOLINT(*) - -/*! - * \brief Set the last error message needed by C API - * \param msg The error message to set. - */ -void MXAPISetLastError(const char* msg); -/*! - * \brief handle exception throwed out - * \param e the exception - * \return the return value of API after exception is handled - */ -int MXAPIHandleException(const std::exception &e); - -namespace mxnet { -extern void on_enter_api(const char *function); -extern void on_exit_api(); -} #endif // MXNET_C_API_ERROR_H_ diff --git a/include/mxnet/runtime/c_runtime_api.h b/include/mxnet/runtime/c_runtime_api.h index 69de9ca27d12..d07f62b8674f 100644 --- a/include/mxnet/runtime/c_runtime_api.h +++ b/include/mxnet/runtime/c_runtime_api.h @@ -26,6 +26,7 @@ #define MXNET_RUNTIME_C_RUNTIME_API_H_ #include +#include #ifdef __cplusplus extern "C" { @@ -177,4 +178,50 @@ MXNET_DLL int MXNetObjectTypeKey2Index(const char* type_key, unsigned* out_tinde #ifdef __cplusplus } // extern "C" #endif + + +/*! + * \brief Macros to guard beginning and end section of all functions + * every function starts with API_BEGIN() + * and finishes with API_END() or API_END_HANDLE_ERROR() + * The finally clause contains procedure to cleanup states when an error happens. + */ +#define MX_API_BEGIN() \ + try { \ + on_enter_api(__FUNCTION__); +#define MX_API_END() \ + } \ + catch (const std::exception &_except_) { \ + on_exit_api(); \ + return MXAPIHandleException(_except_); \ + } \ + on_exit_api(); \ + return 0; // NOLINT(*) +#define MX_API_END_HANDLE_ERROR(Finalize) \ + } \ + catch (const std::exception &_except_) { \ + Finalize; \ + on_exit_api(); \ + return MXAPIHandleException(_except_); \ + } \ + on_exit_api(); \ + return 0; // NOLINT(*) + +/*! + * \brief Set the last error message needed by C API + * \param msg The error message to set. + */ +void MXAPISetLastError(const char* msg); +/*! + * \brief handle exception throwed out + * \param e the exception + * \return the return value of API after exception is handled + */ +int MXAPIHandleException(const std::exception &e); + +namespace mxnet { +extern void on_enter_api(const char *function); +extern void on_exit_api(); +} + #endif // MXNET_RUNTIME_C_RUNTIME_API_H_ diff --git a/python/mxnet/error.py b/python/mxnet/error.py index b4110809c95a..674bf09c6626 100644 --- a/python/mxnet/error.py +++ b/python/mxnet/error.py @@ -57,3 +57,7 @@ def __init__(self, msg): register_error("AttributeError", AttributeError) register_error("IndexError", IndexError) register_error("NotImplementedError", NotImplementedError) +register_error("InternalError", InternalError) +register_error("IOError", IOError) +register_error("FloatingPointError", FloatingPointError) +register_error("RuntimeError", RuntimeError) diff --git a/src/c_api/c_api_common.h b/src/c_api/c_api_common.h index b88a087de617..e48a82529e13 100644 --- a/src/c_api/c_api_common.h +++ b/src/c_api/c_api_common.h @@ -19,8 +19,8 @@ /*! * Copyright (c) 2015 by Contributors - * \file c_api_error.h - * \brief Error handling for C API. + * \file c_api_common.h + * \brief Common C API utils */ #ifndef MXNET_C_API_C_API_COMMON_H_ #define MXNET_C_API_C_API_COMMON_H_ @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/c_api/c_api_error.cc b/src/c_api/c_api_error.cc deleted file mode 100644 index b7df6a5e51fe..000000000000 --- a/src/c_api/c_api_error.cc +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/*! - * Copyright (c) 2015 by Contributors - * \file c_api_error.cc - * \brief C error handling - */ -#include -#include "./c_api_common.h" - -#ifndef _LIBCPP_SGX_NO_IOSTREAMS -//-------------------------------------------------------- -// Error handling mechanism -// ------------------------------------------------------- -// Standard error message format, {} means optional -//-------------------------------------------------------- -// {error_type:} {message0} -// {message1} -// {message2} -// {Stack trace:} // stack traces follow by this line -// {trace 0} // two spaces in the begining. -// {trace 1} -// {trace 2} -//-------------------------------------------------------- -/*! - * \brief Normalize error message - * - * Parse them header generated by by LOG(FATAL) and CHECK - * and reformat the message into the standard format. - * - * This function will also merge all the stack traces into - * one trace and trim them. - * - * \param err_msg The error message. - * \return normalized message. - */ -std::string NormalizeError(std::string err_msg) { - // ------------------------------------------------------------------------ - // log with header, {} indicates optional - //------------------------------------------------------------------------- - // [timestamp] file_name:line_number: {check_msg:} {error_type:} {message0} - // {message1} - // Stack trace: - // {stack trace 0} - // {stack trace 1} - //------------------------------------------------------------------------- - // Normalzied version - //------------------------------------------------------------------------- - // error_type: check_msg message0 - // {message1} - // Stack trace: - // File file_name, line lineno - // {stack trace 0} - // {stack trace 1} - //------------------------------------------------------------------------- - int line_number = 0; - std::istringstream is(err_msg); - std::string line, file_name, error_type, check_msg; - - // Parse log header and set the fields, - // Return true if it the log is in correct format, - // return false if something is wrong. - auto parse_log_header = [&]() { - // skip timestamp - if (is.peek() != '[') { - getline(is, line); - return true; - } - if (!(is >> line)) return false; - // get filename - while (is.peek() == ' ') is.get(); - if (!getline(is, file_name, ':')) { - return false; - } else { - if (is.peek() == '\\' || is.peek() == '/') { - // windows path - if (!getline(is, line, ':')) return false; - file_name = file_name + ':' + line; - } - } - // get line number - if (!(is >> line_number)) return false; - // get rest of the message. - while (is.peek() == ' ' || is.peek() == ':') is.get(); - if (!getline(is, line)) return false; - // detect check message, rewrite to remote extra : - if (line.compare(0, 13, "Check failed:") == 0) { - size_t end_pos = line.find(':', 13); - if (end_pos == std::string::npos) return false; - check_msg = line.substr(0, end_pos + 1) + ' '; - line = line.substr(end_pos + 1); - } - return true; - }; - // if not in correct format, do not do any rewrite. - if (!parse_log_header()) return err_msg; - // Parse error type. - { - size_t start_pos = 0, end_pos; - for (; start_pos < line.length() && line[start_pos] == ' '; ++start_pos) {} - for (end_pos = start_pos; end_pos < line.length(); ++end_pos) { - char ch = line[end_pos]; - if (ch == ':') { - error_type = line.substr(start_pos, end_pos - start_pos); - break; - } - // [A-Z0-9a-z_.] - if (!std::isalpha(ch) && !std::isdigit(ch) && ch != '_' && ch != '.') break; - } - if (error_type.length() != 0) { - // if we successfully detected error_type: trim the following space. - for (start_pos = end_pos + 1; - start_pos < line.length() && line[start_pos] == ' '; ++start_pos) {} - line = line.substr(start_pos); - } else { - // did not detect error_type, use default value. - line = line.substr(start_pos); - error_type = "MXNetError"; - } - } - // Seperate out stack trace. - std::ostringstream os; - os << error_type << ": " << check_msg << line << '\n'; - - bool trace_mode = true; - std::vector stack_trace; - while (getline(is, line)) { - if (trace_mode) { - if (line.compare(0, 2, " ") == 0) { - stack_trace.push_back(line); - } else { - trace_mode = false; - // remove EOL trailing stacktrace. - if (line.length() == 0) continue; - } - } - if (!trace_mode) { - if (line.compare(0, 11, "Stack trace") == 0) { - trace_mode = true; - } else { - os << line << '\n'; - } - } - } - if (stack_trace.size() != 0 || file_name.length() != 0) { - os << "Stack trace:\n"; - if (file_name.length() != 0) { - os << " File \"" << file_name << "\", line " << line_number << "\n"; - } - // Print out stack traces, optionally trim the c++ traces - // about the frontends (as they will be provided by the frontends). - bool ffi_boundary = false; - for (const auto& line : stack_trace) { - // Heuristic to detect python ffi. - if (line.find("libffi.so") != std::string::npos || - line.find("core.cpython") != std::string::npos) { - ffi_boundary = true; - } - // If the backtrace is not c++ backtrace with the prefix " [bt]", - // then we can stop trimming. - if (ffi_boundary && line.compare(0, 6, " [bt]") != 0) { - ffi_boundary = false; - } - if (!ffi_boundary) { - os << line << '\n'; - } - } - } - return os.str(); -} - -#else -std::string NormalizeError(std::string err_msg) { - return err_msg; -} -#endif - -int MXAPIHandleException(const std::exception &e) { - MXAPISetLastError(NormalizeError(e.what()).c_str()); - return -1; -} - -const char *MXGetLastError() { - return NNGetLastError(); -} - -void MXAPISetLastError(const char* msg) { - NNAPISetLastError(msg); -} diff --git a/src/c_api/c_api_profile.cc b/src/c_api/c_api_profile.cc index 79d11b92dff6..e28b69057629 100644 --- a/src/c_api/c_api_profile.cc +++ b/src/c_api/c_api_profile.cc @@ -37,8 +37,6 @@ namespace mxnet { -// #define PROFILE_API_INCLUDE_AS_EVENT - static profiler::ProfileDomain api_domain("MXNET_C_API"); static profiler::ProfileCounter api_call_counter("MXNet C API Calls", &api_domain); static profiler::ProfileCounter api_concurrency_counter("MXNet C API Concurrency", @@ -48,9 +46,6 @@ static profiler::ProfileCounter api_concurrency_counter("MXNet C API Concurrency struct APICallTimingData { const char *name_; profiler::ProfileTask *task_; -#ifdef PROFILE_API_INCLUDE_AS_EVENT - profiler::ProfileEvent *event_; -#endif // PROFILE_API_INCLUDE_AS_EVENT }; /*! @@ -79,23 +74,6 @@ class ProfilingThreadData { return iter->second.get(); } -#ifdef PROFILE_API_INCLUDE_AS_EVENT - /*! - * \brief Retreive ProfileEvent object of the given name, or create if it doesn't exist - * \param name Name of the event - * \return Pointer to the stored or created ProfileEvent object - */ - profiler::ProfileEvent *profile_event(const char *name) { - // Per-thread so no lock necessary - auto iter = events_.find(name); - if (iter == events_.end()) { - iter = events_.emplace(std::make_pair(name, - std::make_unique(name))).first; - } - return iter->second.get(); - } -#endif // PROFILE_API_INCLUDE_AS_EVENT - /*! \brief nestable call stack */ std::stack calls_; /*! \brief Whether profiling actions should be ignored/excluded */ @@ -104,10 +82,6 @@ class ProfilingThreadData { private: /*! \brief tasks */ std::unordered_map> tasks_; -#ifdef PROFILE_API_INCLUDE_AS_EVENT - /*! \brief events */ - std::unordered_map> events_; -#endif // PROFILE_API_INCLUDE_AS_EVENT }; #if DMLC_CXX11_THREAD_LOCAL @@ -124,15 +98,9 @@ extern void on_enter_api(const char *function) { APICallTimingData data = { function , thread_profiling_data.profile_task(function, &api_domain) -#ifdef PROFILE_API_INCLUDE_AS_EVENT - , thread_profiling_data.profile_event(function) -#endif // PROFILE_API_INCLUDE_AS_EVENT }; thread_profiling_data.calls_.push(data); data.task_->start(); -#ifdef PROFILE_API_INCLUDE_AS_EVENT - data.event_->start(); -#endif // PROFILE_API_INCLUDE_AS_EVENT } } } @@ -141,9 +109,6 @@ extern void on_exit_api() { if (!thread_profiling_data.ignore_call_) { CHECK(!thread_profiling_data.calls_.empty()); APICallTimingData data = thread_profiling_data.calls_.top(); -#ifdef PROFILE_API_INCLUDE_AS_EVENT - data.event_->stop(); -#endif // PROFILE_API_INCLUDE_AS_EVENT data.task_->stop(); thread_profiling_data.calls_.pop(); --api_concurrency_counter; diff --git a/src/initialize.cc b/src/initialize.cc index 784e54f9a9d7..b207423a4fb5 100644 --- a/src/initialize.cc +++ b/src/initialize.cc @@ -23,15 +23,8 @@ * \brief initialize mxnet library */ #include "initialize.h" -#include -#include -#include "./engine/openmp.h" -#include "./operator/custom/custom-inl.h" -#if MXNET_USE_OPENCV -#include -#endif // MXNET_USE_OPENCV -#include "common/utils.h" -#include "engine/openmp.h" +#include +#include #if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) #include @@ -52,21 +45,28 @@ void win_err(char **err) { 0, nullptr); } #else +#include #include +#if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE +#include +#endif +#include #endif -#include +#include +#include +#include +#include "./engine/openmp.h" +#include "./operator/custom/custom-inl.h" +#if MXNET_USE_OPENCV +#include +#endif // MXNET_USE_OPENCV +#include "common/utils.h" +#include "engine/openmp.h" -namespace mxnet { -#if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE -static void SegfaultLogger(int sig) { - fprintf(stderr, "\nSegmentation fault: %d\n\n", sig); - fprintf(stderr, "%s", dmlc::StackTrace().c_str()); - abort(); -} -#endif +namespace mxnet { // pthread_atfork handlers, delegated to LibraryInitializer members. @@ -94,7 +94,6 @@ LibraryInitializer::LibraryInitializer() mp_cv_num_threads_(dmlc::GetEnv("MXNET_MP_OPENCV_NUM_THREADS", 0)) { dmlc::InitLogging("mxnet"); engine::OpenMP::Get(); // force OpenMP initialization - install_signal_handlers(); install_pthread_atfork_handlers(); } @@ -225,16 +224,159 @@ void LibraryInitializer::install_pthread_atfork_handlers() { #endif } -void LibraryInitializer::install_signal_handlers() { + + + #if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE - struct sigaction sa; - sigaction(SIGSEGV, nullptr, &sa); - if (sa.sa_handler == nullptr) { - signal(SIGSEGV, SegfaultLogger); + +static inline void printStackTrace(FILE *out = stderr, + const unsigned int max_frames = 63) { + +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__WINDOWS__) + // storage array for stack trace address data + void* addrlist[max_frames+1]; + + // retrieve current stack addresses + size_t addrlen = backtrace(addrlist, sizeof(addrlist)/sizeof(void*)); + + if (addrlen < 5) { + return; + } else { + addrlen = std::min(addrlen, dmlc::LogStackTraceLevel()); + } + fprintf(out, "Stack trace:\n"); + + + // resolve addresses into strings containing "filename(function+address)", + // Actually it will be ## program address function + offset + // this array must be free()-ed + char** symbollist = backtrace_symbols(addrlist, addrlen); + + size_t funcnamesize = 1024; + char funcname[1024]; + + // iterate over the returned symbol lines. skip the first, it is the + // address of this function. + for (unsigned int i = 4; i < addrlen ; i++) { + char* begin_name = nullptr; + char* begin_offset = nullptr; + char* end_offset = nullptr; + + // find parentheses and +address offset surrounding the mangled name +#ifdef DARWIN + // OSX style stack trace + for (char *p = symbollist[i]; *p; ++p) { + if (*p == '_' && *(p-1) == ' ') { + begin_name = p-1; + } else if (*p == '+') { + begin_offset = p-1; + } + } + + if (begin_name && begin_offset && begin_name < begin_offset) { + *begin_name++ = '\0'; + *begin_offset++ = '\0'; + + // mangled name is now in [begin_name, begin_offset) and caller + // offset in [begin_offset, end_offset). now apply + // __cxa_demangle(): + int status; + char* ret = abi::__cxa_demangle(begin_name, &funcname[0], + &funcnamesize, &status); + if (status == 0) { + funcname = ret; // use possibly realloc()-ed string + fprintf(out, " %-30s %-40s %s\n", + symbollist[i], funcname, begin_offset); + } else { + // demangling failed. Output function name as a C function with + // no arguments. + fprintf(out, " %-30s %-38s() %s\n", + symbollist[i], begin_name, begin_offset); + } + } else { + // couldn't parse the line? print the whole line. + fprintf(out, " %-40s\n", symbollist[i]); + } +#else + for (char *p = symbollist[i]; *p; ++p) { + if (*p == '(') { + begin_name = p; + } else if (*p == '+') { + begin_offset = p; + } else if (*p == ')' && (begin_offset || begin_name)) { + end_offset = p; + } + } + + if (begin_name && end_offset && begin_name < end_offset) { + *begin_name++ = '\0'; + *end_offset++ = '\0'; + if (begin_offset) { + *begin_offset++ = '\0'; + } + + // mangled name is now in [begin_name, begin_offset) and caller + // offset in [begin_offset, end_offset). now apply + // __cxa_demangle(): + + int status = 0; + char* ret = abi::__cxa_demangle(begin_name, funcname, + &funcnamesize, &status); + char* fname = begin_name; + if (status == 0) { + fname = ret; + } + + if (begin_offset) { + fprintf(out, " %-30s ( %-40s + %-6s) %s\n", + symbollist[i], fname, begin_offset, end_offset); + } else { + fprintf(out, " %-30s ( %-40s %-6s) %s\n", + symbollist[i], fname, "", end_offset); + } + } else { + // couldn't parse the line? print the whole line. + fprintf(out, " %-40s\n", symbollist[i]); + } +#endif // !DARWIN - but is posix } + free(symbollist); #endif } +#define SIGNAL_HANDLER(SIGNAL, HANDLER_NAME, IS_FATAL) \ +std::shared_ptr HANDLER_NAME( \ + signal(SIGNAL, [](int signum) { \ + if (IS_FATAL) { \ + printf("\nFatal Error: %s\n", strsignal(SIGNAL)); \ + printStackTrace(); \ + signal(signum, SIG_DFL); \ + raise(signum); \ + } else { \ + switch (signum) { \ + case SIGSEGV: \ + LOG(FATAL) << "InternalError: " << strsignal(SIGNAL); \ + break; \ + case SIGFPE: \ + LOG(FATAL) << "FloatingPointError: " << strsignal(SIGNAL); \ + break; \ + case SIGBUS: \ + LOG(FATAL) << "IOError: " << strsignal(SIGNAL); \ + break; \ + default: \ + LOG(FATAL) << "RuntimeError: " << strsignal(SIGNAL); \ + break; \ + } \ + } \ + }), \ + [](auto f) { signal(SIGNAL, f); }); + +SIGNAL_HANDLER(SIGSEGV, SIGSEGVHandler, true); +SIGNAL_HANDLER(SIGFPE, SIGFPEHandler, false); +SIGNAL_HANDLER(SIGBUS, SIGBUSHandler, false); + +#endif + void LibraryInitializer::close_open_libs() { for (const auto& l : loaded_libs) { lib_close(l.second); diff --git a/src/runtime/c_runtime_api.cc b/src/runtime/c_runtime_api.cc index d0754ea6293c..dbc3b6cd0c6e 100644 --- a/src/runtime/c_runtime_api.cc +++ b/src/runtime/c_runtime_api.cc @@ -24,7 +24,6 @@ // Acknowledgement: This file originates from incubator-tvm #include - #include #include #include @@ -42,8 +41,10 @@ using namespace mxnet::runtime; struct MXNetRuntimeEntry { std::string ret_str; MXNetByteArray ret_bytes; + std::string last_error; }; + typedef dmlc::ThreadLocalStore MXNetAPIRuntimeStore; int MXNetFuncFree(MXNetFunctionHandle func) { @@ -81,3 +82,183 @@ int MXNetFuncCall(MXNetFunctionHandle func, } API_END(); } + +#ifndef _LIBCPP_SGX_NO_IOSTREAMS +//-------------------------------------------------------- +// Error handling mechanism +// ------------------------------------------------------- +// Standard error message format, {} means optional +//-------------------------------------------------------- +// {error_type:} {message0} +// {message1} +// {message2} +// {Stack trace:} // stack traces follow by this line +// {trace 0} // two spaces in the begining. +// {trace 1} +// {trace 2} +//-------------------------------------------------------- +/*! + * \brief Normalize error message + * + * Parse them header generated by by LOG(FATAL) and CHECK + * and reformat the message into the standard format. + * + * This function will also merge all the stack traces into + * one trace and trim them. + * + * \param err_msg The error message. + * \return normalized message. + */ +std::string NormalizeError(std::string err_msg) { + // ------------------------------------------------------------------------ + // log with header, {} indicates optional + //------------------------------------------------------------------------- + // [timestamp] file_name:line_number: {check_msg:} {error_type:} {message0} + // {message1} + // Stack trace: + // {stack trace 0} + // {stack trace 1} + //------------------------------------------------------------------------- + // Normalzied version + //------------------------------------------------------------------------- + // error_type: check_msg message0 + // {message1} + // Stack trace: + // File file_name, line lineno + // {stack trace 0} + // {stack trace 1} + //------------------------------------------------------------------------- + int line_number = 0; + std::istringstream is(err_msg); + std::string line, file_name, error_type, check_msg; + + // Parse log header and set the fields, + // Return true if it the log is in correct format, + // return false if something is wrong. + auto parse_log_header = [&]() { + // skip timestamp + if (is.peek() != '[') { + getline(is, line); + return true; + } + if (!(is >> line)) return false; + // get filename + while (is.peek() == ' ') is.get(); + if (!getline(is, file_name, ':')) { + return false; + } else { + if (is.peek() == '\\' || is.peek() == '/') { + // windows path + if (!getline(is, line, ':')) return false; + file_name = file_name + ':' + line; + } + } + // get line number + if (!(is >> line_number)) return false; + // get rest of the message. + while (is.peek() == ' ' || is.peek() == ':') is.get(); + if (!getline(is, line)) return false; + // detect check message, rewrite to remote extra : + if (line.compare(0, 13, "Check failed:") == 0) { + size_t end_pos = line.find(':', 13); + if (end_pos == std::string::npos) return false; + check_msg = line.substr(0, end_pos + 1) + ' '; + line = line.substr(end_pos + 1); + } + return true; + }; + // if not in correct format, do not do any rewrite. + if (!parse_log_header()) return err_msg; + // Parse error type. + { + size_t start_pos = 0, end_pos; + for (; start_pos < line.length() && line[start_pos] == ' '; ++start_pos) {} + for (end_pos = start_pos; end_pos < line.length(); ++end_pos) { + char ch = line[end_pos]; + if (ch == ':') { + error_type = line.substr(start_pos, end_pos - start_pos); + break; + } + // [A-Z0-9a-z_.] + if (!std::isalpha(ch) && !std::isdigit(ch) && ch != '_' && ch != '.') break; + } + if (error_type.length() != 0) { + // if we successfully detected error_type: trim the following space. + for (start_pos = end_pos + 1; + start_pos < line.length() && line[start_pos] == ' '; ++start_pos) {} + line = line.substr(start_pos); + } else { + // did not detect error_type, use default value. + line = line.substr(start_pos); + error_type = "MXNetError"; + } + } + // Seperate out stack trace. + std::ostringstream os; + os << error_type << ": " << check_msg << line << '\n'; + + bool trace_mode = true; + std::vector stack_trace; + while (getline(is, line)) { + if (trace_mode) { + if (line.compare(0, 2, " ") == 0) { + stack_trace.push_back(line); + } else { + trace_mode = false; + // remove EOL trailing stacktrace. + if (line.length() == 0) continue; + } + } + if (!trace_mode) { + if (line.compare(0, 11, "Stack trace") == 0) { + trace_mode = true; + } else { + os << line << '\n'; + } + } + } + if (stack_trace.size() != 0 || file_name.length() != 0) { + os << "Stack trace:\n"; + if (file_name.length() != 0) { + os << " File \"" << file_name << "\", line " << line_number << "\n"; + } + // Print out stack traces, optionally trim the c++ traces + // about the frontends (as they will be provided by the frontends). + bool ffi_boundary = false; + for (const auto& line : stack_trace) { + // Heuristic to detect python ffi. + if (line.find("libffi.so") != std::string::npos || + line.find("core.cpython") != std::string::npos) { + ffi_boundary = true; + } + // If the backtrace is not c++ backtrace with the prefix " [bt]", + // then we can stop trimming. + if (ffi_boundary && line.compare(0, 6, " [bt]") != 0) { + ffi_boundary = false; + } + if (!ffi_boundary) { + os << line << '\n'; + } + } + } + return os.str(); +} + +#else +std::string NormalizeError(std::string err_msg) { + return err_msg; +} +#endif + +int MXAPIHandleException(const std::exception &e) { + MXAPISetLastError(NormalizeError(e.what()).c_str()); + return -1; +} + +const char *MXGetLastError() { + return MXNetAPIRuntimeStore::Get()->last_error.c_str(); +} + +void MXAPISetLastError(const char* msg) { + MXNetAPIRuntimeStore::Get()->last_error = msg; +}