From bfd3bb8972b7e4a9cd8487c4ab6e6583202f3259 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 9 Aug 2019 14:12:55 -0700 Subject: [PATCH] Refactor LibraryInitializer so it's thread safe. Fixes random sporadical concurrency crashes. (#15762) * Refactor LibraryInitializer so it's thread safe. Fixes #13438 Fixes #14979 * Refactor around lib loading * Fix lint * CR * Add option to choose between OMP implementations * Fix bug * Fix from CR --- CMakeLists.txt | 12 +- docs/faq/env_var.md | 10 +- src/c_api/c_api.cc | 4 +- src/common/library.cc | 125 ------------ src/common/library.h | 57 ------ src/common/utils.h | 12 ++ src/engine/threaded_engine_perdevice.cc | 4 +- src/initialize.cc | 247 +++++++++++++++++++----- src/initialize.h | 126 ++++++++++++ src/profiler/profiler.h | 15 +- 10 files changed, 359 insertions(+), 253 deletions(-) delete mode 100644 src/common/library.cc delete mode 100644 src/common/library.h create mode 100644 src/initialize.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 85f302f867e2..976c736f5f35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ mxnet_option(USE_OLDCMAKECUDA "Build with old cmake cuda" OFF) mxnet_option(USE_NCCL "Use NVidia NCCL with CUDA" OFF) mxnet_option(USE_OPENCV "Build with OpenCV support" ON) mxnet_option(USE_OPENMP "Build with Openmp support" ON) +mxnet_option(USE_OPENMP_BUNDLED_LLVM "Build with bundled llvm openmp from 3rdparty" OFF) mxnet_option(USE_CUDNN "Build with cudnn support" ON) # one could set CUDNN_ROOT for search path mxnet_option(USE_SSE "Build with x86 SSE instruction support" ON IF NOT ARM) mxnet_option(USE_F16C "Build with x86 F16C instruction support" ON) # autodetects support if ON @@ -433,11 +434,11 @@ if(USE_OPENMP) find_package(OpenMP REQUIRED) # This should build on Windows, but there's some problem and I don't have a Windows box, so # could a Windows user please fix? - if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt - AND SYSTEM_ARCHITECTURE STREQUAL "x86_64" - AND NOT MSVC - AND NOT CMAKE_CROSSCOMPILING) - + if(USE_OPENMP_BUNDLED_LLVM AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt + AND SYSTEM_ARCHITECTURE STREQUAL "x86_64" + AND NOT MSVC + AND NOT CMAKE_CROSSCOMPILING) + message("Using bundlded LLVM OpenMP") # Intel/llvm OpenMP: https://github.com/llvm-mirror/openmp set(OPENMP_STANDALONE_BUILD TRUE) set(LIBOMP_ENABLE_SHARED TRUE) @@ -451,6 +452,7 @@ if(USE_OPENMP) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") add_definitions(-DMXNET_USE_OPENMP=1) else() + message("Using platform provided OpenMP") if(OPENMP_FOUND) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") diff --git a/docs/faq/env_var.md b/docs/faq/env_var.md index 24d62f342983..b33a104fd315 100644 --- a/docs/faq/env_var.md +++ b/docs/faq/env_var.md @@ -39,6 +39,9 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 ## Set the Number of Threads +* MXNET_OMP_MAX_THREADS + - Values: Int ```(default=Number of processors / Number of processors * 2 in X86)``` + - Maximum number of threads to use in individual operators through OpenMP. If not set, OMP_NUM_THREADS is considered after. * MXNET_GPU_WORKER_NTHREADS - Values: Int ```(default=2)``` - The maximum number of threads to use on each GPU. This parameter is used to parallelize the computation within a single GPU card. @@ -47,7 +50,7 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - The maximum number of concurrent threads that do the memory copy job on each GPU. * MXNET_CPU_WORKER_NTHREADS - Values: Int ```(default=1)``` - - The maximum number of scheduling threads on CPU. It specifies how many operators can be run in parallel. Note that most CPU operators are parallelized by OpenMP. To change the number of threads used by individual operators, please set `OMP_NUM_THREADS` instead. + - The maximum number of scheduling threads on CPU. It specifies how many operators can be run in parallel. Note that most CPU operators are parallelized by OpenMP. To change the number of threads used by individual operators, please set `MXNET_OMP_MAX_THREADS` instead. * MXNET_CPU_PRIORITY_NTHREADS - Values: Int ```(default=4)``` - The number of threads given to prioritized CPU jobs. @@ -56,10 +59,13 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - The number of threads used for NNPACK. NNPACK package aims to provide high-performance implementations of some layers for multi-core CPUs. Checkout [NNPACK](http://mxnet.io/faq/nnpack.html) to know more about it. * MXNET_MP_WORKER_NTHREADS - Values: Int ```(default=1)``` - - The number of scheduling threads on CPU given to multiprocess workers. Enlarge this number allows more operators to run in parallel in individual workers but please consider reducing the overall `num_workers` to avoid thread contention (not available on Windows). + - The number of scheduling threads on CPU given to multiprocess workers (after fork). Enlarge this number allows more operators to run in parallel in individual workers but please consider reducing the overall `num_workers` to avoid thread contention (not available on Windows). * MXNET_MP_OPENCV_NUM_THREADS - Values: Int ```(default=0)``` - The number of OpenCV execution threads given to multiprocess workers. OpenCV multithreading is disabled if `MXNET_MP_OPENCV_NUM_THREADS` < 1 (default). Enlarge this number may boost the performance of individual workers when executing underlying OpenCV functions but please consider reducing the overall `num_workers` to avoid thread contention (not available on Windows). +* MXNET_GPU_COPY_NTHREADS + - Values:: Int ```(default=2)``` + - Number of threads for copying data from CPU to GPU. ## Memory Options diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 13f22191955d..17648458ba22 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -46,12 +46,12 @@ #include "mxnet/libinfo.h" #include "mxnet/imperative.h" #include "mxnet/lib_api.h" +#include "../initialize.h" #include "./c_api_common.h" #include "../operator/custom/custom-inl.h" #include "../operator/tensor/matrix_op-inl.h" #include "../operator/tvmop/op_module.h" #include "../common/utils.h" -#include "../common/library.h" using namespace mxnet; @@ -95,7 +95,7 @@ inline int MXAPIGetFunctionRegInfo(const FunRegType *e, // Loads library and initializes it int MXLoadLib(const char *path) { API_BEGIN(); - void *lib = load_lib(path); + void *lib = LibraryInitializer::Get()->lib_load(path); if (!lib) LOG(FATAL) << "Unable to load library"; diff --git a/src/common/library.cc b/src/common/library.cc deleted file mode 100644 index 9e79b5dbe1bc..000000000000 --- a/src/common/library.cc +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/*! - * Copyright (c) 2015 by Contributors - * \file library.cc - * \brief Dynamically loading accelerator library - * and accessing its functions - */ - -#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) -#include -#else -#include -#endif - -#include -#include "library.h" - -#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) -/*! - * \brief Retrieve the system error message for the last-error code - * \param err string that gets the error message - */ -void win_err(char **err) { - uint32_t dw = GetLastError(); - FormatMessage( - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - dw, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - reinterpret_cast(err), - 0, NULL); -} -#endif - - -/*! - * \brief Loads the dynamic shared library file - * \param path library file location - * \return handle a pointer for the loaded library, nullptr if loading unsuccessful - */ -void* load_lib(const char* path) { - void *handle = nullptr; - std::string path_str(path); - // check if library was already loaded - if (loaded_libs.find(path_str) == loaded_libs.end()) { - // if not, load it -#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) - handle = LoadLibrary(path); - if (!handle) { - char *err_msg = nullptr; - win_err(&err_msg); - LOG(FATAL) << "Error loading library: '" << path << "'\n" << err_msg; - LocalFree(err_msg); - return nullptr; - } -#else - handle = dlopen(path, RTLD_LAZY); - if (!handle) { - LOG(FATAL) << "Error loading library: '" << path << "'\n" << dlerror(); - return nullptr; - } -#endif // _WIN32 or _WIN64 or __WINDOWS__ - // then store the pointer to the library - loaded_libs[path_str] = handle; - } else { - // otherwise just look up the pointer - handle = loaded_libs[path_str]; - } - return handle; -} - -/*! - * \brief Closes the loaded dynamic shared library file - * \param handle library file handle - */ -void close_lib(void* handle) { -#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) - FreeLibrary((HMODULE)handle); -#else - dlclose(handle); -#endif // _WIN32 or _WIN64 or __WINDOWS__ -} - -/*! - * \brief Obtains address of given function in the loaded library - * \param handle pointer for the loaded library - * \param func function pointer that gets output address - * \param name function name to be fetched - */ -void get_sym(void* handle, void** func, char* name) { -#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) - *func = GetProcAddress((HMODULE)handle, name); - if (!(*func)) { - char *err_msg = nullptr; - win_err(&err_msg); - LOG(FATAL) << "Error getting function '" << name << "' from library\n" << err_msg; - LocalFree(err_msg); - } -#else - *func = dlsym(handle, name); - if (!(*func)) { - LOG(FATAL) << "Error getting function '" << name << "' from library\n" << dlerror(); - } -#endif // _WIN32 or _WIN64 or __WINDOWS__ -} diff --git a/src/common/library.h b/src/common/library.h deleted file mode 100644 index d6eff4184191..000000000000 --- a/src/common/library.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/*! - * Copyright (c) 2015 by Contributors - * \file library.h - * \brief Defining library loading functions - */ -#ifndef MXNET_COMMON_LIBRARY_H_ -#define MXNET_COMMON_LIBRARY_H_ - -#include -#include -#include -#include "dmlc/io.h" - -// map of libraries loaded -static std::map loaded_libs; - -void* load_lib(const char* path); -void close_lib(void* handle); -void get_sym(void* handle, void** func, char* name); - -/*! - * \brief a templated function that fetches from the library - * a function pointer of any given datatype and name - * \param T a template parameter for data type of function pointer - * \param lib library handle - * \param func_name function name to search for in the library - * \return func a function pointer - */ -template -T get_func(void *lib, char *func_name) { - T func; - get_sym(lib, reinterpret_cast(&func), func_name); - if (!func) - LOG(FATAL) << "Unable to get function '" << func_name << "' from library"; - return func; -} - -#endif // MXNET_COMMON_LIBRARY_H_ diff --git a/src/common/utils.h b/src/common/utils.h index 251a8fe3c190..9dad5df84fd2 100644 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -50,9 +50,21 @@ #include "../operator/nn/mkldnn/mkldnn_base-inl.h" #endif +#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) +#include +#else +#include +#endif + + namespace mxnet { namespace common { +#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) +inline size_t current_process_id() { return ::GetCurrentProcessId(); } +#else +inline size_t current_process_id() { return getpid(); } +#endif /*! * \brief IndPtr should be non-negative, in non-decreasing order, start with 0 * and end with value equal with size of indices. diff --git a/src/engine/threaded_engine_perdevice.cc b/src/engine/threaded_engine_perdevice.cc index bcb101e9e1bb..2184d784a414 100644 --- a/src/engine/threaded_engine_perdevice.cc +++ b/src/engine/threaded_engine_perdevice.cc @@ -28,6 +28,7 @@ #include #include #include +#include "../initialize.h" #include "./threaded_engine.h" #include "./thread_pool.h" #include "../common/lazy_alloc_array.h" @@ -76,7 +77,8 @@ class ThreadedEnginePerDevice : public ThreadedEngine { void Start() override { if (is_worker_) return; gpu_worker_nthreads_ = common::GetNumThreadsPerGPU(); - cpu_worker_nthreads_ = dmlc::GetEnv("MXNET_CPU_WORKER_NTHREADS", 1); + // MXNET_CPU_WORKER_NTHREADS + cpu_worker_nthreads_ = LibraryInitializer::Get()->cpu_worker_nthreads_; gpu_copy_nthreads_ = dmlc::GetEnv("MXNET_GPU_COPY_NTHREADS", 2); // create CPU task int cpu_priority_nthreads = dmlc::GetEnv("MXNET_CPU_PRIORITY_NTHREADS", 4); diff --git a/src/initialize.cc b/src/initialize.cc index 04952be7072a..071e8d32e548 100644 --- a/src/initialize.cc +++ b/src/initialize.cc @@ -1,4 +1,4 @@ -/* + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -22,17 +22,43 @@ * \file initialize.cc * \brief initialize mxnet library */ +#include "initialize.h" #include #include #include #include "./engine/openmp.h" #include "./operator/custom/custom-inl.h" -#include "./common/library.h" #if MXNET_USE_OPENCV #include #endif // MXNET_USE_OPENCV +#include "common/utils.h" +#include "engine/openmp.h" + + +#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) +#include +/*! + * \brief Retrieve the system error message for the last-error code + * \param err string that gets the error message + */ +void win_err(char **err) { + uint32_t dw = GetLastError(); + FormatMessage( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + dw, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + reinterpret_cast(err), + 0, NULL); +} +#else +#include +#endif namespace mxnet { + #if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE static void SegfaultLogger(int sig) { fprintf(stderr, "\nSegmentation fault: %d\n\n", sig); @@ -41,65 +67,188 @@ static void SegfaultLogger(int sig) { } #endif -class LibraryInitializer { - public: - LibraryInitializer() { - dmlc::InitLogging("mxnet"); -#if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE - struct sigaction sa; - sigaction(SIGSEGV, nullptr, &sa); - if (sa.sa_handler == nullptr) { - signal(SIGSEGV, SegfaultLogger); +// pthread_atfork handlers, delegated to LibraryInitializer members. + +void pthread_atfork_prepare() { + LibraryInitializer* library_initializer = LibraryInitializer::Get(); + library_initializer->atfork_prepare(); +} + +void pthread_atfork_parent() { + LibraryInitializer* library_initializer = LibraryInitializer::Get(); + library_initializer->atfork_parent(); +} + +void pthread_atfork_child() { + LibraryInitializer* library_initializer = LibraryInitializer::Get(); + library_initializer->atfork_child(); +} + +// LibraryInitializer member functions + +LibraryInitializer::LibraryInitializer() + : original_pid_(common::current_process_id()), + mp_worker_nthreads_(dmlc::GetEnv("MXNET_MP_WORKER_NTHREADS", 1)), + cpu_worker_nthreads_(dmlc::GetEnv("MXNET_CPU_WORKER_NTHREADS", 1)), + mp_cv_num_threads_(dmlc::GetEnv("MXNET_MP_OPENCV_NUM_THREADS", 0)) { + dmlc::InitLogging("mxnet"); + engine::OpenMP::Get(); // force OpenMP initialization + install_signal_handlers(); + install_pthread_atfork_handlers(); +} + +LibraryInitializer::~LibraryInitializer() { + close_open_libs(); +} + +bool LibraryInitializer::lib_is_loaded(const std::string& path) const { + return loaded_libs.count(path) > 0; +} + +/*! + * \brief Loads the dynamic shared library file + * \param path library file location + * \return handle a pointer for the loaded library, throws dmlc::error if library can't be loaded + */ +void* LibraryInitializer::lib_load(const char* path) { + void *handle = nullptr; + // check if library was already loaded + if (!lib_is_loaded(path)) { + // if not, load it +#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) + handle = LoadLibrary(path); + if (!handle) { + char *err_msg = nullptr; + win_err(&err_msg); + LOG(FATAL) << "Error loading library: '" << path << "'\n" << err_msg; + LocalFree(err_msg); + return nullptr; } -#endif +#else + handle = dlopen(path, RTLD_LAZY); + if (!handle) { + LOG(FATAL) << "Error loading library: '" << path << "'\n" << dlerror(); + return nullptr; + } +#endif // _WIN32 or _WIN64 or __WINDOWS__ + // then store the pointer to the library + loaded_libs[path] = handle; + } else { + handle = loaded_libs.at(path); + } + return handle; +} -// disable openmp for multithreaded workers -#ifndef _WIN32 - using op::custom::CustomOperator; - pthread_atfork( - []() { - CustomOperator::Get()->Stop(); - Engine::Get()->Stop(); - }, - []() { - Engine::Get()->Start(); - CustomOperator::Get()->Start(); - }, - []() { - // Conservative thread management for multiprocess workers - const size_t mp_worker_threads = dmlc::GetEnv("MXNET_MP_WORKER_NTHREADS", 1); - dmlc::SetEnv("MXNET_CPU_WORKER_NTHREADS", mp_worker_threads); - dmlc::SetEnv("OMP_NUM_THREADS", 1); +/*! + * \brief Closes the loaded dynamic shared library file + * \param handle library file handle + */ +void LibraryInitializer::lib_close(void* handle) { + std::string libpath; + for (const auto& l : loaded_libs) { + if (l.second == handle) { + libpath = l.first; + break; + } + } + CHECK(!libpath.empty()); +#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) + FreeLibrary((HMODULE)handle); +#else + if (dlclose(handle)) { + LOG(WARNING) << "LibraryInitializer::lib_close: couldn't close library at address: " << handle + << " loaded from: '" << libpath << "': " << dlerror(); + } +#endif // _WIN32 or _WIN64 or __WINDOWS__ + loaded_libs.erase(libpath); +} + +/*! + * \brief Obtains address of given function in the loaded library + * \param handle pointer for the loaded library + * \param func function pointer that gets output address + * \param name function name to be fetched + */ +void LibraryInitializer::get_sym(void* handle, void** func, char* name) { +#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) + *func = GetProcAddress((HMODULE)handle, name); + if (!(*func)) { + char *err_msg = nullptr; + win_err(&err_msg); + LOG(FATAL) << "Error getting function '" << name << "' from library\n" << err_msg; + LocalFree(err_msg); + } +#else + *func = dlsym(handle, name); + if (!(*func)) { + LOG(FATAL) << "Error getting function '" << name << "' from library\n" << dlerror(); + } +#endif // _WIN32 or _WIN64 or __WINDOWS__ +} + +bool LibraryInitializer::was_forked() const { + return common::current_process_id() != original_pid_; +} + +void LibraryInitializer::atfork_prepare() { + using op::custom::CustomOperator; + CustomOperator::Get()->Stop(); + Engine::Get()->Stop(); +} + +void LibraryInitializer::atfork_parent() { + using op::custom::CustomOperator; + Engine::Get()->Start(); + CustomOperator::Get()->Start(); +} + +void LibraryInitializer::atfork_child() { + using op::custom::CustomOperator; + // Conservative thread management for multiprocess workers + this->cpu_worker_nthreads_ = this->mp_worker_nthreads_; #if MXNET_USE_OPENCV && !__APPLE__ - const size_t mp_cv_num_threads = dmlc::GetEnv("MXNET_MP_OPENCV_NUM_THREADS", 0); - cv::setNumThreads(mp_cv_num_threads); // disable opencv threading + cv::setNumThreads(mp_cv_num_threads_); #endif // MXNET_USE_OPENCV - engine::OpenMP::Get()->set_enabled(false); - Engine::Get()->Start(); - CustomOperator::Get()->Start(); - }); + engine::OpenMP::Get()->set_thread_max(1); + engine::OpenMP::Get()->set_enabled(false); + Engine::Get()->Start(); + CustomOperator::Get()->Start(); +} + + +void LibraryInitializer::install_pthread_atfork_handlers() { +#ifndef _WIN32 + pthread_atfork(pthread_atfork_prepare, pthread_atfork_parent, pthread_atfork_child); #endif - } +} - ~LibraryInitializer() { - // close opened libraries - for (auto const& lib : loaded_libs) { - close_lib(lib.second); - } +void LibraryInitializer::install_signal_handlers() { +#if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE + struct sigaction sa; + sigaction(SIGSEGV, nullptr, &sa); + if (sa.sa_handler == nullptr) { + signal(SIGSEGV, SegfaultLogger); } +#endif +} - static LibraryInitializer* Get(); -}; - -LibraryInitializer* LibraryInitializer::Get() { - static LibraryInitializer inst; - return &inst; +void LibraryInitializer::close_open_libs() { + for (const auto& l : loaded_libs) { + lib_close(l.second); + } } +/** + * Perform static initialization + */ #ifdef __GNUC__ -// Don't print an unused variable message since this is intentional +// In GCC we use constructor to perform initialization before any static initializer is able to run +__attribute__((constructor)) static void LibraryInitializerEntry() { #pragma GCC diagnostic ignored "-Wunused-variable" + volatile LibraryInitializer* library_init = LibraryInitializer::Get(); +} +#else +static LibraryInitializer* __library_init = LibraryInitializer::Get(); #endif -static LibraryInitializer* __library_init = LibraryInitializer::Get(); } // namespace mxnet diff --git a/src/initialize.h b/src/initialize.h new file mode 100644 index 000000000000..8a6dc3aa5f7f --- /dev/null +++ b/src/initialize.h @@ -0,0 +1,126 @@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2019 by Contributors + * \file initialize.h + * \brief Library initialization + */ + +#include +#include +#include +#include "dmlc/io.h" + + +#ifndef MXNET_INITIALIZE_H_ +#define MXNET_INITIALIZE_H_ + +namespace mxnet { + + + +void pthread_atfork_prepare(); +void pthread_atfork_parent(); +void pthread_atfork_child(); + +/** + * Perform library initialization and control multiprocessing behaviour. + */ +class LibraryInitializer { + public: + typedef std::map loaded_libs_t; + static LibraryInitializer* Get() { + static LibraryInitializer inst; + return &inst; + } + + /** + * Library initialization. Called on library loading via constructor attributes or + * C++ static initialization. + */ + LibraryInitializer(); + + ~LibraryInitializer(); + + /** + * @return true if the current pid doesn't match the one that initialized the library + */ + bool was_forked() const; + + + // Library loading + bool lib_is_loaded(const std::string& path) const; + void* lib_load(const char* path); + void lib_close(void* handle); + static void get_sym(void* handle, void** func, char* name); + + /** + * Original pid of the process which first loaded and initialized the library + */ + size_t original_pid_; + size_t mp_worker_nthreads_; + size_t cpu_worker_nthreads_; + size_t omp_num_threads_; + size_t mp_cv_num_threads_; + + // Actual code for the atfork handlers as member functions. + void atfork_prepare(); + void atfork_parent(); + void atfork_child(); + + private: + /** + * Pthread atfork handlers are used to reset the concurrency state of modules like CustomOperator + * and Engine when forking. When forking only the thread that forks is kept alive and memory is + * copied to the new process so state is inconsistent. This call install the handlers. + * Has no effect on Windows. + * + * https://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_atfork.html + */ + void install_pthread_atfork_handlers(); + + /** + * Install signal handlers (UNIX). Has no effect on Windows. + */ + void install_signal_handlers(); + + void close_open_libs(); + + loaded_libs_t loaded_libs; +}; + +/*! + * \brief fetches from the library a function pointer of any given datatype and name + * \param T a template parameter for data type of function pointer + * \param lib library handle + * \param func_name function name to search for in the library + * \return func a function pointer + */ +template +T get_func(void *lib, char *func_name) { + T func; + LibraryInitializer::Get()->get_sym(lib, reinterpret_cast(&func), func_name); + if (!func) + LOG(FATAL) << "Unable to get function '" << func_name << "' from library"; + return func; +} + +} // namespace mxnet +#endif // MXNET_INITIALIZE_H_ diff --git a/src/profiler/profiler.h b/src/profiler/profiler.h index 6df7ce4339da..f9f997fe8bb8 100644 --- a/src/profiler/profiler.h +++ b/src/profiler/profiler.h @@ -36,22 +36,13 @@ #include "./vtune.h" #include "./aggregate_stats.h" #include "./nvtx.h" +#include "../common/utils.h" -#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) -#include -#else -#include -#include -#endif namespace mxnet { namespace profiler { -#if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__) -inline size_t current_process_id() { return ::GetCurrentProcessId(); } -#else -inline size_t current_process_id() { return getpid(); } -#endif + /*! * \brief Constant-sized character array class with simple string API to avoid allocations @@ -132,7 +123,7 @@ struct ProfileStat { bool enable_aggregate_ = true; /* !\brief Process id */ - size_t process_id_ = current_process_id(); + size_t process_id_ = common::current_process_id(); /*! \brief id of thread which operation run on. *