From 445297429f10de094d5350b77bcb2c15bb374ff6 Mon Sep 17 00:00:00 2001 From: JZZ-NOTE Date: Wed, 7 Sep 2022 03:38:18 +0000 Subject: [PATCH 1/5] delay tensorrt registry --- .../ir_passes/tensorrt_subgraph_pass.cc | 6 ++++ .../inference/tensorrt/plugin/trt_plugin.h | 31 +++++++++++++++++-- paddle/fluid/platform/dynload/tensorrt.cc | 20 +++--------- paddle/phi/backends/dynload/tensorrt.cc | 19 +++--------- 4 files changed, 43 insertions(+), 33 deletions(-) diff --git a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc index cd79b3fcde0ef..60e0864a9be43 100644 --- a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc +++ b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc @@ -32,6 +32,7 @@ #include "paddle/fluid/inference/tensorrt/engine.h" #include "paddle/fluid/inference/tensorrt/helper.h" #include "paddle/fluid/inference/tensorrt/op_teller.h" +#include "paddle/fluid/inference/tensorrt/plugin/trt_plugin.h" #include "paddle/fluid/inference/utils/io_utils.h" #include "paddle/phi/common/backend.h" #include "paddle/phi/common/data_type.h" @@ -117,6 +118,11 @@ void analysis::TensorRtSubgraphPass::ApplyImpl( framework::ir::Graph *graph) const { framework::ir::FusePassBase::Init("tensorrt_subgraph_pass", graph); + static std::once_flag trt_plugin_registered; + std::call_once(trt_plugin_registered, []() { + tensorrt::plugin::TrtPluginRegistry::Global()->RegistToTrt(); + }); + auto model_precision = static_cast(Get("model_precision")); if (model_precision == phi::DataType::BFLOAT16) { diff --git a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h index 8c105230d27d4..d587bc104a04d 100644 --- a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h +++ b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h @@ -372,6 +372,26 @@ class TensorRTPluginCreator : public nvinfer1::IPluginCreator { std::vector plugin_attributes_; }; +class TrtPluginRegistry { + public: + static TrtPluginRegistry* Global() { + static TrtPluginRegistry registry; + return ®istry; + } + bool Regist(const std::string& name, std::function func) { + map.emplace(name, func); + return true; + } + void RegistToTrt() { + for (auto it : map) { + it.second(); + } + } + + private: + std::unordered_map> map; +}; + template class TrtPluginRegistrarV2 { public: @@ -386,9 +406,14 @@ class TrtPluginRegistrarV2 { T creator; }; -#define REGISTER_TRT_PLUGIN_V2(name) \ - static paddle::inference::tensorrt::plugin::TrtPluginRegistrarV2 \ - plugin_registrar_##name {} +#define REGISTER_TRT_PLUGIN_V2(name) REGISTER_TRT_PLUGIN_V2_HELPER(name) + +#define REGISTER_TRT_PLUGIN_V2_HELPER(name) \ + __attribute__((unused)) static bool REGISTER_TRT_PLUGIN_V2_HELPER##name = \ + TrtPluginRegistry::Global()->Regist(#name, []() -> void { \ + static paddle::inference::tensorrt::plugin::TrtPluginRegistrarV2 \ + plugin_registrar_##name{}; \ + }); } // namespace plugin } // namespace tensorrt diff --git a/paddle/fluid/platform/dynload/tensorrt.cc b/paddle/fluid/platform/dynload/tensorrt.cc index 8d700faac0c14..d3f3eeadee1eb 100644 --- a/paddle/fluid/platform/dynload/tensorrt.cc +++ b/paddle/fluid/platform/dynload/tensorrt.cc @@ -41,21 +41,11 @@ void* GetDsoHandle(const std::string& dso_name) { void* dso_handle = dlopen(dso_name.c_str(), dynload_flags); - if (nullptr == dso_handle) { - auto error_msg = - "You are using Paddle compiled with TensorRT, but TensorRT dynamic " - "library is not found. Ignore this if TensorRT is not needed.\n" - "The TensorRT that Paddle depends on is not configured correctly.\n" - " Suggestions:\n" - " 1. Check if the TensorRT is installed correctly and its version" - " is matched with paddlepaddle you installed.\n" - " 2. Configure environment variables as " - "follows:\n" - " - Linux: set LD_LIBRARY_PATH by `export LD_LIBRARY_PATH=...`\n" - " - Windows: set PATH by `set PATH=XXX;%PATH%`\n" - " - Mac: set DYLD_LIBRARY_PATH by `export DYLD_LIBRARY_PATH=...`\n"; - LOG(WARNING) << error_msg; - } + PADDLE_ENFORCE_NOT_NULL(dso_handle, + paddle::platform::errors::NotFound( + "TensorRT is needed, " + "but TensorRT dynamic library is not found.")); + return dso_handle; } diff --git a/paddle/phi/backends/dynload/tensorrt.cc b/paddle/phi/backends/dynload/tensorrt.cc index 4552570102025..2e2319a47cc54 100644 --- a/paddle/phi/backends/dynload/tensorrt.cc +++ b/paddle/phi/backends/dynload/tensorrt.cc @@ -40,21 +40,10 @@ void* GetDsoHandle(const std::string& dso_name) { void* dso_handle = dlopen(dso_name.c_str(), dynload_flags); - if (nullptr == dso_handle) { - auto error_msg = - "You are using Paddle compiled with TensorRT, but TensorRT dynamic " - "library is not found. Ignore this if TensorRT is not needed.\n" - "The TensorRT that Paddle depends on is not configured correctly.\n" - " Suggestions:\n" - " 1. Check if the TensorRT is installed correctly and its version" - " is matched with paddlepaddle you installed.\n" - " 2. Configure environment variables as " - "follows:\n" - " - Linux: set LD_LIBRARY_PATH by `export LD_LIBRARY_PATH=...`\n" - " - Windows: set PATH by `set PATH=XXX;%PATH%`\n" - " - Mac: set DYLD_LIBRARY_PATH by `export DYLD_LIBRARY_PATH=...`\n"; - LOG(WARNING) << error_msg; - } + PADDLE_ENFORCE_NOT_NULL(dso_handle, + paddle::platform::errors::NotFound( + "TensorRT is needed, " + "but TensorRT dynamic library is not found.")); return dso_handle; } From 586dacbb5e8bb2970070ede257dc7aac49a50fa6 Mon Sep 17 00:00:00 2001 From: JZZ-NOTE Date: Wed, 7 Sep 2022 16:05:05 +0000 Subject: [PATCH 2/5] add unused define --- paddle/fluid/inference/tensorrt/plugin/trt_plugin.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h index d587bc104a04d..9be153bd488e0 100644 --- a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h +++ b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h @@ -38,6 +38,13 @@ namespace inference { namespace tensorrt { namespace plugin { +#if defined(_WIN32) +#define UNUSED +#define __builtin_expect(EXP, C) (EXP) +#else +#define UNUSED __attribute__((unused)) +#endif + class PluginTensorRT; typedef std::function @@ -409,7 +416,7 @@ class TrtPluginRegistrarV2 { #define REGISTER_TRT_PLUGIN_V2(name) REGISTER_TRT_PLUGIN_V2_HELPER(name) #define REGISTER_TRT_PLUGIN_V2_HELPER(name) \ - __attribute__((unused)) static bool REGISTER_TRT_PLUGIN_V2_HELPER##name = \ + UNUSED static bool REGISTER_TRT_PLUGIN_V2_HELPER##name = \ TrtPluginRegistry::Global()->Regist(#name, []() -> void { \ static paddle::inference::tensorrt::plugin::TrtPluginRegistrarV2 \ plugin_registrar_##name{}; \ From c67679d5a3c0277827eb2e6208364b5f2aa8550a Mon Sep 17 00:00:00 2001 From: JZZ-NOTE Date: Mon, 12 Sep 2022 12:31:36 +0000 Subject: [PATCH 3/5] fix tensorrt test --- paddle/fluid/inference/tensorrt/test_dynamic_engine.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/fluid/inference/tensorrt/test_dynamic_engine.cc b/paddle/fluid/inference/tensorrt/test_dynamic_engine.cc index 6ac23e32856be..43e219232d111 100644 --- a/paddle/fluid/inference/tensorrt/test_dynamic_engine.cc +++ b/paddle/fluid/inference/tensorrt/test_dynamic_engine.cc @@ -284,6 +284,7 @@ class TensorRTDynamicTestFusedTokenPrune : public ::testing::Test { TEST_F(TensorRTDynamicTestFusedTokenPrune, test_fused_token_prune) { #if IS_TRT_VERSION_GE(8000) + tensorrt::plugin::TrtPluginRegistry::Global()->RegistToTrt(); auto *attn = engine_->DeclareInput( "attn", nvinfer1::DataType::kHALF, nvinfer1::Dims4{-1, 1, 4, 4}); auto *x = engine_->DeclareInput( From 98858608b26f0bb59c2d8303988f01d90ef51486 Mon Sep 17 00:00:00 2001 From: JZZ-NOTE Date: Tue, 13 Sep 2022 04:18:13 +0000 Subject: [PATCH 4/5] fix function to reference --- paddle/fluid/inference/tensorrt/plugin/trt_plugin.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h index 9be153bd488e0..f61293e06398a 100644 --- a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h +++ b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h @@ -385,12 +385,12 @@ class TrtPluginRegistry { static TrtPluginRegistry registry; return ®istry; } - bool Regist(const std::string& name, std::function func) { + bool Regist(const std::string& name, const std::function& func) { map.emplace(name, func); return true; } void RegistToTrt() { - for (auto it : map) { + for (const auto& it : map) { it.second(); } } From d4b9e0373898e8022579f5f7c10116199fecb3a4 Mon Sep 17 00:00:00 2001 From: JingZhuangzhuang <75348594+JZZ-NOTE@users.noreply.github.com> Date: Tue, 13 Sep 2022 18:55:32 +0800 Subject: [PATCH 5/5] Update trt_plugin.h --- paddle/fluid/inference/tensorrt/plugin/trt_plugin.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h index f61293e06398a..f08a8a75ba406 100644 --- a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h +++ b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h @@ -390,7 +390,7 @@ class TrtPluginRegistry { return true; } void RegistToTrt() { - for (const auto& it : map) { + for (auto& it : map) { it.second(); } }