Adding support for building BERT plugins with GPU_ARCHS specified (#255)

NVIDIA · Dec 5, 2019 · 98ab844 · 98ab844
1 parent fb305de
commit 98ab844
Show file tree

Hide file tree

Showing 33 changed files with 102 additions and 93 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -77,6 +77,33 @@ endif()
 
 set(CMAKE_CXX_FLAGS "-Wno-deprecated-declarations ${CMAKE_CXX_FLAGS} -DBUILD_SYSTEM=cmake_oss")
 
+if (DEFINED GPU_ARCHS)
+  message(STATUS "GPU_ARCHS defined as ${GPU_ARCHS}. Generating CUDA code for SM ${GPU_ARCHS}")
+  separate_arguments(GPU_ARCHS)
+else()
+  list(APPEND GPU_ARCHS
+      35
+      53
+      61
+      70
+      75
+    )
+  message(STATUS "GPU_ARCHS is not defined. Generating CUDA code for default SMs: ${GPU_ARCHS}")
+endif()
+set(BERT_GENCODES)
+# Generate SASS for each architecture
+foreach(arch ${GPU_ARCHS})
+    if (${arch} GREATER_EQUAL 70)
+        set(BERT_GENCODES "${BERT_GENCODES} -gencode arch=compute_${arch},code=sm_${arch}")
+    endif()
+    set(GENCODES "${GENCODES} -gencode arch=compute_${arch},code=sm_${arch}")
+endforeach()
+# Generate PTX for the last architecture in the list.
+list(GET GPU_ARCHS -1 LATEST_SM)
+set(GENCODES "${GENCODES} -gencode arch=compute_${LATEST_SM},code=compute_${LATEST_SM}")
+if (${LATEST_SM} GREATER_EQUAL 70)
+    set(BERT_GENCODES "${BERT_GENCODES} -gencode arch=compute_${LATEST_SM},code=compute_${LATEST_SM}")
+endif()
 set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler -Wno-deprecated-declarations")
 
 ################################### DEPENDENCIES ##########################################

diff --git a/README.md b/README.md
@@ -174,11 +174,12 @@ NOTE: Along with the TensorRT OSS components, the following source packages will
 
 	- `CUB_VERSION`: The version of CUB to use, for example [`1.8.0`].
 
-	- `GPU_ARCHS`: GPU (SM) architectures to target. By default we generate CUDA code for the latest SM version. If lower SM versions are desired, they can be specified here as a comma separated list. Table of compute capabilities of NVIDIA GPUs can be found [here](https://developer.nvidia.com/cuda-gpus). Examples:
+	- `GPU_ARCHS`: GPU (SM) architectures to target. By default we generate CUDA code for all major SMs. Specific SM versions can be specified here as a quoted space-separated list to reduce compilation time and binary size. Table of compute capabilities of NVIDIA GPUs can be found [here](https://developer.nvidia.com/cuda-gpus). Examples:
 	  - Titan V: `-DGPU_ARCHS="70"`
 	  - Tesla V100: `-DGPU_ARCHS="70"`
 	  - GeForce RTX 2080: `-DGPU_ARCHS="75"`
 	  - Tesla T4: `-DGPU_ARCHS="75"`
+	  - Multiple SMs: `-DGPU_ARCHS="70 75"`
 
 ## Install the TensorRT OSS Components [Optional]
 

diff --git a/plugin/CMakeLists.txt b/plugin/CMakeLists.txt
@@ -26,17 +26,10 @@ if(${CMAKE_BUILD_TYPE} MATCHES "Debug")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
 endif()
 
-set(PLUGIN_SRCS)
 set(PLUGIN_SOURCES)
-set(CUDA_SRCS)
-set(COMMON_SRCS)
+set(PLUGIN_CU_SOURCES)
 
 set(PLUGIN_LISTS
-    embLayerNormPlugin
-    fcPlugin
-    geluPlugin
-    bertQKVToContextPlugin
-    skipLayerNormPlugin
     nmsPlugin
     normalizePlugin
     priorBoxPlugin
@@ -57,6 +50,19 @@ set(PLUGIN_LISTS
     instanceNormalizationPlugin
     )
 
+# Add BERT sources if ${BERT_GENCODES} was populated
+if(BERT_GENCODES)
+    set(BERT_CU_SOURCES)
+    set(PLUGIN_LISTS
+        ${PLUGIN_LISTS}
+        embLayerNormPlugin
+        fcPlugin
+        geluPlugin
+        bertQKVToContextPlugin
+        skipLayerNormPlugin
+        )
+endif()
+
 include_directories(common common/kernels ../samples/common)
 
 foreach(PLUGIN_ITER ${PLUGIN_LISTS})
@@ -67,6 +73,14 @@ endforeach(PLUGIN_ITER)
 # Add common
 add_subdirectory(common)
 
+# Set gencodes
+set_source_files_properties(${PLUGIN_CU_SOURCES} PROPERTIES COMPILE_FLAGS ${GENCODES})
+list(APPEND PLUGIN_SOURCES "${PLUGIN_CU_SOURCES}")
+if (BERT_CU_SOURCES)
+    set_source_files_properties(${BERT_CU_SOURCES} PROPERTIES COMPILE_FLAGS ${BERT_GENCODES})
+    list(APPEND PLUGIN_SOURCES "${BERT_CU_SOURCES}")
+endif()
+
 list(APPEND PLUGIN_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/InferPlugin.cpp")
 list(APPEND PLUGIN_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/../samples/common/logger.cpp")
 

diff --git a/plugin/batchTilePlugin/CMakeLists.txt b/plugin/batchTilePlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/batchedNMSPlugin/CMakeLists.txt b/plugin/batchedNMSPlugin/CMakeLists.txt
@@ -13,8 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
+file(GLOB CU_SRCS *.cu)
+set(PLUGIN_CU_SOURCES ${PLUGIN_CU_SOURCES} ${CU_SRCS})
+set(PLUGIN_CU_SOURCES ${PLUGIN_CU_SOURCES} PARENT_SCOPE)
 
 
diff --git a/plugin/bertQKVToContextPlugin/CMakeLists.txt b/plugin/bertQKVToContextPlugin/CMakeLists.txt
@@ -13,15 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-string(FIND ${CMAKE_CUDA_FLAGS} "sm_7" POS_SM)
-string(FIND ${CMAKE_CUDA_FLAGS} "compute_7" POS_COMPUTE)
-
-if(${POS_SM} GREATER_EQUAL 0 OR ${POS_COMPUTE} GREATER_EQUAL 0)
-    file(GLOB SRCS *.cpp *.cu)
-
-    set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
-    set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
-endif()
-
-
+file(GLOB CU_SRCS *.cu)
+set(BERT_CU_SOURCES ${BERT_CU_SOURCES} ${CU_SRCS})
+set(BERT_CU_SOURCES ${BERT_CU_SOURCES} PARENT_SCOPE)
diff --git a/plugin/bertQKVToContextPlugin/README.md b/plugin/bertQKVToContextPlugin/README.md
@@ -72,4 +72,4 @@ This is the first release of this `README.md` file.
 
 ## Known issues
 
-There are no known issues in this plugin.
+This plugin only supports GPUs with compute capability >= 7.0. For more information see the [CUDA GPU Compute Capability Support Matrix](https://developer.nvidia.com/cuda-gpus#compute)
diff --git a/plugin/common/CMakeLists.txt b/plugin/common/CMakeLists.txt
@@ -13,10 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
-
 add_subdirectory(kernels)
-
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
+file(GLOB CU_SRCS *.cu)
+set(PLUGIN_CU_SOURCES ${PLUGIN_CU_SOURCES} ${CU_SRCS})
+set(PLUGIN_CU_SOURCES ${PLUGIN_CU_SOURCES} PARENT_SCOPE)
+
 
diff --git a/plugin/common/bertCommon.h b/plugin/common/bertCommon.h
@@ -35,6 +35,8 @@ extern LogStreamConsumer gLogFatal;
 
 void setReportableSeverity(Logger::Severity severity);
 
+#define TRT_UNUSED (void)
+
 #include <numeric>
 #include <vector>
 

diff --git a/plugin/common/kernels/CMakeLists.txt b/plugin/common/kernels/CMakeLists.txt
@@ -13,6 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
+file(GLOB CU_SRCS *.cu)
+set(PLUGIN_CU_SOURCES ${PLUGIN_CU_SOURCES} ${CU_SRCS})
+set(PLUGIN_CU_SOURCES ${PLUGIN_CU_SOURCES} PARENT_SCOPE)
diff --git a/plugin/cropAndResizePlugin/CMakeLists.txt b/plugin/cropAndResizePlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/detectionLayerPlugin/CMakeLists.txt b/plugin/detectionLayerPlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/embLayerNormPlugin/CMakeLists.txt b/plugin/embLayerNormPlugin/CMakeLists.txt
@@ -13,15 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-string(FIND ${CMAKE_CUDA_FLAGS} "sm_7" POS_SM)
-string(FIND ${CMAKE_CUDA_FLAGS} "compute_7" POS_COMPUTE)
-
-if(${POS_SM} GREATER_EQUAL 0 OR ${POS_COMPUTE} GREATER_EQUAL 0)
-    file(GLOB SRCS *.cpp *.cu)
-
-    set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
-    set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
-endif()
+file(GLOB CU_SRCS *.cu)
+set(BERT_CU_SOURCES ${BERT_CU_SOURCES} ${CU_SRCS})
+set(BERT_CU_SOURCES ${BERT_CU_SOURCES} PARENT_SCOPE)
 
 
diff --git a/plugin/embLayerNormPlugin/README.md b/plugin/embLayerNormPlugin/README.md
@@ -92,4 +92,4 @@ This is the first release of this `README.md` file.
 
 ## Known issues
 
-There are no known issues in this plugin.
+This plugin only supports GPUs with compute capability >= 7.0. For more information see the [CUDA GPU Compute Capability Support Matrix](https://developer.nvidia.com/cuda-gpus#compute)
diff --git a/plugin/fcPlugin/CMakeLists.txt b/plugin/fcPlugin/CMakeLists.txt
@@ -13,15 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-string(FIND ${CMAKE_CUDA_FLAGS} "sm_7" POS_SM)
-string(FIND ${CMAKE_CUDA_FLAGS} "compute_7" POS_COMPUTE)
-
-if(${POS_SM} GREATER_EQUAL 0 OR ${POS_COMPUTE} GREATER_EQUAL 0)
-    file(GLOB SRCS *.cpp *.cu)
-
-    set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
-    set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
-endif()
-
-
+file(GLOB CU_SRCS *.cu)
+set(BERT_CU_SOURCES ${BERT_CU_SOURCES} ${CU_SRCS})
+set(BERT_CU_SOURCES ${BERT_CU_SOURCES} PARENT_SCOPE)
diff --git a/plugin/fcPlugin/README.md b/plugin/fcPlugin/README.md
@@ -57,4 +57,4 @@ This is the first release of this `README.md` file.
 
 ## Known issues
 
-There are no known issues in this plugin.
+This plugin only supports GPUs with compute capability >= 7.0. For more information see the [CUDA GPU Compute Capability Support Matrix](https://developer.nvidia.com/cuda-gpus#compute)
diff --git a/plugin/flattenConcat/CMakeLists.txt b/plugin/flattenConcat/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/geluPlugin/CMakeLists.txt b/plugin/geluPlugin/CMakeLists.txt
@@ -13,15 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-string(FIND ${CMAKE_CUDA_FLAGS} "sm_7" POS_SM)
-string(FIND ${CMAKE_CUDA_FLAGS} "compute_7" POS_COMPUTE)
-
-if(${POS_SM} GREATER_EQUAL 0 OR ${POS_COMPUTE} GREATER_EQUAL 0)
-    file(GLOB SRCS *.cpp *.cu)
-
-    set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
-    set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
-endif()
-
-
+file(GLOB CU_SRCS *.cu)
+set(BERT_CU_SOURCES ${BERT_CU_SOURCES} ${CU_SRCS})
+set(BERT_CU_SOURCES ${BERT_CU_SOURCES} PARENT_SCOPE)
diff --git a/plugin/geluPlugin/README.md b/plugin/geluPlugin/README.md
@@ -61,4 +61,4 @@ This is the first release of this `README.md` file.
 
 ## Known issues
 
-There are no known issues in this plugin.
+This plugin only supports GPUs with compute capability >= 7.0. For more information see the [CUDA GPU Compute Capability Support Matrix](https://developer.nvidia.com/cuda-gpus#compute)
diff --git a/plugin/gridAnchorPlugin/CMakeLists.txt b/plugin/gridAnchorPlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/nmsPlugin/CMakeLists.txt b/plugin/nmsPlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/normalizePlugin/CMakeLists.txt b/plugin/normalizePlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/nvFasterRCNN/CMakeLists.txt b/plugin/nvFasterRCNN/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/priorBoxPlugin/CMakeLists.txt b/plugin/priorBoxPlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/proposalLayerPlugin/CMakeLists.txt b/plugin/proposalLayerPlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/proposalPlugin/CMakeLists.txt b/plugin/proposalPlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/pyramidROIAlignPlugin/CMakeLists.txt b/plugin/pyramidROIAlignPlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/regionPlugin/CMakeLists.txt b/plugin/regionPlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/reorgPlugin/CMakeLists.txt b/plugin/reorgPlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/resizeNearestPlugin/CMakeLists.txt b/plugin/resizeNearestPlugin/CMakeLists.txt
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-file(GLOB SRCS *.cpp *.cu)
+file(GLOB SRCS *.cpp)
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
 set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
diff --git a/plugin/skipLayerNormPlugin/CMakeLists.txt b/plugin/skipLayerNormPlugin/CMakeLists.txt
@@ -13,15 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-string(FIND ${CMAKE_CUDA_FLAGS} "sm_7" POS_SM)
-string(FIND ${CMAKE_CUDA_FLAGS} "compute_7" POS_COMPUTE)
-
-if(${POS_SM} GREATER_EQUAL 0 OR ${POS_COMPUTE} GREATER_EQUAL 0)
-    file(GLOB SRCS *.cpp *.cu)
-
-    set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS})
-    set(PLUGIN_SOURCES ${PLUGIN_SOURCES} PARENT_SCOPE)
-endif()
-
-
+file(GLOB CU_SRCS *.cu)
+set(BERT_CU_SOURCES ${BERT_CU_SOURCES} ${CU_SRCS})
+set(BERT_CU_SOURCES ${BERT_CU_SOURCES} PARENT_SCOPE)
diff --git a/plugin/skipLayerNormPlugin/README.md b/plugin/skipLayerNormPlugin/README.md
@@ -68,4 +68,4 @@ This is the first release of this `README.md` file.
 
 ## Known issues
 
-There are no known issues in this plugin.
+This plugin only supports GPUs with compute capability >= 7.0. For more information see the [CUDA GPU Compute Capability Support Matrix](https://developer.nvidia.com/cuda-gpus#compute)