From fbe81d2f580b191dead5ceae5408b65bf7ed1153 Mon Sep 17 00:00:00 2001 From: Tristan Konolige Date: Mon, 19 Sep 2022 10:40:09 -0700 Subject: [PATCH 1/8] [BUILD] Re-enable ccache by default Previously ccache was disabled because of possible issues with hexagon. Re-enabling it to provide a best effort attempt at using it. --- CMakeLists.txt | 36 ++++++++++++++++++++++++++++++++++++ cmake/config.cmake | 11 +++++++++++ docs/install/from_source.rst | 2 ++ 3 files changed, 49 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c355238b8c8..052dd9c9f6ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -460,6 +460,42 @@ if(USE_PIPELINE_EXECUTOR) list(APPEND RUNTIME_SRCS ${RUNTIME_PIPELINE_SRCS}) endif(USE_PIPELINE_EXECUTOR) +#Caches the build. +#Note that ccache-3.x doesn't support nvcc well, so CUDA kernels may never hit the cache and still +#need to be re-compiled every time. Using ccache 4.0+ can resolve this issue. + +if(USE_CCACHE) # True for AUTO, ON, /path/to/ccache + if(DEFINED CXX_COMPILER_LAUNCHER OR DEFINED C_COMPILER_LAUNCHER) + message(STATUS "CXX_COMPILER_LAUNCHER or C_COMPILER_LAUNCHER already defined, not using ccache") + else() + if("${USE_CCACHE}" STREQUAL "AUTO") # Auto mode + find_program(CCACHE_FOUND ccache) + if(CCACHE_FOUND) + message(STATUS "Found the path to ccache, enabling ccache") + set(PATH_TO_CCACHE ccache) + else() + message(STATUS "Didn't find the path to CCACHE, disabling ccache") + endif(CCACHE_FOUND) + elseif("${USE_CCACHE}" MATCHES ${IS_TRUE_PATTERN}) + find_program(CCACHE_FOUND ccache) + if(CCACHE_FOUND) + message(STATUS "Found the path to ccache, enabling ccache") + set(PATH_TO_CCACHE ccache) + else() + message(FATAL_ERROR "Cannot find ccache. Set USE_CCACHE mode to AUTO or OFF to build without ccache. USE_CCACHE=" "${USE_CCACHE}") + endif(CCACHE_FOUND) + else() # /path/to/ccache + set(PATH_TO_CCACHE USE_CCACHE) + message(STATUS "Setting ccache path to " "${PATH_TO_CCACHE}") + endif() + # Set the flag for ccache + if(DEFINED PATH_TO_CCACHE) + set(CXX_COMPILER_LAUNCHER PATH_TO_CCACHE) + set(C_COMPILER_LAUNCHER PATH_TO_CCACHE) + endif() + endif() +endif(USE_CCACHE) + # Module rules include(cmake/modules/VTA.cmake) include(cmake/modules/StandaloneCrt.cmake) diff --git a/cmake/config.cmake b/cmake/config.cmake index 18725de844b2..7067af42e9f1 100644 --- a/cmake/config.cmake +++ b/cmake/config.cmake @@ -352,6 +352,17 @@ set(USE_LIBBACKTRACE AUTO) # runtime functions to be unavailable to the program. set(BUILD_STATIC_RUNTIME OFF) +# Caches the build so that building is faster when switching between branches. +# If you switch branches, build and then encounter a linking error, you may +# need to regenerate the build tree through "make .." (the cache will +# still provide significant speedups). +# Possible values: +# - AUTO: search for path to ccache, disable if not found. +# - ON: enable ccache by searching for the path to ccache, report an error if not found +# - OFF: disable ccache +# - /path/to/ccache: use specific path to ccache +set(USE_CCACHE AUTO) + # Whether to enable PAPI support in profiling. PAPI provides access to hardware # counters while profiling. # Possible values: diff --git a/docs/install/from_source.rst b/docs/install/from_source.rst index e5622b40a173..63d8aab33623 100644 --- a/docs/install/from_source.rst +++ b/docs/install/from_source.rst @@ -141,6 +141,8 @@ The configuration of TVM can be modified by editing `config.cmake` and/or by pas - On supported platforms, the `Ccache compiler wrapper `_ may be helpful for reducing TVM's build time. There are several ways to enable CCache in TVM builds: + - Leave `USE_CCACHE=AUTO` in `build/config.cmake`. CCache will be used if it is found. + - Ccache's Masquerade mode. This is typically enabled during the Ccache installation process. To have TVM use Ccache in masquerade, simply specify the appropriate C/C++ compiler paths when configuring TVM's build system. For example: From 33114f96722fa468687c4a4d2a819345e457a9a1 Mon Sep 17 00:00:00 2001 From: Tristan Konolige Date: Mon, 19 Sep 2022 13:55:25 -0700 Subject: [PATCH 2/8] set tvm_option, set variables correctly --- CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 052dd9c9f6ca..17806977ce14 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ tvm_option(USE_PAPI "Use Performance Application Programming Interface (PAPI) to tvm_option(USE_GTEST "Use GoogleTest for C++ sanity tests" AUTO) tvm_option(USE_CUSTOM_LOGGING "Use user-defined custom logging, tvm::runtime::detail::LogFatalImpl and tvm::runtime::detail::LogMessageImpl must be implemented" OFF) tvm_option(USE_ALTERNATIVE_LINKER "Use 'mold' or 'lld' if found when invoking compiler to link artifact" AUTO) +tvm_option(USE_CCACHE "Use ccache if found when invoking compiler" AUTO) # 3rdparty libraries tvm_option(DLPACK_PATH "Path to DLPACK" "3rdparty/dlpack/include") @@ -490,8 +491,8 @@ if(USE_CCACHE) # True for AUTO, ON, /path/to/ccache endif() # Set the flag for ccache if(DEFINED PATH_TO_CCACHE) - set(CXX_COMPILER_LAUNCHER PATH_TO_CCACHE) - set(C_COMPILER_LAUNCHER PATH_TO_CCACHE) + set(CXX_COMPILER_LAUNCHER "${PATH_TO_CCACHE}") + set(C_COMPILER_LAUNCHER "${PATH_TO_CCACHE}") endif() endif() endif(USE_CCACHE) From 841b6b64e6522afd48db33523e035d697b9969fc Mon Sep 17 00:00:00 2001 From: Tristan Konolige Date: Mon, 19 Sep 2022 14:11:00 -0700 Subject: [PATCH 3/8] clean up comment, fatal error if launcher is defined with USE_CCACHE=ON --- CMakeLists.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 17806977ce14..b4cab3af3b25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -461,13 +461,17 @@ if(USE_PIPELINE_EXECUTOR) list(APPEND RUNTIME_SRCS ${RUNTIME_PIPELINE_SRCS}) endif(USE_PIPELINE_EXECUTOR) -#Caches the build. -#Note that ccache-3.x doesn't support nvcc well, so CUDA kernels may never hit the cache and still -#need to be re-compiled every time. Using ccache 4.0+ can resolve this issue. +# Caches the build. +# Note that ccache-3.x doesn't support nvcc well, so CUDA kernels may never hit the cache and still +# need to be re-compiled every time. Using ccache 4.0+ can resolve this issue. if(USE_CCACHE) # True for AUTO, ON, /path/to/ccache if(DEFINED CXX_COMPILER_LAUNCHER OR DEFINED C_COMPILER_LAUNCHER) - message(STATUS "CXX_COMPILER_LAUNCHER or C_COMPILER_LAUNCHER already defined, not using ccache") + if("${USE_CCACHE}" STREQUAL "AUTO") + message(STATUS "CXX_COMPILER_LAUNCHER or C_COMPILER_LAUNCHER already defined, not using ccache") + elseif("${USE_CCACHE}" MATCHES ${IS_TRUE_PATTERN}) + message(FATAL_ERROR "CXX_COMPILER_LAUNCHER or C_COMPILER_LAUNCHER is already defined, refusing to override with ccache. Either unset or disable ccache.") + endif() else() if("${USE_CCACHE}" STREQUAL "AUTO") # Auto mode find_program(CCACHE_FOUND ccache) From 9644b40db0fd18ff488714eca2d629ee3c2a4c2e Mon Sep 17 00:00:00 2001 From: Tristan Konolige Date: Mon, 19 Sep 2022 15:37:19 -0700 Subject: [PATCH 4/8] add ccache to libinfo --- cmake/modules/LibInfo.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/modules/LibInfo.cmake b/cmake/modules/LibInfo.cmake index 6bc8f6b46390..73d3a9dbbe10 100644 --- a/cmake/modules/LibInfo.cmake +++ b/cmake/modules/LibInfo.cmake @@ -117,6 +117,7 @@ function(add_lib_info src_file) TVM_INFO_USE_CLML="${USE_CLML}" TVM_INFO_USE_CLML_GRAPH_EXECUTOR="${USE_CLML_GRAPH_EXECUTOR}" TVM_INFO_USE_UMA="${USE_UMA}" + TVM_INFO_USE_CCACHE="${USE_CCACHE}" ) endfunction() From 151346cdeccef3bf0570497807e579c1f645dea5 Mon Sep 17 00:00:00 2001 From: Tristan Konolige Date: Mon, 19 Sep 2022 15:37:52 -0700 Subject: [PATCH 5/8] more libinfo --- src/support/libinfo.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/support/libinfo.cc b/src/support/libinfo.cc index 4b2f6034730d..a7d8e6a1ae2d 100644 --- a/src/support/libinfo.cc +++ b/src/support/libinfo.cc @@ -318,6 +318,7 @@ TVM_DLL Map GetLibInfo() { {"USE_CLML", TVM_INFO_USE_CLML}, {"USE_CLML_GRAPH_EXECUTOR", TVM_INFO_USE_CLML_GRAPH_EXECUTOR}, {"USE_UMA", TVM_INFO_USE_UMA}, + {"USE_CCACHE", TVM_INFO_USE_CCACHE}, }; return result; } From d7ba5166851e76332445f3ee8ce732a2b6ee226d Mon Sep 17 00:00:00 2001 From: Tristan Konolige Date: Mon, 19 Sep 2022 15:45:25 -0700 Subject: [PATCH 6/8] add launcher to summary, move ccache to seperate file --- CMakeLists.txt | 37 +--------------------------- cmake/utils/CCache.cmake | 52 +++++++++++++++++++++++++++++++++++++++ cmake/utils/Summary.cmake | 1 + 3 files changed, 54 insertions(+), 36 deletions(-) create mode 100644 cmake/utils/CCache.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index b4cab3af3b25..188f9fb1c7a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -464,42 +464,7 @@ endif(USE_PIPELINE_EXECUTOR) # Caches the build. # Note that ccache-3.x doesn't support nvcc well, so CUDA kernels may never hit the cache and still # need to be re-compiled every time. Using ccache 4.0+ can resolve this issue. - -if(USE_CCACHE) # True for AUTO, ON, /path/to/ccache - if(DEFINED CXX_COMPILER_LAUNCHER OR DEFINED C_COMPILER_LAUNCHER) - if("${USE_CCACHE}" STREQUAL "AUTO") - message(STATUS "CXX_COMPILER_LAUNCHER or C_COMPILER_LAUNCHER already defined, not using ccache") - elseif("${USE_CCACHE}" MATCHES ${IS_TRUE_PATTERN}) - message(FATAL_ERROR "CXX_COMPILER_LAUNCHER or C_COMPILER_LAUNCHER is already defined, refusing to override with ccache. Either unset or disable ccache.") - endif() - else() - if("${USE_CCACHE}" STREQUAL "AUTO") # Auto mode - find_program(CCACHE_FOUND ccache) - if(CCACHE_FOUND) - message(STATUS "Found the path to ccache, enabling ccache") - set(PATH_TO_CCACHE ccache) - else() - message(STATUS "Didn't find the path to CCACHE, disabling ccache") - endif(CCACHE_FOUND) - elseif("${USE_CCACHE}" MATCHES ${IS_TRUE_PATTERN}) - find_program(CCACHE_FOUND ccache) - if(CCACHE_FOUND) - message(STATUS "Found the path to ccache, enabling ccache") - set(PATH_TO_CCACHE ccache) - else() - message(FATAL_ERROR "Cannot find ccache. Set USE_CCACHE mode to AUTO or OFF to build without ccache. USE_CCACHE=" "${USE_CCACHE}") - endif(CCACHE_FOUND) - else() # /path/to/ccache - set(PATH_TO_CCACHE USE_CCACHE) - message(STATUS "Setting ccache path to " "${PATH_TO_CCACHE}") - endif() - # Set the flag for ccache - if(DEFINED PATH_TO_CCACHE) - set(CXX_COMPILER_LAUNCHER "${PATH_TO_CCACHE}") - set(C_COMPILER_LAUNCHER "${PATH_TO_CCACHE}") - endif() - endif() -endif(USE_CCACHE) +include(cmake/utils/CCache.cmake) # Module rules include(cmake/modules/VTA.cmake) diff --git a/cmake/utils/CCache.cmake b/cmake/utils/CCache.cmake new file mode 100644 index 000000000000..f38a36b5dee8 --- /dev/null +++ b/cmake/utils/CCache.cmake @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(USE_CCACHE) # True for AUTO, ON, /path/to/ccache + if(DEFINED CXX_COMPILER_LAUNCHER OR DEFINED C_COMPILER_LAUNCHER) + if("${USE_CCACHE}" STREQUAL "AUTO") + message(STATUS "CXX_COMPILER_LAUNCHER or C_COMPILER_LAUNCHER already defined, not using ccache") + elseif("${USE_CCACHE}" MATCHES ${IS_TRUE_PATTERN}) + message(FATAL_ERROR "CXX_COMPILER_LAUNCHER or C_COMPILER_LAUNCHER is already defined, refusing to override with ccache. Either unset or disable ccache.") + endif() + else() + if("${USE_CCACHE}" STREQUAL "AUTO") # Auto mode + find_program(CCACHE_FOUND "ccache") + if(CCACHE_FOUND) + message(STATUS "Found the path to ccache, enabling ccache") + set(PATH_TO_CCACHE "ccache") + else() + message(STATUS "Didn't find the path to CCACHE, disabling ccache") + endif(CCACHE_FOUND) + elseif("${USE_CCACHE}" MATCHES ${IS_TRUE_PATTERN}) + find_program(CCACHE_FOUND "ccache") + if(CCACHE_FOUND) + message(STATUS "Found the path to ccache, enabling ccache") + set(PATH_TO_CCACHE "ccache") + else() + message(FATAL_ERROR "Cannot find ccache. Set USE_CCACHE mode to AUTO or OFF to build without ccache. USE_CCACHE=" "${USE_CCACHE}") + endif(CCACHE_FOUND) + else() # /path/to/ccache + set(PATH_TO_CCACHE "${USE_CCACHE}") + message(STATUS "Setting ccache path to " "${PATH_TO_CCACHE}") + endif() + # Set the flag for ccache + if(DEFINED PATH_TO_CCACHE) + set(CXX_COMPILER_LAUNCHER "${PATH_TO_CCACHE}") + set(C_COMPILER_LAUNCHER "${PATH_TO_CCACHE}") + endif() + endif() +endif(USE_CCACHE) diff --git a/cmake/utils/Summary.cmake b/cmake/utils/Summary.cmake index 1b973f253a00..f9aa96b36d40 100644 --- a/cmake/utils/Summary.cmake +++ b/cmake/utils/Summary.cmake @@ -42,6 +42,7 @@ macro(print_summary) message(STATUS " C++ compiler ID : ${CMAKE_CXX_COMPILER_ID}") message(STATUS " C++ compiler version : ${CMAKE_CXX_COMPILER_VERSION}") message(STATUS " CXX flags : ${CMAKE_CXX_FLAGS}") + message(STATUS " CXX launcher : ${CMAKE_CXX_LAUNCHER}") message(STATUS " Linker flags : ${CMAKE_SHARED_LINKER_FLAGS}") message(STATUS " Build type : ${CMAKE_BUILD_TYPE}") get_directory_property(READABLE_COMPILE_DEFS DIRECTORY ${PROJECT_SOURCE_DIR} COMPILE_DEFINITIONS) From a509fb9d8a79ac5ad5e5d30b5f079d16ca754532 Mon Sep 17 00:00:00 2001 From: Tristan Konolige Date: Mon, 19 Sep 2022 15:52:50 -0700 Subject: [PATCH 7/8] Update cmake/utils/Summary.cmake Co-authored-by: driazati <9407960+driazati@users.noreply.github.com> --- cmake/utils/Summary.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/utils/Summary.cmake b/cmake/utils/Summary.cmake index f9aa96b36d40..acb5703f600a 100644 --- a/cmake/utils/Summary.cmake +++ b/cmake/utils/Summary.cmake @@ -42,7 +42,7 @@ macro(print_summary) message(STATUS " C++ compiler ID : ${CMAKE_CXX_COMPILER_ID}") message(STATUS " C++ compiler version : ${CMAKE_CXX_COMPILER_VERSION}") message(STATUS " CXX flags : ${CMAKE_CXX_FLAGS}") - message(STATUS " CXX launcher : ${CMAKE_CXX_LAUNCHER}") + message(STATUS " CXX launcher : ${CMAKE_CXX_COMPILER_LAUNCHER}") message(STATUS " Linker flags : ${CMAKE_SHARED_LINKER_FLAGS}") message(STATUS " Build type : ${CMAKE_BUILD_TYPE}") get_directory_property(READABLE_COMPILE_DEFS DIRECTORY ${PROJECT_SOURCE_DIR} COMPILE_DEFINITIONS) From dd2bf9cff71af77814f367e265df841c2344d316 Mon Sep 17 00:00:00 2001 From: Tristan Konolige Date: Mon, 19 Sep 2022 15:55:35 -0700 Subject: [PATCH 8/8] correct name for Summary.cmake --- cmake/utils/Summary.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/utils/Summary.cmake b/cmake/utils/Summary.cmake index acb5703f600a..e3ea925a9ae1 100644 --- a/cmake/utils/Summary.cmake +++ b/cmake/utils/Summary.cmake @@ -42,7 +42,7 @@ macro(print_summary) message(STATUS " C++ compiler ID : ${CMAKE_CXX_COMPILER_ID}") message(STATUS " C++ compiler version : ${CMAKE_CXX_COMPILER_VERSION}") message(STATUS " CXX flags : ${CMAKE_CXX_FLAGS}") - message(STATUS " CXX launcher : ${CMAKE_CXX_COMPILER_LAUNCHER}") + message(STATUS " CXX launcher : ${CXX_COMPILER_LAUNCHER}") message(STATUS " Linker flags : ${CMAKE_SHARED_LINKER_FLAGS}") message(STATUS " Build type : ${CMAKE_BUILD_TYPE}") get_directory_property(READABLE_COMPILE_DEFS DIRECTORY ${PROJECT_SOURCE_DIR} COMPILE_DEFINITIONS)