diff --git a/cmake/cinn.cmake b/cmake/cinn.cmake
index 838f318a5a3bc1..4d530ca4be5f14 100644
--- a/cmake/cinn.cmake
+++ b/cmake/cinn.cmake
@@ -26,12 +26,21 @@ else()
 endif()
 
 if(NOT DEFINED ENV{runtime_include_dir})
-  message(
-    STATUS
-      "set runtime_include_dir: ${CMAKE_SOURCE_DIR}/paddle/cinn/runtime/cuda")
-  set(ENV{runtime_include_dir} "${CMAKE_SOURCE_DIR}/paddle/cinn/runtime/cuda")
-  add_definitions(
-    -DRUNTIME_INCLUDE_DIR="${CMAKE_SOURCE_DIR}/paddle/cinn/runtime/cuda")
+  if(WITH_GPU)
+    message(
+      STATUS
+        "set runtime_include_dir: ${CMAKE_SOURCE_DIR}/paddle/cinn/runtime/cuda")
+    set(ENV{runtime_include_dir} "${CMAKE_SOURCE_DIR}/paddle/cinn/runtime/cuda")
+    add_definitions(
+      -DRUNTIME_INCLUDE_DIR="${CMAKE_SOURCE_DIR}/paddle/cinn/runtime/cuda")
+  elseif(WITH_ROCM)
+    message(
+      STATUS
+        "set runtime_include_dir: ${CMAKE_SOURCE_DIR}/paddle/cinn/runtime/hip")
+    set(ENV{runtime_include_dir} "${CMAKE_SOURCE_DIR}/paddle/cinn/runtime/hip")
+    add_definitions(
+      -DRUNTIME_INCLUDE_DIR="${CMAKE_SOURCE_DIR}/paddle/cinn/runtime/hip")
+  endif()
 endif()
 
 if(WITH_TESTING)
@@ -117,6 +126,10 @@ if(WITH_ROCM)
     add_definitions(-DCINN_WITH_HIP)
   endif()
   link_libraries(${ROCM_HIPRTC_LIB})
+
+  message(
+    STATUS "copy paddle/cinn/common/float16.h to $ENV{runtime_include_dir}")
+  file(COPY paddle/cinn/common/float16.h DESTINATION $ENV{runtime_include_dir})
 endif()
 
 set(cinnapi_src CACHE INTERNAL "" FORCE)
diff --git a/python/setup_cinn.py.in b/python/setup_cinn.py.in
index eb5878558e422a..0eda876bf0ee1c 100644
--- a/python/setup_cinn.py.in
+++ b/python/setup_cinn.py.in
@@ -155,6 +155,7 @@ if '${WITH_GPU}' == 'ON':
 
 if '${WITH_ROCM}' == 'ON':
     cinnlibs.append('${CMAKE_BINARY_DIR}/dist/cinn/include/paddle/cinn/runtime/hip/cinn_hip_runtime_source.h')
+    cinnlibs.append('${CMAKE_BINARY_DIR}/dist/cinn/include/paddle/cinn/runtime/hip/float16.h')
 
 if '${CINN_WITH_SYCL}' == 'ON':
     cinnlibs.append('${CMAKE_BINARY_DIR}/dist/cinn/include/paddle/cinn/runtime/sycl/cinn_sycl_runtime_source.h')