diff --git a/.gitignore b/.gitignore
index 6be36bf8c243e..708126b3bb070 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,8 @@ paddle/fluid/API_PR.spec
 paddle/fluid/op_use_default_grad_maker_DEV.spec
 paddle/fluid/op_use_default_grad_maker_PR.spec
 paddle/pten/api/*/api*
+paddle/pten/include/*
+paddle/pten/extension.h
 
 *.DS_Store
 *.vs
diff --git a/cmake/pten_kernel.cmake b/cmake/pten.cmake
similarity index 82%
rename from cmake/pten_kernel.cmake
rename to cmake/pten.cmake
index c2928376a02f8..70d61027da872 100644
--- a/cmake/pten_kernel.cmake
+++ b/cmake/pten.cmake
@@ -12,6 +12,48 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+function(generate_unify_header DIR_NAME)
+    set(options "")
+    set(oneValueArgs HEADER_NAME SKIP_SUFFIX)
+    set(multiValueArgs "")
+    cmake_parse_arguments(generate_unify_header "${options}" "${oneValueArgs}"
+        "${multiValueArgs}" ${ARGN})
+
+    # get header name and suffix
+    set(header_name "${DIR_NAME}")
+    list(LENGTH generate_unify_header_HEADER_NAME generate_unify_header_HEADER_NAME_len)
+    if(${generate_unify_header_HEADER_NAME_len} GREATER 0)
+        set(header_name "${generate_unify_header_HEADER_NAME}")
+    endif()
+    set(skip_suffix "")
+    list(LENGTH generate_unify_header_SKIP_SUFFIX generate_unify_header_SKIP_SUFFIX_len)
+    if(${generate_unify_header_SKIP_SUFFIX_len} GREATER 0)
+        set(skip_suffix "${generate_unify_header_SKIP_SUFFIX}")
+    endif()
+
+    # generate target header file
+    set(header_file ${CMAKE_CURRENT_SOURCE_DIR}/include/${header_name}.h)
+    file(WRITE ${header_file} "// Header file generated by paddle/pten/CMakeLists.txt for external users,\n// DO NOT edit or include it within paddle.\n\n#pragma once\n\n")
+
+    # get all top-level headers and write into header file
+    file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}\/${DIR_NAME}\/*.h")
+    foreach(header ${HEADERS})
+        if("${skip_suffix}" STREQUAL "")
+            string(REPLACE "${PADDLE_SOURCE_DIR}\/" "" header "${header}")
+            file(APPEND ${header_file} "#include \"${header}\"\n")
+        else()
+            string(FIND "${header}" "${skip_suffix}.h" skip_suffix_found)
+            if(${skip_suffix_found} EQUAL -1)
+                string(REPLACE "${PADDLE_SOURCE_DIR}\/" "" header "${header}")
+                file(APPEND ${header_file} "#include \"${header}\"\n")
+            endif()
+        endif()
+    endforeach()
+    # append header into extension.h
+    string(REPLACE "${PADDLE_SOURCE_DIR}\/" "" header_file "${header_file}")
+    file(APPEND ${pten_extension_header_file} "#include \"${header_file}\"\n")
+endfunction()
+
 # call kernel_declare need to make sure whether the target of input exists
 function(kernel_declare TARGET_LIST)
     foreach(kernel_path ${TARGET_LIST})
diff --git a/paddle/pten/CMakeLists.txt b/paddle/pten/CMakeLists.txt
index cde5e719e316d..671ed28313af9 100644
--- a/paddle/pten/CMakeLists.txt
+++ b/paddle/pten/CMakeLists.txt
@@ -1,3 +1,6 @@
+# pten auto cmake utils
+include(pten)
+
 # paddle experimental common components
 add_subdirectory(common)
 
@@ -25,3 +28,12 @@ message(STATUS "All standard pten kernels: ${pten_kernels}")
 set(PTEN_DEPS ${PTEN_DEPS} ${pten_kernels})
 
 cc_library(pten DEPS ${PTEN_DEPS})
+
+set(pten_extension_header_file ${CMAKE_CURRENT_SOURCE_DIR}/extension.h CACHE INTERNAL "pten/extension.h file")
+file(WRITE ${pten_extension_header_file} "// Header file generated by paddle/pten/CMakeLists.txt for external users,\n// DO NOT edit or include it within paddle.\n\n#pragma once\n\n")
+
+# generate inner headers include dir for users
+generate_unify_header(backends)
+generate_unify_header(core)
+generate_unify_header(infermeta)
+generate_unify_header(kernels SKIP_SUFFIX grad_kernel)
diff --git a/paddle/pten/core/CMakeLists.txt b/paddle/pten/core/CMakeLists.txt
index 97ea613174c28..f6f0e1f3e26ec 100644
--- a/paddle/pten/core/CMakeLists.txt
+++ b/paddle/pten/core/CMakeLists.txt
@@ -15,6 +15,9 @@ cc_library(tensor_meta SRCS tensor_meta.cc DEPS enforce mixed_vector)
 cc_library(lod_utils SRCS lod_utils.cc DEPS enforce mixed_vector)
 cc_library(dense_tensor SRCS dense_tensor.cc DEPS convert_utils tensor_meta tensor_base)
 cc_library(pten_device_context SRCS device_context.cc DEPS tensor_base )
+
+cc_library(meta_tensor SRCS meta_tensor.cc DEPS tensor_base tensor_meta dense_tensor)
+
 cc_test(unroll_array_ops_test SRCS unroll_array_ops_test.cc)
 cc_library(ddim SRCS ddim.cc DEPS eigen3 boost enforce)
 cc_test(ddim_test SRCS ddim_test.cc DEPS ddim)
diff --git a/paddle/pten/core/kernel_registry.h b/paddle/pten/core/kernel_registry.h
index 194ab52d25688..e1160ea6b7d5d 100644
--- a/paddle/pten/core/kernel_registry.h
+++ b/paddle/pten/core/kernel_registry.h
@@ -24,6 +24,7 @@
 #include "paddle/pten/core/kernel_def.h"
 #include "paddle/pten/core/kernel_factory.h"
 #include "paddle/pten/core/kernel_utils.h"
+#include "paddle/pten/core/macros.h"
 
 #include "paddle/fluid/platform/enforce.h"
 
@@ -158,33 +159,6 @@ struct KernelRegistrar {
   }
 };
 
-#define PT_STATIC_ASSERT_GLOBAL_NAMESPACE(uniq_name, msg) \
-  _PT_STATIC_ASSERT_GLOBAL_NAMESPACE(uniq_name, msg)
-
-#define _PT_STATIC_ASSERT_GLOBAL_NAMESPACE(uniq_name, msg)                    \
-  struct __test_global_namespace_##uniq_name##__ {};                          \
-  static_assert(std::is_same<::__test_global_namespace_##uniq_name##__,       \
-                             __test_global_namespace_##uniq_name##__>::value, \
-                msg)
-
-#ifdef __COUNTER__
-#define PT_ID __COUNTER__
-#else
-#define PT_ID __LINE__
-#endif
-
-#if defined(_WIN32)
-#define UNUSED
-#define __builtin_expect(EXP, C) (EXP)
-#else
-#define UNUSED __attribute__((unused))
-#endif
-
-#define PT_CONCATENATE(arg1, arg2) PT_CONCATENATE1(arg1, arg2)
-#define PT_CONCATENATE1(arg1, arg2) PT_CONCATENATE2(arg1, arg2)
-#define PT_CONCATENATE2(arg1, arg2) arg1##arg2
-#define PT_EXPAND(x) x
-
 /**
  * Reference:
  *
@@ -834,6 +808,9 @@ struct KernelRegistrar {
  * to avoid being removed by linker
  */
 #define PT_DECLARE_KERNEL(kernel_name, backend, layout)                   \
+  PT_STATIC_ASSERT_GLOBAL_NAMESPACE(                                      \
+      pt_declare_tp_kernel_ns_check_##kernel_name##_##backend##_##layout, \
+      "PT_DECLARE_KERNEL must be called in global namespace.");           \
   extern int TouchKernelSymbolFor_##kernel_name##_##backend##_##layout(); \
   UNUSED static int                                                       \
       __declare_kernel_symbol_for_##kernel_name##_##backend##_##layout =  \
diff --git a/paddle/pten/core/macros.h b/paddle/pten/core/macros.h
new file mode 100644
index 0000000000000..fec67b1a3dc25
--- /dev/null
+++ b/paddle/pten/core/macros.h
@@ -0,0 +1,56 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+namespace pten {
+
+// Disable the copy and assignment operator for a class.
+#ifndef DISABLE_COPY_AND_ASSIGN
+#define DISABLE_COPY_AND_ASSIGN(classname)         \
+ private:                                          \
+  classname(const classname&) = delete;            \
+  classname(classname&&) = delete;                 \
+  classname& operator=(const classname&) = delete; \
+  classname& operator=(classname&&) = delete
+#endif
+
+#define PT_STATIC_ASSERT_GLOBAL_NAMESPACE(uniq_name, msg) \
+  _PT_STATIC_ASSERT_GLOBAL_NAMESPACE(uniq_name, msg)
+
+#define _PT_STATIC_ASSERT_GLOBAL_NAMESPACE(uniq_name, msg)                    \
+  struct __test_global_namespace_##uniq_name##__ {};                          \
+  static_assert(std::is_same<::__test_global_namespace_##uniq_name##__,       \
+                             __test_global_namespace_##uniq_name##__>::value, \
+                msg)
+
+#ifdef __COUNTER__
+#define PT_ID __COUNTER__
+#else
+#define PT_ID __LINE__
+#endif
+
+#if defined(_WIN32)
+#define UNUSED
+#define __builtin_expect(EXP, C) (EXP)
+#else
+#define UNUSED __attribute__((unused))
+#endif
+
+#define PT_CONCATENATE(arg1, arg2) PT_CONCATENATE1(arg1, arg2)
+#define PT_CONCATENATE1(arg1, arg2) PT_CONCATENATE2(arg1, arg2)
+#define PT_CONCATENATE2(arg1, arg2) arg1##arg2
+#define PT_EXPAND(x) x
+
+}  // namespace pten
diff --git a/paddle/pten/core/meta_tensor.cc b/paddle/pten/core/meta_tensor.cc
new file mode 100644
index 0000000000000..f52d771b73bb9
--- /dev/null
+++ b/paddle/pten/core/meta_tensor.cc
@@ -0,0 +1,86 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/pten/core/meta_tensor.h"
+
+#include "paddle/pten/core/compat_utils.h"
+#include "paddle/pten/core/dense_tensor.h"
+
+#include "paddle/fluid/platform/enforce.h"
+
+namespace pten {
+
+int64_t MetaTensor::numel() const { return tensor_->numel(); }
+
+DDim MetaTensor::dims() const { return tensor_->dims(); }
+
+DataType MetaTensor::dtype() const { return tensor_->dtype(); }
+
+DataLayout MetaTensor::layout() const { return tensor_->layout(); }
+
+void MetaTensor::set_dims(const DDim& dims) {
+  if (pten::DenseTensor::classof(tensor_)) {
+    CompatibleDenseTensorUtils::GetMutableMeta(
+        static_cast<DenseTensor*>(tensor_))
+        ->dims = dims;
+  } else {
+    PADDLE_THROW(paddle::platform::errors::Unimplemented(
+        "Unsupported setting dims for `%s`.", tensor_->type_info().name()));
+  }
+}
+
+void MetaTensor::set_dtype(DataType dtype) {
+  if (pten::DenseTensor::classof(tensor_)) {
+    CompatibleDenseTensorUtils::GetMutableMeta(
+        static_cast<DenseTensor*>(tensor_))
+        ->dtype = dtype;
+  } else {
+    PADDLE_THROW(paddle::platform::errors::Unimplemented(
+        "Unsupported settting dtype for `%s`.", tensor_->type_info().name()));
+  }
+}
+
+void MetaTensor::set_layout(DataLayout layout) {
+  if (pten::DenseTensor::classof(tensor_)) {
+    CompatibleDenseTensorUtils::GetMutableMeta(
+        static_cast<DenseTensor*>(tensor_))
+        ->layout = layout;
+  } else {
+    PADDLE_THROW(paddle::platform::errors::Unimplemented(
+        "Unsupported settting layout for `%s`.", tensor_->type_info().name()));
+  }
+}
+
+void MetaTensor::share_lod(const MetaTensor& meta_tensor) {
+  if (pten::DenseTensor::classof(tensor_)) {
+    CompatibleDenseTensorUtils::GetMutableMeta(
+        static_cast<DenseTensor*>(tensor_))
+        ->lod = meta_tensor.lod();
+  } else {
+    PADDLE_THROW(paddle::platform::errors::Unimplemented(
+        "Unsupported share lod inplace for `%s`.",
+        tensor_->type_info().name()));
+  }
+}
+
+const LoD& MetaTensor::lod() const {
+  if (pten::DenseTensor::classof(tensor_)) {
+    return static_cast<DenseTensor*>(tensor_)->lod();
+  } else {
+    PADDLE_THROW(paddle::platform::errors::Unimplemented(
+        "Unsupported setting dims for `%s`.", tensor_->type_info().name()));
+  }
+}
+
+}  // namespace pten
diff --git a/paddle/pten/core/meta_tensor.h b/paddle/pten/core/meta_tensor.h
new file mode 100644
index 0000000000000..4273aa6f85b4e
--- /dev/null
+++ b/paddle/pten/core/meta_tensor.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/pten/common/data_type.h"
+#include "paddle/pten/common/layout.h"
+#include "paddle/pten/core/macros.h"
+#include "paddle/pten/core/tensor_base.h"
+#include "paddle/pten/core/tensor_meta.h"
+
+// See Note [ Why still include the fluid headers? ]
+#include "paddle/fluid/framework/ddim.h"
+
+namespace pten {
+
+class MetaTensor {
+ public:
+  explicit MetaTensor(TensorBase* tensor) : tensor_(tensor) {}
+
+  MetaTensor() = default;
+  MetaTensor(const MetaTensor&) = default;
+  MetaTensor(MetaTensor&&) = default;
+  MetaTensor& operator=(const MetaTensor&) = delete;
+  MetaTensor& operator=(MetaTensor&&) = delete;
+
+  virtual ~MetaTensor() = default;
+
+  virtual int64_t numel() const;
+  virtual DDim dims() const;
+  virtual DataType dtype() const;
+  virtual DataLayout layout() const;
+  virtual void set_dims(const DDim& dims);
+  virtual void set_dtype(DataType dtype);
+  virtual void set_layout(DataLayout layout);
+  virtual void share_lod(const MetaTensor& meta_tensor);
+
+ private:
+  const LoD& lod() const;
+  TensorBase* tensor_;
+};
+
+}  // namespace pten
diff --git a/paddle/pten/kernels/CMakeLists.txt b/paddle/pten/kernels/CMakeLists.txt
index 76e112808892d..f838c4d424c2d 100644
--- a/paddle/pten/kernels/CMakeLists.txt
+++ b/paddle/pten/kernels/CMakeLists.txt
@@ -1,5 +1,3 @@
-include(pten_kernel)
-
 set(kernel_declare_file ${PADDLE_BINARY_DIR}/paddle/pten/kernels/declarations.h.tmp CACHE INTERNAL "declarations.h file")
 set(kernel_declare_file_final ${PADDLE_BINARY_DIR}/paddle/pten/kernels/declarations.h)
 file(WRITE ${kernel_declare_file} "// Generated by the paddle/pten/kernels/CMakeLists.txt.  DO NOT EDIT!\n\n#pragma once\n\n")