diff --git a/backends/metax_gpu/common/flags_declare.cc b/backends/metax_gpu/common/flags_declare.cc index fb656878033..0b65d635510 100644 --- a/backends/metax_gpu/common/flags_declare.cc +++ b/backends/metax_gpu/common/flags_declare.cc @@ -101,6 +101,18 @@ PHI_DEFINE_EXPORTED_bool( "faster but it may loss precision in most case. If true, the compute " "type will be set to fp16. Default is false."); +/** + * Torch Compatible related FLAG + * Name: FLAGS_torch_compatible_kernel + * Since Version: 3.2.2 + * Value Range: bool, default=false + * Example: + * Note: Whether use torch compatible version kernel. + */ +PHI_DEFINE_EXPORTED_bool(torch_compatible_kernel, + false, + "Whether use torch compatible version kernel."); + PHI_DEFINE_EXPORTED_string( selected_gpus, "", diff --git a/backends/metax_gpu/kernels/cuda_kernels/gammaln_grad_kernel.cu b/backends/metax_gpu/kernels/cuda_kernels/gammaln_grad_kernel.cu new file mode 100644 index 00000000000..850f0d68bac --- /dev/null +++ b/backends/metax_gpu/kernels/cuda_kernels/gammaln_grad_kernel.cu @@ -0,0 +1,28 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/common/amp_type_traits.h" +#include "paddle/phi/core/kernel_registry.h" +#include "paddle/phi/kernels/gammaln_grad_kernel.h" +#include "paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h" + +PD_CUSTOM_KERNEL_REGISTER(gammaln_grad, + metax_gpu, + ALL_LAYOUT, + phi::GammalnGradKernel, + float, + double, + phi::float16, + phi::bfloat16) {} diff --git a/backends/metax_gpu/kernels/funcs/softmax.cu b/backends/metax_gpu/kernels/funcs/softmax.cu index 44bfd02a308..a587f9ed016 100644 --- a/backends/metax_gpu/kernels/funcs/softmax.cu +++ b/backends/metax_gpu/kernels/funcs/softmax.cu @@ -13,13 +13,13 @@ See the License for the specific language governing permissions and limitations under the License. */ #include +#include "glog/logging.h" #include "kernels/metax_kernel/metax_context.h" #include "paddle/phi/backends/gpu/gpu_context.h" #include "paddle/phi/backends/gpu/gpu_dnn.h" #include "paddle/phi/kernels/funcs/math_function.h" #include "paddle/phi/kernels/funcs/softmax.h" #include "paddle/phi/kernels/funcs/softmax_impl.h" - namespace phi { namespace funcs { @@ -38,6 +38,7 @@ void SoftmaxCUDNNFunctor::operator()( ScopedTensorDescriptor yDesc; std::vector cudnn_tensor_dims = common::vectorize(X->dims()); DataLayout layout = DataLayout::kNCHW; + VLOG(0) << "Enter softmax Kernel22."; if (cudnn_tensor_dims.size() == 5) { layout = DataLayout::kNCDHW; } diff --git a/backends/metax_gpu/kernels/gpudnn/softmax_kernel_dnn.cu b/backends/metax_gpu/kernels/gpudnn/softmax_kernel_dnn.cu new file mode 100644 index 00000000000..b51f92c96a4 --- /dev/null +++ b/backends/metax_gpu/kernels/gpudnn/softmax_kernel_dnn.cu @@ -0,0 +1,70 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "kernels/gpudnn/softmax_gpudnn.h" +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/core/kernel_registry.h" +#include "paddle/phi/kernels/funcs/math_function.h" +#include "paddle/phi/kernels/softmax_kernel.h" + +namespace phi { + +template +void SoftmaxGPUDNNKernel(const Context& dev_ctx, + const DenseTensor& x, + int axis, + DenseTensor* out) { + dev_ctx.template Alloc(out); + if (x.numel() == 0) return; + + const int rank = x.dims().size(); + // For 0D Tensor + if (rank == 0) { + phi::funcs::set_constant(dev_ctx, out, static_cast(1.0)); + return; + } + + SoftmaxForwardCUDAKernelDriver(dev_ctx, x, axis, out); +} + +} // namespace phi + +#ifdef PADDLE_WITH_HIP +PD_REGISTER_PLUGIN_KERNEL(softmax, + metax_gpu, + ALL_LAYOUT, + phi::SoftmaxGPUDNNKernel, + float, + phi::float16, + phi::bfloat16) {} +#else +#if CUDNN_VERSION_MIN(8, 1, 0) +PD_REGISTER_PLUGIN_KERNEL(softmax, + metax_gpu, + ALL_LAYOUT, + phi::SoftmaxGPUDNNKernel, + float, + double, + phi::float16, + phi::bfloat16) {} +#else +PD_REGISTER_PLUGIN_KERNEL(softmax, + metax_gpu, + ALL_LAYOUT, + phi::SoftmaxGPUDNNKernel, + float, + double, + phi::float16) {} +#endif +#endif diff --git a/backends/metax_gpu/kernels/metax_kernel/softmax_kernel_register.cu b/backends/metax_gpu/kernels/metax_kernel/softmax_kernel_register.cu index 0344a81dc19..523a2e4d76b 100644 --- a/backends/metax_gpu/kernels/metax_kernel/softmax_kernel_register.cu +++ b/backends/metax_gpu/kernels/metax_kernel/softmax_kernel_register.cu @@ -11,7 +11,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - +#if 0 #include "paddle/phi/backends/gpu/gpu_context.h" #include "paddle/phi/common/bfloat16.h" #include "paddle/phi/common/float16.h" @@ -27,3 +27,5 @@ PD_REGISTER_PLUGIN_KERNEL(softmax, double, phi::dtype::float16, phi::dtype::bfloat16) {} + +#endif diff --git a/backends/metax_gpu/kernels/metax_kernel/svd_kernel_register.cu b/backends/metax_gpu/kernels/metax_kernel/svd_kernel_register.cu index 5f9d6cc20e0..c8ece09bbae 100644 --- a/backends/metax_gpu/kernels/metax_kernel/svd_kernel_register.cu +++ b/backends/metax_gpu/kernels/metax_kernel/svd_kernel_register.cu @@ -15,7 +15,7 @@ #ifndef PADDLE_WITH_HIP // HIP not support cusolver -#include "kernels/impl/values_vectors_functor.h" +#include "kernels/metax_kernel/metax_context.h" #include "paddle/phi/backends/dynload/cusolver.h" #include "paddle/phi/common/memory_utils.h" #include "paddle/phi/core/kernel_registry.h" @@ -60,7 +60,6 @@ void GesvdjBatched(const phi::GPUContext& dev_ctx, int ldu = m; int ldt = n; int lwork = 0; - // auto handle = dev_ctx.cusolver_dn_handle(); auto handle = GetCusolverDnHandle(dev_ctx.stream(), dev_ctx.GetPlace()); PADDLE_ENFORCE_GPU_SUCCESS( phi::dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params)); @@ -142,7 +141,6 @@ void GesvdjBatched(const phi::GPUContext& dev_ctx, int ldu = m; int ldt = n; int lwork = 0; - // auto handle = dev_ctx.cusolver_dn_handle(); auto handle = GetCusolverDnHandle(dev_ctx.stream(), dev_ctx.GetPlace()); PADDLE_ENFORCE_GPU_SUCCESS( phi::dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params)); @@ -205,17 +203,17 @@ void GesvdjBatched(const phi::GPUContext& dev_ctx, } template <> -void GesvdjBatched>(const phi::GPUContext& dev_ctx, - int batchSize, - int m, - int n, - int k, - phi::dtype::complex* A, - phi::dtype::complex* U, - phi::dtype::complex* V, - float* S, - int* info, - int thin_UV) { +void GesvdjBatched(const phi::GPUContext& dev_ctx, + int batchSize, + int m, + int n, + int k, + phi::complex64* A, + phi::complex64* U, + phi::complex64* V, + float* S, + int* info, + int thin_UV) { /* compute singular vectors */ const cusolverEigMode_t jobz = CUSOLVER_EIG_MODE_VECTOR; /* compute singular vectors */ @@ -224,7 +222,6 @@ void GesvdjBatched>(const phi::GPUContext& dev_ctx, int ldu = m; int ldt = n; int lwork = 0; - // auto handle = dev_ctx.cusolver_dn_handle(); auto handle = GetCusolverDnHandle(dev_ctx.stream(), dev_ctx.GetPlace()); PADDLE_ENFORCE_GPU_SUCCESS( phi::dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params)); @@ -245,10 +242,10 @@ void GesvdjBatched>(const phi::GPUContext& dev_ctx, gesvdj_params)); auto workspace = phi::memory_utils::Alloc( dev_ctx.GetPlace(), - lwork * sizeof(phi::dtype::complex), + lwork * sizeof(phi::complex64), phi::Stream(reinterpret_cast(dev_ctx.stream()))); - phi::dtype::complex* workspace_ptr = - reinterpret_cast*>(workspace->ptr()); + phi::complex64* workspace_ptr = + reinterpret_cast(workspace->ptr()); int stride_A = lda * n; int stride_U = ldu * (thin_UV ? k : m); int stride_V = ldt * (thin_UV ? k : n); @@ -289,17 +286,17 @@ void GesvdjBatched>(const phi::GPUContext& dev_ctx, } template <> -void GesvdjBatched>(const phi::GPUContext& dev_ctx, - int batchSize, - int m, - int n, - int k, - phi::dtype::complex* A, - phi::dtype::complex* U, - phi::dtype::complex* V, - double* S, - int* info, - int thin_UV) { +void GesvdjBatched(const phi::GPUContext& dev_ctx, + int batchSize, + int m, + int n, + int k, + phi::complex128* A, + phi::complex128* U, + phi::complex128* V, + double* S, + int* info, + int thin_UV) { /* compute singular vectors */ const cusolverEigMode_t jobz = CUSOLVER_EIG_MODE_VECTOR; /* compute singular vectors */ @@ -308,7 +305,6 @@ void GesvdjBatched>(const phi::GPUContext& dev_ctx, int ldu = m; int ldt = n; int lwork = 0; - // auto handle = dev_ctx.cusolver_dn_handle(); auto handle = GetCusolverDnHandle(dev_ctx.stream(), dev_ctx.GetPlace()); PADDLE_ENFORCE_GPU_SUCCESS( phi::dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params)); @@ -329,10 +325,10 @@ void GesvdjBatched>(const phi::GPUContext& dev_ctx, gesvdj_params)); auto workspace = phi::memory_utils::Alloc( dev_ctx.GetPlace(), - lwork * sizeof(phi::dtype::complex), + lwork * sizeof(phi::complex128), phi::Stream(reinterpret_cast(dev_ctx.stream()))); - phi::dtype::complex* workspace_ptr = - reinterpret_cast*>(workspace->ptr()); + phi::complex128* workspace_ptr = + reinterpret_cast(workspace->ptr()); int stride_A = lda * n; int stride_U = ldu * (thin_UV ? k : m); int stride_V = ldt * (thin_UV ? k : n); @@ -432,7 +428,7 @@ PD_REGISTER_PLUGIN_KERNEL(svd, // cuda_only phi::SvdKernel, float, double, - phi::dtype::complex, - phi::dtype::complex) {} + phi::complex64, + phi::complex128) {} #endif // not PADDLE_WITH_HIP