diff --git a/cpp/tensorrt_llm/thop/fp8Op.cpp b/cpp/tensorrt_llm/thop/fp8Op.cpp
index 99af87220f62..5254754eef15 100644
--- a/cpp/tensorrt_llm/thop/fp8Op.cpp
+++ b/cpp/tensorrt_llm/thop/fp8Op.cpp
@@ -111,11 +111,12 @@ std::tuple<Tensor, Tensor> e4m3_quantize_helper(Tensor input, at::optional<Tenso
 
     if (scales.has_value())
     {
-        // static quantization will use float scales by default.
-        scales_ = scales.value().clone();
-        CHECK_TH_CUDA(scales_);
-        CHECK_TYPE(scales_, torch::kFloat32);
-        e4m3_static_quantize(input, quantized_input, scales_, stream, quantize_mode);
+        // static quantization will use float scales by default and output scales_ will be ignored.
+        scales_
+            = torch::empty_like(scales.value(), torch::dtype(input.dtype()).device(torch::kCUDA).requires_grad(false));
+        CHECK_TH_CUDA(scales.value());
+        CHECK_TYPE(scales.value(), torch::kFloat32);
+        e4m3_static_quantize(input, quantized_input, scales.value(), stream, quantize_mode);
     }
     else
     {