Skip to content
Merged
Show file tree
Hide file tree
Changes from 55 commits
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
5f13582
fix typos
IwakuraRein Apr 6, 2026
d61d597
WIP
IwakuraRein Apr 6, 2026
aed7865
WIP
IwakuraRein Apr 7, 2026
e94e34c
explicit amax and fp4 quant
IwakuraRein Apr 8, 2026
07c284e
update core.py
IwakuraRein Apr 8, 2026
1fca13e
fix quantization bug when tokens >= 1024
IwakuraRein Apr 9, 2026
cb1717f
fix MoERunner initializer
IwakuraRein Apr 9, 2026
d4ff51e
fix deduce_trtllm_gen_tensor_dtype
IwakuraRein Apr 9, 2026
9011f09
use expanded_idx_to_permuted_idx in amax kernel
IwakuraRein Apr 9, 2026
69fd614
add nvfp4QuantAndPerTokenScaleKernel
IwakuraRein Apr 10, 2026
05e6f55
update test; fix pre-commit
IwakuraRein Apr 11, 2026
7ea8538
make test smaller
IwakuraRein Apr 11, 2026
a41b623
fix typo
IwakuraRein Apr 11, 2026
6e05cf9
perf opt
IwakuraRein Apr 13, 2026
ef5386e
guard expandedIdxToPermutedIdx
IwakuraRein Apr 13, 2026
248162a
make sf layout as a template parameter
IwakuraRein Apr 13, 2026
2b303a9
cache the local amax in smem
IwakuraRein Apr 13, 2026
56f1f6a
use 256 bit vectorized load; create python binding
IwakuraRein Apr 14, 2026
b60d0f8
no need for inline ptx for vectorized loading
IwakuraRein Apr 14, 2026
bfe8d2b
update api; default per_token_scale to None; update artifacts
IwakuraRein Apr 15, 2026
a9b4968
kernel code clean up
IwakuraRein Apr 15, 2026
9e37f75
reduce test memory usage
IwakuraRein Apr 15, 2026
f164ca9
Merge remote-tracking branch 'upstream/main' into per-token-fp4
IwakuraRein Apr 15, 2026
884500f
update checksum
IwakuraRein Apr 15, 2026
3ecaf6f
address comments
IwakuraRein Apr 15, 2026
1225fdd
typo
IwakuraRein Apr 15, 2026
691a6ee
add fp32 quantization kernel
IwakuraRein Apr 16, 2026
bbe4e17
Merge branch 'main' into per-token-fp4
IwakuraRein Apr 20, 2026
5befebd
fix test_scaled_fp4_grouped_quantize
IwakuraRein Apr 21, 2026
c0053de
Merge branch 'main' into per-token-fp4
IwakuraRein Apr 22, 2026
e589c15
filter out pertoken kenrels to speedup
IwakuraRein Apr 22, 2026
a28051e
fix typo
IwakuraRein Apr 22, 2026
311843b
fix typo
IwakuraRein Apr 22, 2026
52079ea
Merge branch 'main' into per-token-fp4
IwakuraRein Apr 23, 2026
39781df
include cub
IwakuraRein Apr 23, 2026
5a0e111
fix compile error
IwakuraRein Apr 24, 2026
17bc44e
remove cub::Max{} as Flashinfer is using cccl 3.0
IwakuraRein Apr 24, 2026
8bf05fc
add usePerChannelScaling option
IwakuraRein Apr 24, 2026
f10fc4c
minor fix
IwakuraRein Apr 24, 2026
99ad4bc
Merge branch 'main' into siyuan/per-token-fp4
zianglih Apr 27, 2026
014327a
Add TE_EXACT_NVFP4 for bitwise align with TE
zianglih Apr 27, 2026
e0358d2
Fold api and expand test
zianglih Apr 27, 2026
ab993f4
Simplify te reference
zianglih Apr 27, 2026
1b3ae2b
Clean up moe test
zianglih Apr 27, 2026
c6eb12d
Tighten tests
zianglih Apr 27, 2026
40379ee
revert the cubin hash
IwakuraRein Apr 29, 2026
00bd14a
Merge branch 'main' into per-token-fp4
IwakuraRein Apr 29, 2026
29a783c
cache input values in smem
IwakuraRein Apr 30, 2026
f553c49
dont cache in smem; change block size to 128
IwakuraRein Apr 30, 2026
f309baf
Merge remote-tracking branch 'upstream/main' into per-token-opt
IwakuraRein May 1, 2026
fed5615
update intrinsic
IwakuraRein May 4, 2026
cdb1ed4
update intrinsic
IwakuraRein May 4, 2026
c76aa26
add launch control; revert math intrinsic
IwakuraRein May 5, 2026
f13c230
replace TE_EXACT_FP4 with environment TRTLLM_DISABLE_FP4_QUANT_FAST_MATH
IwakuraRein May 5, 2026
ee8bdd2
Merge remote-tracking branch 'upstream/main' into per-token-opt
IwakuraRein May 5, 2026
573cc7a
automatically set env in the test
IwakuraRein May 5, 2026
62dadb3
fix select_kernel_fp8
IwakuraRein May 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions csrc/nv_internal/cpp/common/envUtils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -356,4 +356,6 @@ int getEnvMoeA2ACombineBlockSize() {

bool getEnvEplbForceGdrcopy() { return getBoolEnv("TRTLLM_EPLB_FORCE_GDRCOPY"); }

bool getEnvDisableFP4QuantFastMath() { return getBoolEnv("TRTLLM_DISABLE_FP4_QUANT_FAST_MATH"); }
Comment thread
IwakuraRein marked this conversation as resolved.
Resolved

} // namespace tensorrt_llm::common
68 changes: 40 additions & 28 deletions csrc/nv_internal/cpp/kernels/quantization.cu
Original file line number Diff line number Diff line change
Expand Up @@ -236,17 +236,18 @@ CUtensorMap make_3d_tma_copy_desc(T* global_address, uint64_t gmem_dim[3],
////////////////////////////////////////////////////////////////////////////////////////////////////
// Per-token Nvfp4 kernel

#define DISPATCH_NVP4_QUANT_AND_PER_TOKEN_SCALE_KERNEL(SF_LAYOUT) \
#define DISPATCH_NVP4_QUANT_AND_PER_TOKEN_SCALE_KERNEL(SF_LAYOUT, DISABLE_FP4_QUANT_FAST_MATH) \
if constexpr (std::is_same_v<T, float>) \
nvfp4QuantAndPerTokenScaleFP32Kernel<BLOCK_SIZE, QuantizationSFLayout::SF_LAYOUT, \
CACHE_LOCAL_AMAX><<<grid, block, smem_size, stream>>>( \
m, n, input, globalScaleInv, expandedIdxToPermutedIdx, weightOutput, scaleOutput, \
perTokenScaleOutput); \
nvfp4QuantAndPerTokenScaleFP32Kernel<BLOCK_SIZE, QuantizationSFLayout::SF_LAYOUT> \
<<<grid, block, smem_size, stream>>>(m, n, input, globalScaleInv, \
expandedIdxToPermutedIdx, weightOutput, scaleOutput, \
perTokenScaleOutput); \
else \
nvfp4QuantAndPerTokenScaleKernel<T, BLOCK_SIZE, QuantizationSFLayout::SF_LAYOUT, \
CACHE_LOCAL_AMAX, true><<<grid, block, smem_size, stream>>>( \
m, n, input, globalScaleInv, expandedIdxToPermutedIdx, weightOutput, scaleOutput, \
perTokenScaleOutput);
/*CACHE_INPUT*/ false, DISABLE_FP4_QUANT_FAST_MATH> \
<<<grid, block, smem_size, stream>>>(m, n, input, globalScaleInv, \
expandedIdxToPermutedIdx, weightOutput, scaleOutput, \
perTokenScaleOutput);

template <typename T>
void invokeNvfp4QuantAndPerTokenScale(uint32_t m, uint32_t n, T const* input, float globalScaleInv,
Expand All @@ -255,26 +256,43 @@ void invokeNvfp4QuantAndPerTokenScale(uint32_t m, uint32_t n, T const* input, fl
QuantizationSFLayout sfLayout, cudaStream_t stream) {
// Kernel packs 16 values per thread via PackedVec load/store.
TLLM_CHECK_WITH_INFO(n % 16 == 0, "n must be a multiple of 16 for NVFP4 quantization");
// TODO(siyuan): cache local amax in registers.
// currently caching the amax doesn't bring perf improvement.
constexpr bool CACHE_LOCAL_AMAX = std::is_same_v<T, float>;
constexpr uint32_t ELTS_PER_THREAD =
std::is_same_v<T, float> ? 8 : CVT_FP16_TO_FP4_ELTS_PER_THREAD;

constexpr uint32_t BLOCK_SIZE = 256;
uint32_t smem_size =
CACHE_LOCAL_AMAX ? n / ELTS_PER_THREAD * sizeof(float) : 0; // for caching the local amax
// bf16/fp16 path caches input vectors in shared memory (n*sizeof(T) bytes per block);
// fp32 path caches per-vec scales (n/SF_VEC_SIZE floats, but we conservatively allocate
// the same n/ELTS_PER_THREAD floats as before to keep this simple).
constexpr uint32_t ELTS_PER_THREAD = std::is_same_v<T, float> ? 8 : 16;

constexpr uint32_t BLOCK_SIZE = 128;
uint32_t smem_size;
if constexpr (std::is_same_v<T, float>) {
smem_size = n / ELTS_PER_THREAD * sizeof(float);
} else {
// caching input in shared memory doesn't improve speed.
smem_size = 0;
}
Comment thread
IwakuraRein marked this conversation as resolved.
Resolved
dim3 block(BLOCK_SIZE);
dim3 grid(m);
bool disableFP4QuantFastMath = tensorrt_llm::common::getEnvDisableFP4QuantFastMath();
switch (sfLayout) {
case QuantizationSFLayout::LINEAR:
DISPATCH_NVP4_QUANT_AND_PER_TOKEN_SCALE_KERNEL(LINEAR);
if (disableFP4QuantFastMath) {
DISPATCH_NVP4_QUANT_AND_PER_TOKEN_SCALE_KERNEL(LINEAR, true);
} else {
DISPATCH_NVP4_QUANT_AND_PER_TOKEN_SCALE_KERNEL(LINEAR, false);
}
break;
case QuantizationSFLayout::SWIZZLED_128x4:
DISPATCH_NVP4_QUANT_AND_PER_TOKEN_SCALE_KERNEL(SWIZZLED_128x4);
if (disableFP4QuantFastMath) {
DISPATCH_NVP4_QUANT_AND_PER_TOKEN_SCALE_KERNEL(SWIZZLED_128x4, true);
} else {
DISPATCH_NVP4_QUANT_AND_PER_TOKEN_SCALE_KERNEL(SWIZZLED_128x4, false);
}
break;
case QuantizationSFLayout::SWIZZLED_8x4:
DISPATCH_NVP4_QUANT_AND_PER_TOKEN_SCALE_KERNEL(SWIZZLED_8x4);
if (disableFP4QuantFastMath) {
DISPATCH_NVP4_QUANT_AND_PER_TOKEN_SCALE_KERNEL(SWIZZLED_8x4, true);
} else {
DISPATCH_NVP4_QUANT_AND_PER_TOKEN_SCALE_KERNEL(SWIZZLED_8x4, false);
}
break;
default:
TLLM_CHECK_WITH_INFO(false,
Expand Down Expand Up @@ -575,14 +593,11 @@ __global__ void block_scale_interleave_kernel(int numBatches, int numRows, int n
sf = SFIn[inOffset];
}

std::optional<int> batchIdxOpt = batchIdx;
std::optional<int> numRowsOpt = numRows;

// Without batching, the math in get_sf_out_offset is the same as
// int const numSfTilesK = (numCols + 4 - 1) / 4;
// int const tileOffset = ((mi / 128) * numSfTilesK + ki / 4) * 512;
// int const dstIdx = tileOffset + (mi % 32) * 16 + ((mi % 128) / 32) * 4 + ki % 4;
auto dstIdx = get_sf_out_offset_128x4(batchIdxOpt, rowIdx, colIdx, numRowsOpt, numCols);
auto dstIdx = get_sf_out_offset_128x4(batchIdx, rowIdx, colIdx, numRows, numCols);
SFOutput[dstIdx] = sf;
}
}
Expand All @@ -594,11 +609,8 @@ __global__ void block_scale_interleave_reverse_kernel(int numBatches, int numRow
for (int rowIdx = blockIdx.x; rowIdx < numRows; rowIdx += gridDim.x) {
for (int batchIdx = 0; batchIdx < numBatches; batchIdx++) {
for (int colIdx = threadIdx.x; colIdx < numCols; colIdx += blockDim.x) {
std::optional<int> batchIdxOpt = batchIdx;
std::optional<int> numRowsOpt = numRows;

// Get the swizzled input index using the same swizzling pattern
auto srcIdx = get_sf_out_offset_128x4(batchIdxOpt, rowIdx, colIdx, numRowsOpt, numCols);
auto srcIdx = get_sf_out_offset_128x4(batchIdx, rowIdx, colIdx, numRows, numCols);
auto sf = SFIn[srcIdx];

// Output goes to linear layout
Expand Down
3 changes: 3 additions & 0 deletions csrc/nv_internal/tensorrt_llm/common/envUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -101,4 +101,7 @@ int getEnvMoeA2ADispatchBlockSize();
// Block size (threads per block) for MoE A2A Combine kernels (default 256 if unset or invalid)
int getEnvMoeA2ACombineBlockSize();

// Disable the fast fp4 quantization math and align with the TransformerEngine
bool getEnvDisableFP4QuantFastMath();

} // namespace tensorrt_llm::common
122 changes: 66 additions & 56 deletions csrc/nv_internal/tensorrt_llm/kernels/quantization.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -686,49 +686,55 @@ cvt_fp16_to_fp4_expert(
__global__ void block_scale_interleave_kernel(int numbatches, int numRows, int numCols,
uint8_t const* SFIn, uint8_t* SFOutput);

template <typename T, uint32_t BLOCK_SIZE, QuantizationSFLayout SF_LAYOUT, bool CACHE_LOCAL_AMAX,
bool TE_EXACT_NVFP4 = false>
template <typename T, uint32_t BLOCK_SIZE, QuantizationSFLayout SF_LAYOUT, bool CACHE_INPUT = true,
bool DISABLE_FP4_QUANT_FAST_MATH = false>
__global__ void nvfp4QuantAndPerTokenScaleKernel(
// input
uint32_t m, uint32_t n, T const* input, float globalScaleInv, int32_t* expandedIdxToPermutedIdx,
// output
uint8_t* weightOutput, uint8_t* scaleOutput, float* perTokenScaleOutput) {
static constexpr int ELTS_PER_THREAD = CVT_FP16_TO_FP4_ELTS_PER_THREAD;
static constexpr int SF_VEC_SIZE = 16;
static constexpr int NUM_THREADS_PER_SF = SF_VEC_SIZE / ELTS_PER_THREAD; // 2
constexpr int SF_VEC_SIZE = 16;
int rowIdx = blockIdx.x;
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
cudaGridDependencySynchronize();
#endif
if (rowIdx >= m) return;
if (expandedIdxToPermutedIdx != nullptr) {
rowIdx = expandedIdxToPermutedIdx[rowIdx];
}
if (rowIdx < 0) return;
extern __shared__ float
localAmaxSmem[]; // n / ELTS_PER_THREAD float values to store all local amax
using VecType = PackedVec<T, ELTS_PER_THREAD>; // bf16x8
using PackedFp4Type = std::conditional_t<ELTS_PER_THREAD == 16, uint64_t, uint32_t>;
VecType vec;
uint8_t fp8Scale{0};
PackedFp4Type fp4Vals{0};
using InType = PackedVec<T, SF_VEC_SIZE>; // bf16x8
using PackedFp4Type = std::conditional_t<SF_VEC_SIZE == 16, uint64_t, uint32_t>;
InType vec_in;

// Cache the input vectors in shared memory so the quantization pass does not
// re-read them from global memory. Caching only the local amax doesn't help because
// the dominant cost is the second gmem read of `input`, not the amax recomputation.
// Mirrors the smem-caching pattern in `perTokenQuantization` above.
// Sized for `num_vecs_per_row * sizeof(InType)` bytes, set by the launcher.
extern __shared__ uint32_t smem[];
constexpr uint32_t BANKS_PER_THREAD = sizeof(InType) / sizeof(uint32_t);

float localAmax = 0.f;
uint32_t num_vecs_per_row = (n + ELTS_PER_THREAD - 1) / ELTS_PER_THREAD;
for (uint32_t vecIdx = threadIdx.x; vecIdx < num_vecs_per_row; vecIdx += blockDim.x) {
int64_t vecOffset = rowIdx * num_vecs_per_row + vecIdx;
loadPackedVec(vec, reinterpret_cast<VecType const*>(input) + vecOffset);
#pragma unroll
for (int i = 0; i < ELTS_PER_THREAD / 2; ++i) {
auto element = cuda_abs(vec.elts[i]);
localAmax = fmaxf(localAmax, static_cast<float>(cuda_max(element.x, element.y)));
}
uint32_t num_vecs_per_row = (n + SF_VEC_SIZE - 1) / SF_VEC_SIZE;

if constexpr (CACHE_LOCAL_AMAX) {
if constexpr (NUM_THREADS_PER_SF > 1) {
// use warp shuffle to get the amax of 16 elements and store it to SMEM
localAmax =
fmaxf(__shfl_xor_sync(__activemask(), localAmax, NUM_THREADS_PER_SF / 2), localAmax);
for (uint32_t vecIdx = threadIdx.x; vecIdx < num_vecs_per_row; vecIdx += BLOCK_SIZE) {
int64_t vecOffset = static_cast<int64_t>(rowIdx) * num_vecs_per_row + vecIdx;
loadPackedVec(vec_in, reinterpret_cast<InType const*>(input) + vecOffset);
if constexpr (CACHE_INPUT) {
#pragma unroll
for (uint32_t bank = 0; bank < BANKS_PER_THREAD; ++bank) {
constexpr uint32_t STRIDE = BANKS_PER_THREAD + 1;
auto reg = reinterpret_cast<uint32_t*>(&vec_in)[bank];
smem[vecIdx * STRIDE + bank] = reg;
}
localAmaxSmem[vecIdx] = localAmax;
}
std::remove_reference_t<decltype(vec_in.elts[0])> localAmax_(0.f, 0.f);
#pragma unroll
for (int i = 0; i < SF_VEC_SIZE / 2; ++i) {
localAmax_ = __hmax2(localAmax_, __habs2(vec_in.elts[i]));
}
localAmax = fmaxf(localAmax, static_cast<float>(__hmax(localAmax_.x, localAmax_.y)));
}

using BlockReduce = cub::BlockReduce<float, BLOCK_SIZE>;
Expand All @@ -737,7 +743,7 @@ __global__ void nvfp4QuantAndPerTokenScaleKernel(

float perTokenScale;
float globalEncodeScale;
if constexpr (TE_EXACT_NVFP4) {
if constexpr (DISABLE_FP4_QUANT_FAST_MATH) {
if (threadIdx.x == 0) {
float const globalScale = __fdiv_rn(1.0f, globalScaleInv);
float const rowEncodeScale =
Expand All @@ -761,42 +767,46 @@ __global__ void nvfp4QuantAndPerTokenScaleKernel(
globalEncodeScale = reciprocal_approximate_ftz(perTokenScale);
}

// quantize to fp4 with per-token scale
for (uint32_t vecIdx = threadIdx.x; vecIdx < num_vecs_per_row; vecIdx += blockDim.x) {
int64_t vecOffset = rowIdx * num_vecs_per_row + vecIdx;
loadPackedVec(vec, reinterpret_cast<VecType const*>(input) + vecOffset);

if constexpr (CACHE_LOCAL_AMAX) {
localAmax = localAmaxSmem[vecIdx];
fp4Vals =
cvt_warp_fp16_to_fp4_with_vec_max<T, SF_VEC_SIZE, ELTS_PER_THREAD, false, TE_EXACT_NVFP4>(
vec, globalEncodeScale, perTokenScale, localAmax, &fp8Scale);
// quantize to fp4 with per-token scale, reading inputs from smem (each thread reads
// back the same indices it wrote, so no extra barrier is needed beyond the existing
// syncthreads after the block reduction / per-token scale broadcast).
for (uint32_t vecIdx = threadIdx.x; vecIdx < num_vecs_per_row; vecIdx += BLOCK_SIZE) {
uint32_t vecOffset = rowIdx * num_vecs_per_row + vecIdx;
Comment thread
IwakuraRein marked this conversation as resolved.
Resolved
if constexpr (CACHE_INPUT) {
#pragma unroll
for (uint32_t bank = 0; bank < BANKS_PER_THREAD; ++bank) {
constexpr uint32_t STRIDE = BANKS_PER_THREAD + 1;
auto reg = smem[vecIdx * STRIDE + bank];
reinterpret_cast<uint32_t*>(&vec_in)[bank] = reg;
}
} else {
fp4Vals = cvt_warp_fp16_to_fp4<T, SF_VEC_SIZE, ELTS_PER_THREAD, false, TE_EXACT_NVFP4>(
vec, globalEncodeScale, &fp8Scale);
loadPackedVec(vec_in, reinterpret_cast<InType const*>(input) + vecOffset);
}
uint8_t fp8Scale;
auto fp4Vals =
cvt_warp_fp16_to_fp4<T, SF_VEC_SIZE, SF_VEC_SIZE, false, DISABLE_FP4_QUANT_FAST_MATH>(
vec_in, globalEncodeScale, &fp8Scale);
reinterpret_cast<PackedFp4Type*>(weightOutput)[vecOffset] = fp4Vals;

if (threadIdx.x % NUM_THREADS_PER_SF == 0) {
uint32_t num_sf_vecs_per_row = (n + SF_VEC_SIZE - 1) / SF_VEC_SIZE;
auto sfVecIdx = vecIdx / NUM_THREADS_PER_SF;
int64_t sfOffset;
if constexpr (SF_LAYOUT == QuantizationSFLayout::LINEAR) {
sfOffset = rowIdx * num_sf_vecs_per_row + sfVecIdx;
} else if constexpr (SF_LAYOUT == QuantizationSFLayout::SWIZZLED_128x4) {
sfOffset = get_sf_out_offset_128x4(std::nullopt, rowIdx, sfVecIdx, m, num_sf_vecs_per_row);
} else {
sfOffset = get_sf_out_offset_8x4(std::nullopt, rowIdx, sfVecIdx, m, num_sf_vecs_per_row);
}
scaleOutput[sfOffset] = fp8Scale;
uint32_t sfOffset;
if constexpr (SF_LAYOUT == QuantizationSFLayout::LINEAR) {
sfOffset = rowIdx * num_vecs_per_row + vecIdx;
Comment thread
IwakuraRein marked this conversation as resolved.
Resolved
} else if constexpr (SF_LAYOUT == QuantizationSFLayout::SWIZZLED_128x4) {
sfOffset = get_sf_out_offset_128x4(rowIdx, vecIdx, num_vecs_per_row);
} else {
sfOffset = get_sf_out_offset_8x4(rowIdx, vecIdx, num_vecs_per_row);
}
scaleOutput[sfOffset] = fp8Scale;
}
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
cudaTriggerProgrammaticLaunchCompletion();
#endif
}

// Fast approximation of nvfp4 quantization.
// This kernel first quantizes the input to fp4 with local amax only,
// then calculates the e4m3 scales with the global amax and cached local amax.
template <uint32_t BLOCK_SIZE, QuantizationSFLayout SF_LAYOUT, bool CACHE_LOCAL_AMAX>
template <uint32_t BLOCK_SIZE, QuantizationSFLayout SF_LAYOUT>
__global__ void nvfp4QuantAndPerTokenScaleFP32Kernel(
// input
uint32_t m, uint32_t n, float const* input, float globalScaleInv,
Expand Down Expand Up @@ -872,13 +882,13 @@ __global__ void nvfp4QuantAndPerTokenScaleFP32Kernel(
float localScale = localScaleSmem[vecIdx];
float fp32Scale = reciprocal_approximate_ftz(perTokenScale * localScale);
fp8Scale = __nv_fp8_e4m3(fp32Scale).__x;
int64_t sfOffset;
uint32_t sfOffset;
if constexpr (SF_LAYOUT == QuantizationSFLayout::LINEAR) {
sfOffset = rowIdx * num_sf_vecs_per_row + vecIdx;
Comment thread
IwakuraRein marked this conversation as resolved.
Resolved
} else if constexpr (SF_LAYOUT == QuantizationSFLayout::SWIZZLED_128x4) {
sfOffset = get_sf_out_offset_128x4(std::nullopt, rowIdx, vecIdx, m, num_sf_vecs_per_row);
sfOffset = get_sf_out_offset_128x4(rowIdx, vecIdx, num_sf_vecs_per_row);
} else {
sfOffset = get_sf_out_offset_8x4(std::nullopt, rowIdx, vecIdx, m, num_sf_vecs_per_row);
sfOffset = get_sf_out_offset_8x4(rowIdx, vecIdx, num_sf_vecs_per_row);
}
scaleOutput[sfOffset] = fp8Scale;
}
Expand Down
2 changes: 1 addition & 1 deletion csrc/nv_internal/tensorrt_llm/kernels/quantization.h
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ template <typename T>
void invokeNvfp4QuantAndPerTokenScale(uint32_t m, uint32_t n, T const* input, float globalScaleInv,
int32_t* expanded_idx_to_permuted_idx, uint8_t* weightOutput,
uint8_t* scaleOutput, float* perTokenScaleOutput,
QuantizationSFLayout sfLayout, cudaStream_t stream);
QuantizationSFLayout sfLayout, cudaStream_t stream = 0);

template <typename T>
void invokeBlockScaleInterleave(int b, int m, int m_padded, int n, int n_padded, T const* SFIn,
Expand Down
Loading
Loading