Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions csrc/cake_kimi_k3_mla/.clang-format
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Receipt-bound generated sources (Cake export): bytes are attested by the source registry.
DisableFormat: true
SortIncludes: Never
18 changes: 18 additions & 0 deletions csrc/cake_kimi_k3_mla/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# CAKE Kimi-K3 MLA FP8 paged attention (SM100 / SM103)

Generated CUDA for `flashinfer.mla.cake_kimi_k3_mla` (`trtllm_batch_decode_with_kv_cache_mla(...,
backend="cake")`): Kimi-K3 multi-head latent attention over an FP8 (E4M3) paged latent cache
(latent 512 + rope 64, page size 64), FP8 query, BF16 output. Two attention schedules: one
swapped-AB tcgen05 kernel per row tile (`main_rt16` .. `main_rt96`: packed (token, head) rows per
CTA) and the two-CTA wide kernel (`main_wide`: 128 packed rows per cluster of two CTAs, K tokens
split across the pair; taken for requests with more than 64 packed rows whose longest KV is at
least 16384 tokens), plus the split-KV merge kernels shared by both (`reduce_w1` / `reduce_w2` /
`reduce_w4` warp-per-row reducers for `num_split <= 32`, `reduce_cta` otherwise). The wide
kernel is a cluster kernel; its generated binding sets the cluster launch attribute (2, 1, 1).

* Source of truth: Cake `loom/examples/weave/kimi_k3_mla_fp8_paged_attention.py` (row tiles, host
planner) and `loom/examples/weave/kimi_k3_mla_wide.py` (wide kernel) (Linear CAKE-645).
* Exporter: Cake `exports/kimi_k3_mla_fp8_paged_attention/export.py` writes `sm_100a/` and
`sm_103a/` (`*_kernel.cu` + `*_binding.cu` per physical module) and fills `MODULES` / `ROUTES` in
`flashinfer/jit/cake_kimi_k3_mla.py`.
* Do not edit the generated files by hand; regenerate through the exporter.
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
/*
* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

// Generated by the Cake source exporter — do not edit.
// tvm-ffi direct-source launcher for Cake kernel 'kernel_cake_kimi_k3_mla_fp8_paged_attention_0ce2970cff662167c83a'.
#include <cuda.h>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cuda_runtime.h>

#include "tvm_ffi_utils.h"

#include <atomic>
#include <cstdint>
#include <cstring>
#include <mutex>
#include <string>
#include <unordered_map>
#include <vector>

extern "C" __global__ void kernel_cake_kimi_k3_mla_fp8_paged_attention_0ce2970cff662167c83a(__nv_bfloat16* __restrict__ partial_O, float* __restrict__ partial_max, float* __restrict__ partial_sum, __nv_bfloat16* __restrict__ O, int* __restrict__ cum_seq_lens_q, int batch, int num_heads, int num_split, float bmm2_scale);


namespace cake_host_shim_3fce8f9eb39c01d7 {

using tvm::ffi::Optional;
using tvm::ffi::TensorView;

class ScopedCudaDevice {
public:
explicit ScopedCudaDevice(int device_id) {
cudaError_t error = cudaGetDevice(&previous_device_);
TVM_FFI_CHECK(error == cudaSuccess, RuntimeError)
<< "cudaGetDevice failed before host-shim launch: cudaError="
<< static_cast<int>(error);
if (previous_device_ != device_id) {
error = cudaSetDevice(device_id);
TVM_FFI_CHECK(error == cudaSuccess, RuntimeError)
<< "cudaSetDevice failed before host-shim launch for cuda:"
<< device_id << ": cudaError=" << static_cast<int>(error);
restore_ = true;
}
}

ScopedCudaDevice(const ScopedCudaDevice&) = delete;
ScopedCudaDevice& operator=(const ScopedCudaDevice&) = delete;

~ScopedCudaDevice() noexcept {
if (restore_) {
(void)cudaSetDevice(previous_device_);
}
}

private:
int previous_device_ = -1;
bool restore_ = false;
};

inline int64_t CakeDeviceMultiprocessorCount(int device_id) {
constexpr int kMaxCachedCudaDevices = 64;
TVM_FFI_CHECK(device_id >= 0 && device_id < kMaxCachedCudaDevices, RuntimeError)
<< "physical-SM-count cache does not cover cuda:" << device_id;
static std::atomic<int> count_by_device[kMaxCachedCudaDevices]{};
int cached = count_by_device[device_id].load(std::memory_order_acquire);
if (cached > 0) return cached;

int count = 0;
cudaError_t error = cudaDeviceGetAttribute(
&count, cudaDevAttrMultiProcessorCount, device_id);
TVM_FFI_CHECK(error == cudaSuccess && count > 0, RuntimeError)
<< "querying multiProcessorCount failed for cuda:" << device_id
<< ": cudaError=" << static_cast<int>(error) << ", count=" << count;
// Concurrent first launches may repeat the immutable device query, but all
// publication is atomic and every later hot-path lookup is one acquire load.
count_by_device[device_id].store(count, std::memory_order_release);
return count;
}

inline void CheckCudaTensor(const TensorView& t, const char* name) {
TVM_FFI_CHECK(t.device().device_type == kDLCUDA, ValueError)
<< name << " must be a CUDA tensor, got device_type=" << (int)t.device().device_type;
}

inline void CheckSameCudaDevice(
const TensorView& t,
const TensorView& reference,
const char* name,
const char* reference_name) {
TVM_FFI_CHECK(t.device().device_id == reference.device().device_id, ValueError)
<< name << " must be on the same CUDA device as " << reference_name
<< ": got cuda:" << t.device().device_id
<< " versus cuda:" << reference.device().device_id;
}

inline void CheckCurrentCudaDevice(
const TensorView& reference,
const char* reference_name) {
int current_device = -1;
cudaError_t error = cudaGetDevice(&current_device);
TVM_FFI_CHECK(error == cudaSuccess, RuntimeError)
<< "cudaGetDevice failed while validating " << reference_name
<< ": cudaError=" << static_cast<int>(error);
TVM_FFI_CHECK(current_device == reference.device().device_id, ValueError)
<< "current CUDA device must match " << reference_name
<< ": current=cuda:" << current_device
<< ", tensor=cuda:" << reference.device().device_id;
}

inline void CheckContiguous(const TensorView& t, const char* name) {
TVM_FFI_CHECK(t.IsContiguous(), ValueError) << name << " must be contiguous";
}

inline void CheckDtype(const TensorView& t, const char* name, int code, int bits, int lanes) {
DLDataType d = t.dtype();
TVM_FFI_CHECK((int)d.code == code && (int)d.bits == bits && (int)d.lanes == lanes, TypeError)
<< name << " dtype mismatch: expected DLDataType(code=" << code << ", bits=" << bits
<< ", lanes=" << lanes << "), got (code=" << (int)d.code << ", bits=" << (int)d.bits
<< ", lanes=" << (int)d.lanes << ")";
}

// A logical axis.outer(trailing) folds every source dim above the trailing
// dimensions. Shape products are independent of physical strides, so verify
// the leading dimensions form one dense row-major chain instead of inventing
// a "folded stride". The descriptor reads its exact adjacent physical step
// separately through stride[-(trailing + 1)].
inline void CheckDenseLeadingFold(const TensorView& t, int trailing, const char* name) {
TVM_FFI_CHECK(trailing > 0 && t.ndim() >= trailing, ValueError)
<< name << " cannot fold leading dimensions above " << trailing
<< " trailing dims from ndim=" << t.ndim();
int outer_last = t.ndim() - trailing - 1;
if (outer_last <= 0) {
return;
}
int64_t step = t.stride(outer_last);
TVM_FFI_CHECK(step > 0, ValueError)
<< name << " physical strides must be positive";
int64_t expected = step;
for (int axis = outer_last - 1; axis >= 0; --axis) {
expected *= t.size(axis + 1);
if (t.size(axis) > 1) {
TVM_FFI_CHECK(t.stride(axis) == expected, ValueError)
<< name << " leading dims are not physically foldable above " << trailing
<< " trailing dims: stride(" << axis << ")=" << t.stride(axis)
<< ", expected " << expected;
}
}
}

void Run(TensorView arg_partial_O, TensorView arg_partial_max, TensorView arg_partial_sum, TensorView arg_O, TensorView arg_cum_seq_lens_q, int64_t arg_batch, int64_t arg_num_heads, int64_t arg_num_split, double arg_bmm2_scale, int64_t grid_x, int64_t grid_y, int64_t grid_z) {
DLDevice dev = arg_partial_O.device();
ScopedCudaDevice device_guard(dev.device_id);
CheckCudaTensor(arg_partial_O, "partial_O");
CheckDtype(arg_partial_O, "partial_O", 4, 16, 1);
CheckContiguous(arg_partial_O, "partial_O");
CheckCudaTensor(arg_partial_max, "partial_max");
CheckDtype(arg_partial_max, "partial_max", 2, 32, 1);
CheckContiguous(arg_partial_max, "partial_max");
CheckCudaTensor(arg_partial_sum, "partial_sum");
CheckDtype(arg_partial_sum, "partial_sum", 2, 32, 1);
CheckContiguous(arg_partial_sum, "partial_sum");
CheckCudaTensor(arg_O, "O");
CheckDtype(arg_O, "O", 4, 16, 1);
CheckContiguous(arg_O, "O");
CheckCudaTensor(arg_cum_seq_lens_q, "cum_seq_lens_q");
CheckDtype(arg_cum_seq_lens_q, "cum_seq_lens_q", 0, 32, 1);
CheckContiguous(arg_cum_seq_lens_q, "cum_seq_lens_q");
TVM_FFI_CHECK(arg_batch >= -2147483648LL && arg_batch <= 2147483647LL, ValueError)
<< "scalar 'batch' value " << arg_batch
<< " is outside i32 range [-2147483648, 2147483647]";
TVM_FFI_CHECK(arg_num_heads >= -2147483648LL && arg_num_heads <= 2147483647LL, ValueError)
<< "scalar 'num_heads' value " << arg_num_heads
<< " is outside i32 range [-2147483648, 2147483647]";
TVM_FFI_CHECK(arg_num_split >= -2147483648LL && arg_num_split <= 2147483647LL, ValueError)
<< "scalar 'num_split' value " << arg_num_split
<< " is outside i32 range [-2147483648, 2147483647]";
CheckSameCudaDevice(arg_partial_max, arg_partial_O, "partial_max", "partial_O");
CheckSameCudaDevice(arg_partial_sum, arg_partial_O, "partial_sum", "partial_O");
CheckSameCudaDevice(arg_O, arg_partial_O, "O", "partial_O");
CheckSameCudaDevice(arg_cum_seq_lens_q, arg_partial_O, "cum_seq_lens_q", "partial_O");
TVM_FFI_CHECK(grid_x > 0 && grid_y > 0 && grid_z > 0, ValueError)
<< "launch grid dimensions must be positive, got (" << grid_x << ", " << grid_y
<< ", " << grid_z << ")";

cudaStream_t stream = (cudaStream_t)TVMFFIEnvGetStream(dev.device_type, dev.device_id);
__nv_bfloat16* p_partial_O = static_cast<__nv_bfloat16*>(arg_partial_O.data_ptr());
float* p_partial_max = static_cast<float*>(arg_partial_max.data_ptr());
float* p_partial_sum = static_cast<float*>(arg_partial_sum.data_ptr());
__nv_bfloat16* p_O = static_cast<__nv_bfloat16*>(arg_O.data_ptr());
int* p_cum_seq_lens_q = static_cast<int*>(arg_cum_seq_lens_q.data_ptr());
int32_t v_batch = (int32_t)arg_batch;
int32_t v_num_heads = (int32_t)arg_num_heads;
int32_t v_num_split = (int32_t)arg_num_split;
float v_bmm2_scale = (float)arg_bmm2_scale;
void* kargs[] = {&p_partial_O, &p_partial_max, &p_partial_sum, &p_O, &p_cum_seq_lens_q, &v_batch, &v_num_heads, &v_num_split, &v_bmm2_scale};


dim3 grid((uint32_t)grid_x, (uint32_t)grid_y, (uint32_t)grid_z);
dim3 block(256u, 1u, 1u);

// Extended direct-source launch for a linked CUDA kernel symbol.
cudaLaunchAttribute attrs[1]{};
int n = 0;
attrs[n].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[n].val.programmaticStreamSerializationAllowed = 1;
++n;
cudaLaunchConfig_t config{};
config.gridDim = grid;
config.blockDim = block;
config.dynamicSmemBytes = 0u;
config.stream = stream;
config.attrs = attrs;
config.numAttrs = n;
cudaError_t launch_status = cudaLaunchKernelExC(
&config, reinterpret_cast<const void*>(kernel_cake_kimi_k3_mla_fp8_paged_attention_0ce2970cff662167c83a), kargs);
TVM_FFI_CHECK(launch_status == cudaSuccess, RuntimeError)
<< "cudaLaunchKernelExC for kernel_cake_kimi_k3_mla_fp8_paged_attention_0ce2970cff662167c83a failed: "
<< cudaGetErrorString(launch_status);
}

} // namespace cake_host_shim_3fce8f9eb39c01d7

TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_host_shim_3fce8f9eb39c01d7::Run);
Loading
Loading