[FlyDSL] One-stage split-K for the a8w8 preshuffle GEMM

aiter/aot/flydsl/gemm.py

@@ -97,7 +97,10 @@
     r"(?P<tile_m>\d+)x(?P<tile_n>\d+)x(?P<tile_k>\d+)_"
     r"(?P<qa>[A-Z0-9]+)_(?P<qw>[A-Z0-9]+)_(?P<out>[A-Z0-9]+)_"
     r"(?P<async_copy>\d+)x(?P<waves_per_eu>\d+)(?:x(?P<xcd_swizzle>\d+))?(?:x(?P<lds_stage>\d+))?_"
-    r"(?P<scheduler>[A-Za-z][A-Za-z0-9]*)$"
+    r"(?!ks\d+$)(?P<scheduler>[A-Za-z][A-Za-z0-9]*)"
+    # Trailing _ksN, emitted only for k_split > 1, so pre-split-K names still
+    # match. Without it they fail fullmatch and drop out of the AOT build.
+    r"(?:_ks(?P<k_split>\d+))?$"
 )
 _SHORT_DTYPE = {
     "F8": "fp8",
@@ -147,6 +150,7 @@ def _parse_preshuffle_kernel_name(name: str) -> dict | None:
         "xcd_swizzle": int(m.group("xcd_swizzle")) if m.group("xcd_swizzle") else 0,
         "lds_stage": int(m.group("lds_stage")) if m.group("lds_stage") else 2,
         "scheduler": m.group("scheduler"),
+        "k_split": int(m.group("k_split")) if m.group("k_split") else 1,
     }
 
 
@@ -354,10 +358,12 @@ def _compile_preshuffle_to_cache(
     xcd_swizzle: int = 0,
     lds_stage: int = 2,
     scheduler: str = "Default",
+    k_split: int = 1,
     **kwargs,
 ):
     del kwargs
     enable_scheduler = str(scheduler).lower() != "off"
+    k_split = int(k_split)
 
     import torch
 
@@ -367,7 +373,26 @@ def _compile_preshuffle_to_cache(
     # FlyDSL preshuffle kernels consume raw quantized bytes for fp8/int8 paths.
     a = torch.empty((m * k,), device=dev, dtype=torch.int8)
     b = torch.empty((n * k,), device=dev, dtype=torch.int8)
+    from aiter.ops.flydsl.gemm_kernels import (
+        PRESHUFFLE_SPLIT_K_MAX_TILES,
+        PRESHUFFLE_SPLIT_K_WORKSPACE_ELEMS,
+    )
+
+    # k_split > 1 accumulates fp32 partials in a workspace that the last
+    # arriving split reduces in-kernel, so the launcher takes the workspace,
+    # the final output and the per-tile semaphore. Both are sized to the bounds
+    # the runtime uses, so the AOT signature matches.
     out = torch.empty((m * n,), device=dev, dtype=out_torch_dtype)
+    workspace = (
+        torch.empty(PRESHUFFLE_SPLIT_K_WORKSPACE_ELEMS, device=dev, dtype=torch.float32)
+        if k_split > 1
+        else out
+    )
+    semaphore = torch.zeros(
+        PRESHUFFLE_SPLIT_K_MAX_TILES if k_split > 1 else 0,
+        device=dev,
+        dtype=torch.int32,
+    )
     scale_a = torch.empty((max(m, 1),), device=dev, dtype=torch.float32)
     scale_b = torch.empty((max(n, 1),), device=dev, dtype=torch.float32)
     bias = torch.empty(0, device=dev, dtype=out_torch_dtype)
@@ -386,13 +411,16 @@ def _compile_preshuffle_to_cache(
         enable_scheduler=enable_scheduler,
         xcd_swizzle=xcd_swizzle,
         lds_stage=lds_stage,
+        split_k=k_split,
     )
     # The layout-API launcher uses fx.Tensor args (it builds views via
     # fx.get_iter/make_view), so pass flat torch tensors directly rather
     # than raw pointers (pointer args would fail GetIterOp type checks).
     _compile_executable_to_cache(
         exe,
+        workspace,
         out,
+        semaphore,
         a,
         b,
         scale_a,

aiter/configs/model_configs/a8w8_bpreshuffle_tuned_gemm_kimik3.csv

@@ -1,10 +1,10 @@
 gfx,cu_num,M,N,K,q_dtype_w,libtype,kernelId,splitK,us,kernelName,tflops,bw,errRatio
-gfx950,256,1,576,7168,torch.float8_e4m3fn,flydsl,617,0,11.3863,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x2x0x2_default,0.73,363.34,0.0
-gfx950,256,2,576,7168,torch.float8_e4m3fn,flydsl,373,0,11.4327,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x1x0x2_default,1.44,362.59,0.0
-gfx950,256,4,576,7168,torch.float8_e4m3fn,flydsl,1059,0,11.4331,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x4x0x2_default,2.89,364.03,0.0
-gfx950,256,8,576,7168,torch.float8_e4m3fn,flydsl,1,0,11.3459,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x0x0x2_default,5.82,369.77,0.0
-gfx950,256,16,576,7168,torch.float8_e4m3fn,ck,10,0,10.7183,a8w8_bpreshuffle_128x16x32x512_16x16_16x16_32x4x1_32x4x1_1x16x1x8_4x4x1_1x1_intrawave_v1,12.33,397.63,0.0
-gfx950,256,32,576,7168,torch.float8_e4m3fn,ck,8,0,10.3684,a8w8_bpreshuffle_128x32x16x512_16x16_16x16_32x4x1_32x4x1_1x32x1x4_4x4x1_1x1_intrawave_v1,25.49,423.88,0.0
+gfx950,256,1,576,7168,torch.float8_e4m3fn,flydsl,961,7,5.0341,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x4x0x2_default_ks7,1.64,821.81,0.0
+gfx950,256,2,576,7168,torch.float8_e4m3fn,flydsl,961,7,5.0499,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x4x0x2_default_ks7,3.27,820.89,0.0
+gfx950,256,4,576,7168,torch.float8_e4m3fn,flydsl,2116,14,5.5256,flydsl_bpreshuflle_16x64x256_F8_F8_B16_0x4x0x1_default_ks14,5.98,753.23,0.0
+gfx950,256,8,576,7168,torch.float8_e4m3fn,flydsl,373,7,5.2815,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x1x0x2_default_ks7,12.51,794.34,0.0
+gfx950,256,16,576,7168,torch.float8_e4m3fn,flydsl,373,7,5.4885,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x1x0x2_default_ks7,24.07,776.51,0.0
+gfx950,256,32,576,7168,torch.float8_e4m3fn,flydsl,961,7,6.0091,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x4x0x2_default_ks7,43.97,731.39,0.0
 gfx950,256,64,576,7168,torch.float8_e4m3fn,ck,10,0,10.7167,a8w8_bpreshuffle_128x16x32x512_16x16_16x16_32x4x1_32x4x1_1x16x1x8_4x4x1_1x1_intrawave_v1,49.31,434.95,0.0
 gfx950,256,128,576,7168,torch.float8_e4m3fn,ck,10,0,9.9632,a8w8_bpreshuffle_128x16x32x512_16x16_16x16_32x4x1_32x4x1_1x16x1x8_4x4x1_1x1_intrawave_v1,106.09,521.29,0.0
 gfx950,256,256,576,7168,torch.float8_e4m3fn,ck,11,0,11.4055,a8w8_bpreshuffle_256x16x64x512_16x16_16x16_32x8x1_32x8x1_1x16x1x16_4x4x1_1x1_intrawave_v1,185.34,548.74,0.0
@@ -16,17 +16,17 @@ gfx950,256,8192,576,7168,torch.float8_e4m3fn,cktile,94,0,48.2593,a8w8_bpreshuffl
 gfx950,256,16384,576,7168,torch.float8_e4m3fn,cktile,232,0,75.7943,a8w8_bpreshuffle_cktile_0x0x8x4x1x0x0x0x0x3_224x192x128_1x4x1_16x16x128_default,1784.98,1852.96,0.0
 gfx950,256,32768,576,7168,torch.float8_e4m3fn,ck,68,0,137.1244,a8w8_bpreshuffle_256x128x192x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8x8x1_2x1_intrawave_v3,1973.27,2018.3,0.0
 gfx950,256,1,1536,128,torch.float8_e4m3fn,cktile,15,0,2.8909,a8w8_bpreshuffle_cktile_0x0x8x4x1x0x0x0x0x1_32x64x128_1x4x1_16x16x128_default,0.14,69.12,0.0
-gfx950,256,1,1536,7168,torch.float8_e4m3fn,flydsl,617,0,11.5477,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x2x0x2_default,1.91,954.33,0.0
+gfx950,256,1,1536,7168,torch.float8_e4m3fn,flydsl,187,7,5.8969,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x0x4x2_default_ks7,3.73,1868.83,0.0
 gfx950,256,2,1536,128,torch.float8_e4m3fn,cktile,165,0,2.7639,a8w8_bpreshuffle_cktile_0x0x8x4x1x0x0x0x0x4_32x64x128_1x4x1_16x16x128_default,0.28,73.45,0.0
-gfx950,256,2,1536,7168,torch.float8_e4m3fn,flydsl,617,0,11.1346,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x2x0x2_default,3.96,990.65,0.0
+gfx950,256,2,1536,7168,torch.float8_e4m3fn,flydsl,2117,7,6.0302,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x4x0x1_default_ks7,7.3,1829.21,0.0
 gfx950,256,4,1536,128,torch.float8_e4m3fn,cktile,15,0,2.9594,a8w8_bpreshuffle_cktile_0x0x8x4x1x0x0x0x0x1_32x64x128_1x4x1_16x16x128_default,0.53,70.76,0.0
-gfx950,256,4,1536,7168,torch.float8_e4m3fn,flydsl,617,0,11.2096,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x2x0x2_default,7.86,985.85,0.0
+gfx950,256,4,1536,7168,torch.float8_e4m3fn,flydsl,435,7,5.9077,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x1x4x2_default_ks7,14.91,1870.61,0.0
 gfx950,256,8,1536,128,torch.float8_e4m3fn,cktile,165,0,2.6116,a8w8_bpreshuffle_cktile_0x0x8x4x1x0x0x0x0x4_32x64x128_1x4x1_16x16x128_default,1.2,85.09,0.0
-gfx950,256,8,1536,7168,torch.float8_e4m3fn,flydsl,737,0,11.9393,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x3x0x2_default,14.75,929.03,0.0
+gfx950,256,8,1536,7168,torch.float8_e4m3fn,flydsl,677,7,6.0842,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x2x4x2_default_ks7,28.95,1823.08,0.0
 gfx950,256,16,1536,128,torch.float8_e4m3fn,cktile,15,0,3.4698,a8w8_bpreshuffle_cktile_0x0x8x4x1x0x0x0x0x1_32x64x128_1x4x1_16x16x128_default,1.81,71.42,0.0
-gfx950,256,16,1536,7168,torch.float8_e4m3fn,ck,10,0,10.3683,a8w8_bpreshuffle_128x16x32x512_16x16_16x16_32x4x1_32x4x1_1x16x1x8_4x4x1_1x1_intrawave_v1,33.98,1077.7,0.0
+gfx950,256,16,1536,7168,torch.float8_e4m3fn,flydsl,1059,7,6.2957,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x4x0x2_default_ks7,55.96,1774.84,0.0
 gfx950,256,32,1536,128,torch.float8_e4m3fn,flydsl,1412,0,2.7372,flydsl_bpreshuflle_32x64x128_F8_F8_B16_0x1x0x1_default,4.6,109.24,0.0
-gfx950,256,32,1536,7168,torch.float8_e4m3fn,ck,8,0,11.2815,a8w8_bpreshuffle_128x32x16x512_16x16_16x16_32x4x1_32x4x1_1x32x1x4_4x4x1_1x1_intrawave_v1,62.46,1004.98,0.0
+gfx950,256,32,1536,7168,torch.float8_e4m3fn,flydsl,258,7,7.2576,flydsl_bpreshuflle_32x64x512_F8_F8_B16_0x1x0x2_default_ks7,97.09,1562.19,0.0
 gfx950,256,64,1536,128,torch.float8_e4m3fn,flydsl,1660,0,3.3145,flydsl_bpreshuflle_32x64x128_F8_F8_B16_0x2x0x1_default,7.59,121.11,0.0
 gfx950,256,64,1536,7168,torch.float8_e4m3fn,ck,10,0,11.1903,a8w8_bpreshuffle_128x16x32x512_16x16_16x16_32x4x1_32x4x1_1x16x1x8_4x4x1_1x1_intrawave_v1,125.94,1042.46,0.0
 gfx950,256,128,1536,128,torch.float8_e4m3fn,flydsl,1900,0,2.7756,flydsl_bpreshuflle_32x64x128_F8_F8_B16_0x3x0x1_default,18.13,218.41,0.0
@@ -47,12 +47,12 @@ gfx950,256,16384,1536,128,torch.float8_e4m3fn,flydsl,931,0,15.3636,flydsl_bpresh
 gfx950,256,16384,1536,7168,torch.float8_e4m3fn,flydsl,667,0,156.303,flydsl_bpreshuflle_128x256x128_F8_F8_B16_1x2x0x2_default,2308.19,1143.82,0.0
 gfx950,256,32768,1536,128,torch.float8_e4m3fn,flydsl,213,0,25.7514,flydsl_bpreshuflle_64x256x128_F8_F8_B16_1x0x4x2_default,500.36,4079.55,0.0
 gfx950,256,32768,1536,7168,torch.float8_e4m3fn,ck,33,0,288.0491,a8w8_bpreshuffle_256x256x256x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8x8x1_2x1_intrawave_v3,2504.97,1203.11,0.0
-gfx950,256,1,2176,7168,torch.float8_e4m3fn,flydsl,849,0,11.4632,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x3x0x2_default,2.72,1361.67,0.0
-gfx950,256,2,2176,7168,torch.float8_e4m3fn,flydsl,125,0,11.0924,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x0x0x2_default,5.62,1408.23,0.0
-gfx950,256,4,2176,7168,torch.float8_e4m3fn,flydsl,373,0,11.1295,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x1x0x2_default,11.21,1405.6,0.0
-gfx950,256,8,2176,7168,torch.float8_e4m3fn,flydsl,617,0,11.9394,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x2x0x2_default,20.9,1314.11,0.0
-gfx950,256,16,2176,7168,torch.float8_e4m3fn,ck,10,0,11.2181,a8w8_bpreshuffle_128x16x32x512_16x16_16x16_32x4x1_32x4x1_1x16x1x8_4x4x1_1x1_intrawave_v1,44.49,1406.82,0.0
-gfx950,256,32,2176,7168,torch.float8_e4m3fn,ck,10,0,11.1468,a8w8_bpreshuffle_128x16x32x512_16x16_16x16_32x4x1_32x4x1_1x16x1x8_4x4x1_1x1_intrawave_v1,89.55,1432.36,0.0
+gfx950,256,1,2176,7168,torch.float8_e4m3fn,flydsl,249,7,6.3084,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x1x0x2_default_ks7,4.95,2474.33,0.0
+gfx950,256,2,2176,7168,torch.float8_e4m3fn,flydsl,125,7,6.5415,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x0x0x2_default_ks7,9.54,2387.92,0.0
+gfx950,256,4,2176,7168,torch.float8_e4m3fn,flydsl,497,7,6.3485,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x2x0x2_default_ks7,19.66,2464.15,0.0
+gfx950,256,8,2176,7168,torch.float8_e4m3fn,flydsl,1,7,6.4615,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x0x0x2_default_ks7,38.62,2428.19,0.0
+gfx950,256,16,2176,7168,torch.float8_e4m3fn,flydsl,961,7,6.8142,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x4x0x2_default_ks7,73.25,2316.03,0.0
+gfx950,256,32,2176,7168,torch.float8_e4m3fn,flydsl,1068,7,7.7552,flydsl_bpreshuflle_32x64x512_F8_F8_B16_1x4x0x2_default_ks7,128.72,2058.77,0.0
 gfx950,256,64,2176,7168,torch.float8_e4m3fn,ck,10,0,11.4336,a8w8_bpreshuffle_128x16x32x512_16x16_16x16_32x4x1_32x4x1_1x16x1x8_4x4x1_1x1_intrawave_v1,174.62,1428.67,0.0
 gfx950,256,128,2176,7168,torch.float8_e4m3fn,flydsl,1713,0,12.5177,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x2x4x1_default,318.99,1363.84,0.0
 gfx950,256,256,2176,7168,torch.float8_e4m3fn,flydsl,2175,0,16.0682,flydsl_bpreshuflle_32x64x512_F8_F8_B16_0x4x4x1_default,497.0,1154.25,0.0
@@ -63,12 +63,12 @@ gfx950,256,4096,2176,7168,torch.float8_e4m3fn,flydsl,1000029,0,73.9852,flydsl_bp
 gfx950,256,8192,2176,7168,torch.float8_e4m3fn,flydsl,1103,0,134.9245,flydsl_bpreshuflle_128x128x256_F8_F8_B16_1x4x0x2_default,1894.03,815.04,0.0
 gfx950,256,16384,2176,7168,torch.float8_e4m3fn,flydsl,1000028,0,222.521,flydsl_bpreshuffle_8w_256x256x128_F8_F8_B16_2x4,2296.87,918.3,0.0
 gfx950,256,32768,2176,7168,torch.float8_e4m3fn,flydsl,1000029,0,398.7208,flydsl_bpreshuffle_8w_256x256x128_F8_F8_B16_2x8,2563.7,985.87,0.0
-gfx950,256,1,2304,1536,torch.float8_e4m3fn,flydsl,497,0,4.6403,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x2x0x2_default,1.53,763.98,0.0
-gfx950,256,2,2304,1536,torch.float8_e4m3fn,flydsl,1,0,4.681,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x0x0x2_default,3.02,758.65,0.0
-gfx950,256,4,2304,1536,torch.float8_e4m3fn,flydsl,249,0,5.1918,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x1x0x2_default,5.45,686.37,0.0
-gfx950,256,8,2304,1536,torch.float8_e4m3fn,flydsl,737,0,4.8189,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x3x0x2_default,11.75,744.59,0.0
+gfx950,256,1,2304,1536,torch.float8_e4m3fn,flydsl,849,0,4.5526,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x3x0x2_default,1.55,778.7,0.0
+gfx950,256,2,2304,1536,torch.float8_e4m3fn,flydsl,1059,0,4.5926,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x4x0x2_default,3.08,773.25,0.0
+gfx950,256,4,2304,1536,torch.float8_e4m3fn,flydsl,1059,0,5.0649,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x4x0x2_default,5.59,703.57,0.0
+gfx950,256,8,2304,1536,torch.float8_e4m3fn,flydsl,1059,0,4.6395,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x4x0x2_default,12.2,773.38,0.0
 gfx950,256,16,2304,1536,torch.float8_e4m3fn,flydsl,737,0,5.2365,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x3x0x2_default,21.63,694.6,0.0
-gfx950,256,32,2304,1536,torch.float8_e4m3fn,flydsl,249,0,4.642,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x1x0x2_default,48.79,804.73,0.0
+gfx950,256,32,2304,1536,torch.float8_e4m3fn,flydsl,1059,0,5.0118,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x4x0x2_default,45.19,745.35,0.0
 gfx950,256,64,2304,1536,torch.float8_e4m3fn,flydsl,249,0,4.8142,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x1x0x2_default,94.09,816.78,0.0
 gfx950,256,128,2304,1536,torch.float8_e4m3fn,flydsl,746,0,5.1826,flydsl_bpreshuflle_32x64x512_F8_F8_B16_0x3x0x2_default,174.81,834.6,0.0
 gfx950,256,256,2304,1536,torch.float8_e4m3fn,flydsl,802,0,6.3352,flydsl_bpreshuflle_32x64x512_F8_F8_B16_0x3x4x2_default,286.01,806.89,0.0
@@ -99,7 +99,7 @@ gfx950,256,1,6400,7168,torch.float8_e4m3fn,flydsl,1,0,12.163,flydsl_bpreshuflle_
 gfx950,256,2,6400,7168,torch.float8_e4m3fn,flydsl,617,0,12.7111,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x2x0x2_default,14.44,3612.21,0.0
 gfx950,256,4,6400,7168,torch.float8_e4m3fn,flydsl,737,0,12.7921,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x3x0x2_default,28.69,3592.46,0.0
 gfx950,256,8,6400,7168,torch.float8_e4m3fn,flydsl,1,0,12.526,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x0x0x2_default,58.6,3675.15,0.0
-gfx950,256,16,6400,7168,torch.float8_e4m3fn,ck,10,0,11.7238,a8w8_bpreshuffle_128x16x32x512_16x16_16x16_32x4x1_32x4x1_1x16x1x8_4x4x1_1x1_intrawave_v1,125.22,3940.25,0.0
+gfx950,256,16,6400,7168,torch.float8_e4m3fn,flydsl,249,2,11.2092,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x1x0x2_default_ks2,130.96,4121.14,0.0
 gfx950,256,32,6400,7168,torch.float8_e4m3fn,flydsl,311,0,13.2236,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x1x4x2_default,222.03,3517.51,0.0
 gfx950,256,64,6400,7168,torch.float8_e4m3fn,flydsl,1117,0,14.7136,flydsl_bpreshuflle_32x64x512_F8_F8_B16_1x4x4x2_default,399.09,3204.73,0.0
 gfx950,256,128,6400,7168,torch.float8_e4m3fn,ck,114,0,21.0686,a8w8_bpreshuffle_256x64x64x256_16x16_16x16_16x16x1_16x16x1_1x32x1x8_8x8x1_2x1_intrawave_v3,557.42,2298.73,0.0
@@ -112,7 +112,7 @@ gfx950,256,8192,6400,7168,torch.float8_e4m3fn,flydsl,1000026,0,294.7446,flydsl_b
 gfx950,256,16384,6400,7168,torch.float8_e4m3fn,flydsl,1000025,0,542.5117,flydsl_bpreshuffle_8w_256x256x128_F8_F8_B16_1x4,2770.89,687.6,0.0
 gfx950,256,32768,6400,7168,torch.float8_e4m3fn,flydsl,1000028,0,1050.5652,flydsl_bpreshuffle_8w_256x256x128_F8_F8_B16_2x4,2861.77,666.49,0.0
 gfx950,256,1,7168,768,torch.float8_e4m3fn,flydsl,1652,0,4.3118,flydsl_bpreshuflle_16x64x256_F8_F8_B16_0x2x0x1_default,2.55,1280.24,0.0
-gfx950,256,1,7168,1536,torch.float8_e4m3fn,flydsl,1,0,5.4868,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x0x0x2_default,4.01,2009.54,0.0
+gfx950,256,1,7168,1536,torch.float8_e4m3fn,flydsl,849,0,5.005,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x3x0x2_default,4.4,2202.98,0.0
 gfx950,256,1,7168,4224,torch.float8_e4m3fn,ck,9,0,17.7187,a8w8_bpreshuffle_128x16x32x128_16x16_16x16_8x16x1_8x16x1_1x16x1x8_4x4x1_1x1_intrawave_v1,3.42,1709.84,0.0
 gfx950,256,2,7168,768,torch.float8_e4m3fn,flydsl,1404,0,4.3292,flydsl_bpreshuflle_16x64x256_F8_F8_B16_0x1x0x1_default,5.09,1278.58,0.0
 gfx950,256,2,7168,1536,torch.float8_e4m3fn,flydsl,497,0,5.1133,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x2x0x2_default,8.61,2159.43,0.0
@@ -124,11 +124,11 @@ gfx950,256,8,7168,768,torch.float8_e4m3fn,flydsl,1652,0,4.4118,flydsl_bpreshufll
 gfx950,256,8,7168,1536,torch.float8_e4m3fn,flydsl,961,0,5.0859,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x4x0x2_default,34.64,2189.78,0.0
 gfx950,256,8,7168,4224,torch.float8_e4m3fn,ck,9,0,17.5696,a8w8_bpreshuffle_128x16x32x128_16x16_16x16_8x16x1_8x16x1_1x16x1x8_4x4x1_1x1_intrawave_v1,27.57,1731.75,0.0
 gfx950,256,16,7168,768,torch.float8_e4m3fn,flydsl,1156,0,4.9038,flydsl_bpreshuflle_16x64x256_F8_F8_B16_0x0x0x1_default,35.92,1171.88,0.0
-gfx950,256,16,7168,1536,torch.float8_e4m3fn,flydsl,1,0,5.237,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x0x0x2_default,67.28,2150.85,0.0
+gfx950,256,16,7168,1536,torch.float8_e4m3fn,flydsl,849,0,5.8241,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x3x0x2_default,60.49,1934.03,0.0
 gfx950,256,16,7168,4224,torch.float8_e4m3fn,ck,9,0,17.963,a8w8_bpreshuffle_128x16x32x128_16x16_16x16_8x16x1_8x16x1_1x16x1x8_4x4x1_1x1_intrawave_v1,53.94,1702.09,0.0
 gfx950,256,32,7168,768,torch.float8_e4m3fn,flydsl,257,0,4.4742,flydsl_bpreshuflle_32x64x256_F8_F8_B16_0x1x0x2_default,78.75,1338.42,0.0
-gfx950,256,32,7168,1536,torch.float8_e4m3fn,flydsl,1662,0,5.5894,flydsl_bpreshuflle_32x64x512_F8_F8_B16_0x2x0x1_default,126.07,2060.68,0.0
-gfx950,256,32,7168,4224,torch.float8_e4m3fn,ck,9,0,18.8765,a8w8_bpreshuffle_128x16x32x128_16x16_16x16_8x16x1_8x16x1_1x16x1x8_4x4x1_1x1_intrawave_v1,102.66,1635.45,0.0
+gfx950,256,32,7168,1536,torch.float8_e4m3fn,flydsl,677,0,5.9228,flydsl_bpreshuflle_16x64x512_F8_F8_B16_1x2x4x2_default,118.97,1944.68,0.0
+gfx950,256,32,7168,4224,torch.float8_e4m3fn,flydsl,624,3,11.1259,flydsl_bpreshuflle_32x64x128_F8_F8_B16_1x2x0x2_default_ks3,174.17,2774.75,0.0
 gfx950,256,64,7168,768,torch.float8_e4m3fn,flydsl,969,0,4.4907,flydsl_bpreshuflle_32x64x256_F8_F8_B16_0x4x0x2_default,156.91,1441.13,0.0
 gfx950,256,64,7168,1536,torch.float8_e4m3fn,flydsl,802,0,5.8575,flydsl_bpreshuflle_32x64x512_F8_F8_B16_0x3x4x2_default,240.6,2053.07,0.0
 gfx950,256,64,7168,4224,torch.float8_e4m3fn,ck,76,0,19.4931,a8w8_bpreshuffle_256x32x64x128_16x16_16x16_8x32x1_8x32x1_1x32x1x8_8x8x1_2x1_intrawave_v1,198.82,1614.19,0.0
@@ -164,7 +164,7 @@ gfx950,256,2,8448,7168,torch.float8_e4m3fn,flydsl,497,0,13.5444,flydsl_bpreshufl
 gfx950,256,4,8448,7168,torch.float8_e4m3fn,flydsl,1,0,13.541,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x0x0x2_default,35.78,4479.1,0.0
 gfx950,256,8,8448,7168,torch.float8_e4m3fn,flydsl,1,0,13.2129,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x0x0x2_default,73.33,4597.61,0.0
 gfx950,256,16,8448,7168,torch.float8_e4m3fn,flydsl,1,0,13.324,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x0x0x2_default,145.43,4573.72,0.0
-gfx950,256,32,8448,7168,torch.float8_e4m3fn,flydsl,1219,0,15.304,flydsl_bpreshuflle_16x64x512_F8_F8_B16_0x0x4x1_default,253.24,4007.14,0.0
+gfx950,256,32,8448,7168,torch.float8_e4m3fn,flydsl,10,0,15.3828,flydsl_bpreshuflle_32x64x512_F8_F8_B16_0x0x0x2_default,251.94,3986.62,0.0
 gfx950,256,64,8448,7168,torch.float8_e4m3fn,flydsl,1228,0,18.7312,flydsl_bpreshuflle_32x64x512_F8_F8_B16_0x0x4x1_default,413.81,3315.08,0.0
 gfx950,256,128,8448,7168,torch.float8_e4m3fn,flydsl,1240,0,24.2685,flydsl_bpreshuflle_64x64x256_F8_F8_B16_0x0x4x1_default,638.78,2622.14,0.0
 gfx950,256,256,8448,7168,torch.float8_e4m3fn,flydsl,88,0,33.9665,flydsl_bpreshuflle_64x192x256_F8_F8_B16_0x0x4x2_default,912.79,1964.16,0.0