diff --git a/flashinfer/experimental/dense_projection_det_gemm/README.md b/flashinfer/experimental/dense_projection_det_gemm/README.md index 467118cacec..97393bd88ab 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/README.md +++ b/flashinfer/experimental/dense_projection_det_gemm/README.md @@ -179,6 +179,34 @@ module's argument plan by keyword (`bind_launch`, fails closed on an unknown name). Given the same device the result is bitwise identical to the Cake launcher's. +### The swapped small-token forward + +The convenience path (`dense_projection_det_gemm`) runs a batched forward row +that the planner would swap (`swap_small_m`: `L > 1`, `M <= 256`, `N >= 2 M`) +on a program that puts the weight rows in the MMA's M and the tokens in its N +(the `swap_ab_` templates), when `A` and `B` are K-major, the output is +BF16 without `transposed_out`, `K` is a multiple of 64 and every view's row and +batch strides and start are in 16-byte units. A CTA streams a +`tile_w x 64` weight box per stage through a deep TMA ring (64 rows over 12 +stages, or 128 rows over 8) against a 64-token tile, single CTA, no split: each +output element is the one FP32 sum of its whole K, rounded once. The tile is +the narrow one while its `ceil(N / 64) * ceil(M / 64) * L` CTAs fit one wave of +the device's SMs, the wide one while its CTAs fit two waves; a row whose wide +tiles need more than two waves runs the planner's program, which streams the +weights faster there (`swap_ab_template`, `SWAP_MAX_WAVES`). The program +reads the caller's views through tensor maps that carry their strides and +stores at `l * out_l + m * ldo + n`, so the token-major `[T, L, K]` / +`[T, L, N]` views of a grouped projection run without a copy. Every other row, +and every prepared launch, runs the planner's program. + +`enable_pdl=True` launches the swapped program as a programmatic dependent +launch: every CTA triggers its dependents at entry, the weight loads start +before `griddepcontrol.wait` and only the token loads wait for the predecessor +grid, so the weight stream overlaps the predecessor's tail. `B` must then not +be written by a kernel still in flight on the stream (a weight is a parameter +of the model). The planner's programs take no PDL wait and launch without the +attribute whatever `enable_pdl` says. + ### K2 contract * `A`, `B`, `out` FP32 2-D views: `A [M, K]` with unit stride on `m` or `k` diff --git a/flashinfer/experimental/dense_projection_det_gemm/build.json b/flashinfer/experimental/dense_projection_det_gemm/build.json index 661ae5e67cd..beadb0d572f 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/build.json +++ b/flashinfer/experimental/dense_projection_det_gemm/build.json @@ -55,9 +55,9 @@ } }, "generator": { - "cake_revision": "1eefe68eea54f0b9925c81fd77aad262064bc2b4", + "cake_revision": "72d0af54b2ddd96cd9f931508e70999338e18cce", "cuda": "13.4.92", - "rendered_at": "2026-10-31T07:25:51Z" + "rendered_at": "2026-11-02T10:01:15Z" }, "libraries": [ { @@ -84,7 +84,70 @@ }, { "arch": "sm_100a", - "closure_sha256": "2ad558f4a1a164f788e04c3cb1c83fd0b5f1dfeead2ab24d49d34182caee4841", + "closure_sha256": "4ad2f313875c6a14b8623afa670eeae2463ec91dc77726b7ef697dfccf136c1a", + "entry": "run", + "gencode": "-gencode=arch=compute_100a,code=sm_100a", + "min_cuda": "12.8", + "name": "cake_dense_projection_det_gemm_02803e1d2393113c0f6c_sm_100a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_sm_100a.so", + "sha256": { + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_binding.cu": "5082c95018bc9c684c0423d45a82d095033a654118accead3e9a89f6df6f0e22", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_kernel.cu": "9745b7e179d722561fd5c0ea1743c662e39451b0a44ecbc5ea4f60c126bb74d3" + }, + "sources": [ + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_kernel.cu", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_binding.cu" + ] + }, + { + "arch": "sm_103a", + "closure_sha256": "34bb3be03c949dd6da7ad52f2947fd8c51b943aaa606630396a8eac78625a460", + "entry": "run", + "gencode": "-gencode=arch=compute_103a,code=sm_103a", + "min_cuda": "12.9", + "name": "cake_dense_projection_det_gemm_02803e1d2393113c0f6c_sm_103a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_sm_103a.so", + "sha256": { + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_binding.cu": "5082c95018bc9c684c0423d45a82d095033a654118accead3e9a89f6df6f0e22", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_kernel.cu": "9745b7e179d722561fd5c0ea1743c662e39451b0a44ecbc5ea4f60c126bb74d3" + }, + "sources": [ + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_kernel.cu", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "7c289573dfbf4a2bf69b3be297cff4df5262478808a469da0afffc7d522de5e8", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_dense_projection_det_gemm_02803e1d2393113c0f6c_sm_107a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_sm_107a.so", + "sha256": { + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_binding.cu": "5082c95018bc9c684c0423d45a82d095033a654118accead3e9a89f6df6f0e22", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_kernel.cu": "9745b7e179d722561fd5c0ea1743c662e39451b0a44ecbc5ea4f60c126bb74d3" + }, + "sources": [ + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_kernel.cu", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_binding.cu" + ] + }, + { + "arch": "sm_100a", + "closure_sha256": "0150e651491c9ea6310244dfe1846ef08853bc4efc9835bddd1bda97ace3d9c0", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -94,7 +157,7 @@ "path": "aot/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_kernel.cu": "ec8bf8f5757c1c2764d39a6b2a25e1198c39f014d78965d93c387547a7784baf" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_kernel.cu": "6ebb45bb7c76b6c2c81749ed6d2824787400ad942ff0e7eb371a46d5261bada8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_kernel.cu", @@ -103,7 +166,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "90b998d3c6db23de300787400ae75f320adb5f62508ef37d13635e3a82c80bb9", + "closure_sha256": "f36981ef40e1b26109158a357210f42f0af3d414c3bcfde78b41a8ed77c4c64f", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -113,7 +176,7 @@ "path": "aot/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_kernel.cu": "ec8bf8f5757c1c2764d39a6b2a25e1198c39f014d78965d93c387547a7784baf" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_kernel.cu": "6ebb45bb7c76b6c2c81749ed6d2824787400ad942ff0e7eb371a46d5261bada8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_kernel.cu", @@ -122,7 +185,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "a0d61c770ebf1c3a630842519fba83796c02ffa2b05b6c03e913053a2ec994ed", + "closure_sha256": "766c159cac06de6316b28f41f9a6f10c5772fe8488173fe585fae125e7bffbc5", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -140,7 +203,7 @@ "path": "aot/cake_dense_projection_det_gemm_087ba4f157914949c26a_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "6cfbd8ec7854b6933dc238c3b7114cf64f7bd47a00c4c32007f87965a9fe7a14" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "30b0f5a712f5426c8829b78ba88ea841000fabaa8525f96bec060d10fc79d3c8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu", @@ -149,7 +212,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "90e403cef14672858d74aa303507accf4a6d97ad1b64257152bdf86e97d22079", + "closure_sha256": "a2931ab01110c403d87765cfad47391a62c282af7a00bbb4318846af9fe7fdca", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -167,7 +230,7 @@ "path": "aot/cake_dense_projection_det_gemm_0a71a49da71c7d34ab37_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0a71a49da71c7d34ab37_kernel.cu": "7aea7d133a3164d5f8c8c5555ce7da3f49377e225d41bfad6b2d946419696cd9" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0a71a49da71c7d34ab37_kernel.cu": "b7fe2c3defbb2b13e23a0da439859349f1552afb92ee4fee945ba33936ff5b7d" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0a71a49da71c7d34ab37_kernel.cu", @@ -353,7 +416,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "9358c97bcfe23515d859c13d9873cfdddbdf74284f893983c44c7e67bed0a903", + "closure_sha256": "0e7b3d60a7e363e1f3cb5d056a1ac524662ecafd47bffbfd7318adfacb607b6a", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -371,7 +434,7 @@ "path": "aot/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu": "642cafa96a81b990aeeb564c571f4fba93e8b520ff3580aa39236dff51dab827" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu": "9a4984ead474fb7a567110f730b0527cc40c64a87705b186c7bf93b728f44cb5" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu", @@ -380,7 +443,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "017033cb6128a90961b10cc6f58ffde3e5a0cafa38121ded968bf6fe0099c693", + "closure_sha256": "8f545045db60024ccb7c8b14fe7ee3edd3d5e7befc0177d2a7846c5f846af67f", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -398,7 +461,7 @@ "path": "aot/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu": "642cafa96a81b990aeeb564c571f4fba93e8b520ff3580aa39236dff51dab827" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu": "9a4984ead474fb7a567110f730b0527cc40c64a87705b186c7bf93b728f44cb5" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu", @@ -407,7 +470,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "ac271609a0c8dc2aade1952cc35c19e7c852b2d33640a634ac34a0bd02320d24", + "closure_sha256": "4b4a2dd6c914bd766c7917ce8b5a5e952c12ec89dc5ca13678097f08cc53a905", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -425,7 +488,7 @@ "path": "aot/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu": "642cafa96a81b990aeeb564c571f4fba93e8b520ff3580aa39236dff51dab827" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu": "9a4984ead474fb7a567110f730b0527cc40c64a87705b186c7bf93b728f44cb5" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu", @@ -612,7 +675,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "41c84470404270497998de62635a11abc4d9d2acab793e85211cce3e18f5a6ad", + "closure_sha256": "feb92c0ce67cb2f2ea13e9b804f132c659bd68a1ba6be5fa9f343a649484b8ee", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -630,7 +693,7 @@ "path": "aot/cake_dense_projection_det_gemm_13d9310f37868a18eace_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_13d9310f37868a18eace_kernel.cu": "9a57c1c2c7cc0855347398a1bf86e2cdbab37d6b43593b6d370dcd6b646e4037" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_13d9310f37868a18eace_kernel.cu": "0590c170d9c0a96f0fe5183a9af68995cd165cc270ff411f7145a1aa9b725295" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_13d9310f37868a18eace_kernel.cu", @@ -639,7 +702,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "8ba766b57d3efdfbb36ac33393f07abb8103c38b886a3861c21d175df8ade1ee", + "closure_sha256": "56931e85e537b783c57a4d2688590e46fe8c11338d5fc230a2abb8907dfd071e", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -657,7 +720,7 @@ "path": "aot/cake_dense_projection_det_gemm_14192244441c1003db74_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_14192244441c1003db74_kernel.cu": "90abc16cb90f9a3e1cb85cdd78c8ce3bab22565427a4b13cd074438390f95453" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_14192244441c1003db74_kernel.cu": "dca50fa30dadc574befce31097137b0ae1b60cb95b5541e0e3f79b14011912ab" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_14192244441c1003db74_kernel.cu", @@ -666,7 +729,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "1fb9c8e82e0078e86d938875c6b97ec3f971dfd3938f4605028294ffb5e4b637", + "closure_sha256": "52652cdb673078a0508a2f976e09a3fabd219579b7c516f003bcf2d135c6aee8", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -685,7 +748,7 @@ "path": "aot/cake_dense_projection_det_gemm_1c1874c560375d29237d_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "6cfbd8ec7854b6933dc238c3b7114cf64f7bd47a00c4c32007f87965a9fe7a14" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "30b0f5a712f5426c8829b78ba88ea841000fabaa8525f96bec060d10fc79d3c8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu", @@ -694,7 +757,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "b045adfcc2f054635ca97a89e8e7d97711a18ba11e1087b9e35620442a7987b3", + "closure_sha256": "877ecd8c2464f7cd08a8a1b1863c87bc3aedfcfc14ddc65d47a31d489ffeefd2", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -713,7 +776,7 @@ "path": "aot/cake_dense_projection_det_gemm_1c1874c560375d29237d_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "6cfbd8ec7854b6933dc238c3b7114cf64f7bd47a00c4c32007f87965a9fe7a14" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "30b0f5a712f5426c8829b78ba88ea841000fabaa8525f96bec060d10fc79d3c8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu", @@ -722,7 +785,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "af5f7cd1e65db02d787d86ffdf74cfc09a66d06d5724ac68342e7d1750220826", + "closure_sha256": "72f07a4c0ecc9dbc4bf8e1529af3b2efcb22b68eb686bc73b231099de61a94a6", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -741,7 +804,7 @@ "path": "aot/cake_dense_projection_det_gemm_1c1874c560375d29237d_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "6cfbd8ec7854b6933dc238c3b7114cf64f7bd47a00c4c32007f87965a9fe7a14" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "30b0f5a712f5426c8829b78ba88ea841000fabaa8525f96bec060d10fc79d3c8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu", @@ -845,7 +908,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "c55e18679c6426a2e37579f7704f47cd7c6d4e5ec88db45ba58b0d72ae82967a", + "closure_sha256": "93265ea4f5d906de33a370e8462fd54f46a598b45e393d24c8229c9d7f212f90", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -863,7 +926,7 @@ "path": "aot/cake_dense_projection_det_gemm_295aef2f2e0332de2372_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_295aef2f2e0332de2372_kernel.cu": "4fc008e3b7bb6f603b5e56695503f530a700e17afff342f045a0f943069b8c5b" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_295aef2f2e0332de2372_kernel.cu": "96664a1b74039224ee0a2b1aafce5f724e6623611b2169057b6e1182edd98fbb" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_295aef2f2e0332de2372_kernel.cu", @@ -872,7 +935,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "7cbf74f2f9e2eac9c1f3e55f1ce8759006d283b3e5a275641552d42d9febd7b2", + "closure_sha256": "561f1ccd4aa7c7ce52f94169680e37989dd9c0e15c8806cf2474eb9285ed9ff2", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -890,7 +953,7 @@ "path": "aot/cake_dense_projection_det_gemm_295aef2f2e0332de2372_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_295aef2f2e0332de2372_kernel.cu": "4fc008e3b7bb6f603b5e56695503f530a700e17afff342f045a0f943069b8c5b" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_295aef2f2e0332de2372_kernel.cu": "96664a1b74039224ee0a2b1aafce5f724e6623611b2169057b6e1182edd98fbb" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_295aef2f2e0332de2372_kernel.cu", @@ -994,7 +1057,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "133c41dc228f9cd1ebdeee228b407f6865886cc24dbb44dc70d762a73dc22cdf", + "closure_sha256": "a4ea3513b3ae823ea12db76e1db452455c27a2ee51633d7c6d767a7cbb7cd3ac", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1012,7 +1075,7 @@ "path": "aot/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu": "1bf78c7f7a638e23d3953cb1a865f9da60388ae0dc9095721b6de55b6c69fe75" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu": "2d63334a27288aaa65e7100515de21fac736cbbc789f89b764316da119a642ca" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu", @@ -1021,7 +1084,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "4c1ebf9ec9cb066f424c2dce55b787bb4079ccfed4d8119134687ed109ff0d7e", + "closure_sha256": "76161447b58cf17848a8b5183d6513b43f0b5d131e93bf1b8496b53d0c5c92ed", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -1039,7 +1102,7 @@ "path": "aot/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu": "1bf78c7f7a638e23d3953cb1a865f9da60388ae0dc9095721b6de55b6c69fe75" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu": "2d63334a27288aaa65e7100515de21fac736cbbc789f89b764316da119a642ca" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu", @@ -1048,7 +1111,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "13340c9eb683b8b4fb8a190ef7d42a20cd5b8fc262508c33996a6a9a5c9e6837", + "closure_sha256": "b6fcbd4f430d5aa430fc97f87049a37f2a407e6c5a935713e0d84a68b4482db6", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -1066,7 +1129,7 @@ "path": "aot/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu": "1bf78c7f7a638e23d3953cb1a865f9da60388ae0dc9095721b6de55b6c69fe75" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu": "2d63334a27288aaa65e7100515de21fac736cbbc789f89b764316da119a642ca" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu", @@ -1075,7 +1138,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "1c7ff55a26f58616d7e7273897bb9b6862846b95dd5300b63a4139b9e7972de7", + "closure_sha256": "b574590f5c68f1b924a10a4989e7c4b569f0d0d018729d7596117bc3465fa6f0", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -1085,7 +1148,7 @@ "path": "aot/cake_dense_projection_det_gemm_31be7faa868eabcedada_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_31be7faa868eabcedada_binding.cu": "e0291fb2f60cc929950a6843b13ae4adff71dea1058317c4f41f95bb84942ae1", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_31be7faa868eabcedada_kernel.cu": "92ab00784b9edea17abb8c2f189383091105984754cab916fba3a05a8c5ec0f8" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_31be7faa868eabcedada_kernel.cu": "e03b70061b3f11339f4fd96831132ea26afc8207bedcd3b92a4ef62f36934101" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_31be7faa868eabcedada_kernel.cu", @@ -1382,7 +1445,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "127f89c15470ff226f9b6abd59371399a6efc21f1793d7ffbcb9f55e74cf83a6", + "closure_sha256": "111b55d0fcc2c4c184d86939345f97a76e95ba123f8a9fd8a8dcc0ab39d2eabb", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1401,7 +1464,7 @@ "path": "aot/cake_dense_projection_det_gemm_39200edabebcd9eb2edd_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu": "1bf78c7f7a638e23d3953cb1a865f9da60388ae0dc9095721b6de55b6c69fe75" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu": "2d63334a27288aaa65e7100515de21fac736cbbc789f89b764316da119a642ca" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu", @@ -1410,7 +1473,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "45a3daa089de4eca6795e0d0421167e84147300e45483d97229321c768a2d9ac", + "closure_sha256": "f9c6e30bd20d1885fd8af4a329be178b4609ecb423226f15745f6ddef094d939", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -1429,7 +1492,7 @@ "path": "aot/cake_dense_projection_det_gemm_39200edabebcd9eb2edd_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu": "1bf78c7f7a638e23d3953cb1a865f9da60388ae0dc9095721b6de55b6c69fe75" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu": "2d63334a27288aaa65e7100515de21fac736cbbc789f89b764316da119a642ca" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu", @@ -1552,7 +1615,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "ab85e0460d08e166937df4a8c016a53a574cba005245873cacbee054ad2473d6", + "closure_sha256": "7edd00a323dc4c302f45f017e8cb2a3cec43c9e4187a1b2c3815eb9a4ccaaa63", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -1570,7 +1633,7 @@ "path": "aot/cake_dense_projection_det_gemm_438841f42a0743b63fb5_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_438841f42a0743b63fb5_kernel.cu": "3626825a65521e41c402c804cbd9ab9e7e1a0e5b0fd833555b16604ec636ee80" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_438841f42a0743b63fb5_kernel.cu": "1ed8722fdaffc30a7e3a0f959405623b530274926da4f5183f0002b83eecdbd9" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_438841f42a0743b63fb5_kernel.cu", @@ -1579,7 +1642,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "552da34f1f7ae611984083fe875d33a5fb322f9d464e1e4fe88b0ed1cdd46cbc", + "closure_sha256": "7a0179c1925988c936ca110bc3a261876d8037938a04fe4904e297a2c53bb8cd", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -1597,7 +1660,7 @@ "path": "aot/cake_dense_projection_det_gemm_446497e750658dfb057b_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_31be7faa868eabcedada_binding.cu": "e0291fb2f60cc929950a6843b13ae4adff71dea1058317c4f41f95bb84942ae1", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_446497e750658dfb057b_kernel.cu": "69a9e4ba927e265e1f1e72014e778a0b1a6cc487c72334b4040c355b343ea94f" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_446497e750658dfb057b_kernel.cu": "b37bb22d62a9f4ee124aaf7be8e8514c1e84f5674e3ef407dcc9f47b04740b8b" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_446497e750658dfb057b_kernel.cu", @@ -1805,7 +1868,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "788370159d4b8c40b0e42529ac8f9ef9b49fb9e1e579a54bc102a71a0ce87580", + "closure_sha256": "369c1abcc1f0e59c2de347d53f804935fdbac84d7ae45df1e11e977a39233f2d", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -1823,7 +1886,7 @@ "path": "aot/cake_dense_projection_det_gemm_505fbe05ce8b62f0a589_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_505fbe05ce8b62f0a589_kernel.cu": "f033c2598cfe7f13a58db54501649713234428bb64917a931f4244c8c2162764" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_505fbe05ce8b62f0a589_kernel.cu": "ff52c5210ec1e72109ea8c00719bceb612a5abd40cf21ad23e6462e9c9d01aa2" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_505fbe05ce8b62f0a589_kernel.cu", @@ -1889,7 +1952,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "284c2e4abf70c114b69fbc8f774ce27979a81dac83adf47ebe74b88d3bdc3f2c", + "closure_sha256": "71b2c9b187bcb2b4d5231e28de6b0fad627d651dfd8cbcae7ec08e840a3a8b0d", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1907,7 +1970,7 @@ "path": "aot/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_kernel.cu": "933928cd8eb97ff95cdcf36f338b9404953486920067127a080e3dddce4d7256" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_kernel.cu": "8a3b399d1739ef542e00539402070969c92f43297ea96944d1df589182f81333" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_kernel.cu", @@ -1916,7 +1979,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "d75734c580d89b38c78c0b6e85a365ed35ee61193916a0129e9251027227a4e6", + "closure_sha256": "bb61d2720f56ccaddf67811bb2ee4d7b78b8d6336a3d22bbaa7e52ba8bfd8f08", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -1934,7 +1997,7 @@ "path": "aot/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_kernel.cu": "933928cd8eb97ff95cdcf36f338b9404953486920067127a080e3dddce4d7256" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_kernel.cu": "8a3b399d1739ef542e00539402070969c92f43297ea96944d1df589182f81333" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_kernel.cu", @@ -1975,7 +2038,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "17a36ad5754ea9dc348af15fe218b40339112ac6c4ad472cb168adb53a8aeb3a", + "closure_sha256": "1ebecfae8a8892e4b980ff229f7a5e0cac981e7e1f8645a52c43068c26e00fd9", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -1993,7 +2056,7 @@ "path": "aot/cake_dense_projection_det_gemm_5445e0f16ad84868fa01_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5445e0f16ad84868fa01_kernel.cu": "090b91f0e43564c5d70e431bcc4295ae78f0a60317f88f91f5eb297b92ac36fc" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5445e0f16ad84868fa01_kernel.cu": "cbba263847838044146efecf9b6baea9aa4a44fb3701ad4257fb226ec2ca96da" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5445e0f16ad84868fa01_kernel.cu", @@ -2021,7 +2084,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "d6f31390575c6dcccaba45f2e238e28a8bd97b3ce23109d3c02dcb1dd8b5d588", + "closure_sha256": "ae61fbcabfdded637c3cd1d7d8d0e46e93caf2a57a18319bbeca6c68e38b1f4d", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -2039,7 +2102,7 @@ "path": "aot/cake_dense_projection_det_gemm_5568657a526e2040f0bc_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5568657a526e2040f0bc_kernel.cu": "cebd36d1e701f96b8cb5a615a82498d16f84830fc0e3a3bf8113bd066f666f8b" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5568657a526e2040f0bc_kernel.cu": "20aa98d7093d6ea08a89266f4eba218109099d2347c8730b687d256ea6128ca5" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5568657a526e2040f0bc_kernel.cu", @@ -2048,7 +2111,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "c594a350b728ec05160787d637e3f3ece8b2456e2bcb554a8dbe87c6fb25e4e1", + "closure_sha256": "15fbe481b9757bc9c51cfd8c0eb0a68128d90e3305041561f34f056ce6567f41", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -2066,7 +2129,7 @@ "path": "aot/cake_dense_projection_det_gemm_5568657a526e2040f0bc_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5568657a526e2040f0bc_kernel.cu": "cebd36d1e701f96b8cb5a615a82498d16f84830fc0e3a3bf8113bd066f666f8b" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5568657a526e2040f0bc_kernel.cu": "20aa98d7093d6ea08a89266f4eba218109099d2347c8730b687d256ea6128ca5" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5568657a526e2040f0bc_kernel.cu", @@ -2139,7 +2202,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "14ee954ffd2a990204f1e20d0cccc8fd11152c1d3178ef7a84d479ba85fa6313", + "closure_sha256": "23a39d1f336bc2f5f127646ebe08c5808a9b44365cf4cf45cbdf7de4be774519", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -2157,7 +2220,7 @@ "path": "aot/cake_dense_projection_det_gemm_56be8af43d5d0822da87_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_56be8af43d5d0822da87_kernel.cu": "4070300a1fb97a4ab03a4c8c65a009a1bdc8b3edfd1aded776078b08a8efbac6" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_56be8af43d5d0822da87_kernel.cu": "077a0f149b0c9c402b225691418e025c3404bdaa5ede07806abcbb0d993dcb07" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_56be8af43d5d0822da87_kernel.cu", @@ -2166,7 +2229,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "f13e8e7365963b600696dea82bbe053eadf27358860bf5bc2f75199986c66432", + "closure_sha256": "7293602b0eb3845817204c204517a16be81d025aa12a0cdc293b02b31603f8b3", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -2184,7 +2247,7 @@ "path": "aot/cake_dense_projection_det_gemm_56be8af43d5d0822da87_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_56be8af43d5d0822da87_kernel.cu": "4070300a1fb97a4ab03a4c8c65a009a1bdc8b3edfd1aded776078b08a8efbac6" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_56be8af43d5d0822da87_kernel.cu": "077a0f149b0c9c402b225691418e025c3404bdaa5ede07806abcbb0d993dcb07" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_56be8af43d5d0822da87_kernel.cu", @@ -2282,7 +2345,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "200ca30bfb1fd811dc0c24e439840bdbae05c94642146469aacd242d932a9685", + "closure_sha256": "d8a54344a9765becc63df289ef5c11cc6aec023ccb54ff04dbd3b8236e9c3b50", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -2300,7 +2363,7 @@ "path": "aot/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_kernel.cu": "027cc18812144cc814900ef935b432cb0779a7afbd464007ae3ae957f6f3e27b" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_kernel.cu": "b6d37d7a94b16a62b8b1597ca5a4123150472dcbd7bcacf2ba6eb41351b5f571" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_kernel.cu", @@ -2309,7 +2372,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "5d2e5ef6b2978db6889af36fa6c39cc713c34d8f8e33e40896e343c5c778700e", + "closure_sha256": "50f8a300bb00843513e767dfe03256a20acdd4dc0b9318bf3680ab4a9b121131", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -2327,7 +2390,7 @@ "path": "aot/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_kernel.cu": "027cc18812144cc814900ef935b432cb0779a7afbd464007ae3ae957f6f3e27b" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_kernel.cu": "b6d37d7a94b16a62b8b1597ca5a4123150472dcbd7bcacf2ba6eb41351b5f571" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_kernel.cu", @@ -2393,7 +2456,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "eb5f2237945962fcb2ba5e2d7f3daa8ae283afc839f79ba588385cdf42cda78a", + "closure_sha256": "a97dcfe2aee399a8b153f21c749f53c99a2865722dd3a4d2d813b786b0eef3e4", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -2411,7 +2474,7 @@ "path": "aot/cake_dense_projection_det_gemm_5e77ef909e88a99c4357_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5e77ef909e88a99c4357_kernel.cu": "9035da483b703a5e37c8a58980540885bb8b5b534abe1cc421d0d0a8112d77bb" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5e77ef909e88a99c4357_kernel.cu": "1ab08697a0e6de1eaa2c8da4268c4a753e8feb27effad1bb5fe5425eb25a89e4" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5e77ef909e88a99c4357_kernel.cu", @@ -2630,7 +2693,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "9811979504cecdfbc84a0cd3863f158137ef0d2d243f269bc0843c1bb1e92a7d", + "closure_sha256": "ce91a8c6f1a39db2116e216e62858af6178f6fb58f721f00cc21fbdc19835c1f", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -2648,7 +2711,7 @@ "path": "aot/cake_dense_projection_det_gemm_66df6446dd2cca3106d3_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_66df6446dd2cca3106d3_kernel.cu": "f641c2f007fcb1deed478f85d78f2c78d51f37529d28317c86726ba3022c140c" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_66df6446dd2cca3106d3_kernel.cu": "de6266445a3858b7a5eac97906fe6593e38506da0cccb838ef4cf43efbc524fd" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_66df6446dd2cca3106d3_kernel.cu", @@ -2906,7 +2969,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "6aea6dac93eecd01d3c2e599eea27670181e203f66890a66e4119d6ea167ff4a", + "closure_sha256": "5881a92cf86fbdee4536f945251bdfbfd22fe47dd7e9e87c7fbc56b7e619d44e", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -2924,7 +2987,7 @@ "path": "aot/cake_dense_projection_det_gemm_765ece06955b70a95b29_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_765ece06955b70a95b29_kernel.cu": "e2fc5de8eeb4c6ee9635f5c34f6e976b9611a806a0e049700c6471230d9dc313" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_765ece06955b70a95b29_kernel.cu": "7c652f2a34d4a96ba40fcd8ffd25edac480eb846578c892208882f3baa81c455" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_765ece06955b70a95b29_kernel.cu", @@ -2933,7 +2996,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "cacff571633da98411de7a6ba8cb29a7d9529803a9e3266a0aa69e20ade31e1f", + "closure_sha256": "7446c3613a6e9d184648ea39316c0485fc8cbed22a5f8769eb19bb0fa6dc0dd4", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -2951,7 +3014,7 @@ "path": "aot/cake_dense_projection_det_gemm_765ece06955b70a95b29_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_765ece06955b70a95b29_kernel.cu": "e2fc5de8eeb4c6ee9635f5c34f6e976b9611a806a0e049700c6471230d9dc313" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_765ece06955b70a95b29_kernel.cu": "7c652f2a34d4a96ba40fcd8ffd25edac480eb846578c892208882f3baa81c455" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_765ece06955b70a95b29_kernel.cu", @@ -3049,7 +3112,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "4f0a1bcbfe0ff9c7d04a5f408819536e1bc2532579f046206d59794a4e41b793", + "closure_sha256": "9adcad891c21759af55b42d5af187dc5d68fcc76b12dad7354910dfa59b98cfa", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -3067,7 +3130,7 @@ "path": "aot/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu": "bede085ad48300d03d11e41cffe483cccc7fac4c3bfddeafff683f4f6f89b1af" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu": "17a8acc279e2ebc7769d2dd35d8e1946662b3717dc0946aa1459e8fc50fb3999" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu", @@ -3076,7 +3139,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "f5c28af72e07925b5c2a32cd3594f5173f243f0da7085a36d9a817bb0b6bce2c", + "closure_sha256": "8e6ea2b2dfad477e99578954d308ce9ece0369d23bfc40f13818d8b6e31d42aa", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -3094,7 +3157,7 @@ "path": "aot/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu": "bede085ad48300d03d11e41cffe483cccc7fac4c3bfddeafff683f4f6f89b1af" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu": "17a8acc279e2ebc7769d2dd35d8e1946662b3717dc0946aa1459e8fc50fb3999" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu", @@ -3103,7 +3166,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "e13bf79e422ece2d6f8467ca5c5f3dc08bb91498947010b67ffd519ac6ec707a", + "closure_sha256": "3163094583559e3f4e8ac000cb15a8af82da4be7df3bbe83ebd381f46e869d3e", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -3121,7 +3184,7 @@ "path": "aot/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu": "bede085ad48300d03d11e41cffe483cccc7fac4c3bfddeafff683f4f6f89b1af" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu": "17a8acc279e2ebc7769d2dd35d8e1946662b3717dc0946aa1459e8fc50fb3999" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu", @@ -3251,7 +3314,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "5131532753957e0439f86e8df03b8f6a307532b49b281119f739d1defd91009b", + "closure_sha256": "5e6e846a7219f928cc45e97f72dc8b3c1854f4c250dbc4bcf26ff4f0e72457a4", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -3269,7 +3332,7 @@ "path": "aot/cake_dense_projection_det_gemm_806927d101fe72fc1317_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu": "85ab847a189bc83ccbb9aa99f77c2ad0a85e841602b7b378bc92a68462fd65a0" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu": "1da60114141530b5a0c3706ef1886c87dc65bbac7d7d8e3b344b0b11acdedca1" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu", @@ -3278,7 +3341,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "5fe9f670f7d95bc03571efee60d75077de96e273f3a068ee823f8c13f00ba4a8", + "closure_sha256": "4fd780d6953f6bba8227b7817c31b073e6231d4ed6c0d9f374b49dec128453c3", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -3296,7 +3359,7 @@ "path": "aot/cake_dense_projection_det_gemm_806927d101fe72fc1317_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu": "85ab847a189bc83ccbb9aa99f77c2ad0a85e841602b7b378bc92a68462fd65a0" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu": "1da60114141530b5a0c3706ef1886c87dc65bbac7d7d8e3b344b0b11acdedca1" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu", @@ -3305,7 +3368,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "c0c60b3cc45bec930f5795b0ded02fd4385c10fc8b9462a417ccf4ab8000b067", + "closure_sha256": "bcf45ec137e18e8e07098c157ed52aa741ec253e84b26b3df64975627e054eab", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -3323,7 +3386,7 @@ "path": "aot/cake_dense_projection_det_gemm_82f698f67a1cca6e4291_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_82f698f67a1cca6e4291_kernel.cu": "6a97d4516be3fa909113d87c1e5b4049d5b722d262b8299233bb3b2bff501fa1" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_82f698f67a1cca6e4291_kernel.cu": "4f283832ec5c869de27a09fa01ec3bea8f57a067cb5126d47ac540ed6d981472" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_82f698f67a1cca6e4291_kernel.cu", @@ -3332,7 +3395,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "1cf9a4c6d5a74945c35d40cc89a8a753e2882c2f626020706557436919c66ee7", + "closure_sha256": "0f49f4338d13687afccd3398920189dd31419ff8fec5ef62169cf0c3def98cbf", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -3350,7 +3413,7 @@ "path": "aot/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_kernel.cu": "ae2f9cc4a45cac07c59e132505a671e1289074b791e2450adf0e92aa9fbddd0d" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_kernel.cu": "e7a8028f3a2f122509568d5a5a4bbfcd55729ac2070ce13f9608e77ad680e6e5" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_kernel.cu", @@ -3359,7 +3422,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "0fc81c8d21da2b60f2307d2454e3f1548c24e1616893128d33b91a9a1836b504", + "closure_sha256": "fcd6b4b877b3740fa285fd805d25e51d8626aa859842e82f8deef9174f7a5c99", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -3377,7 +3440,7 @@ "path": "aot/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_kernel.cu": "ae2f9cc4a45cac07c59e132505a671e1289074b791e2450adf0e92aa9fbddd0d" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_kernel.cu": "e7a8028f3a2f122509568d5a5a4bbfcd55729ac2070ce13f9608e77ad680e6e5" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_kernel.cu", @@ -3386,7 +3449,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "ea7ab035a37e68845d211e64101981e45433d45b1fa34e27b75216640e90fa7f", + "closure_sha256": "5b7a163df83b9299e0529a3453ce5784eafacf5343f33d8bf6e44803b80ea540", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -3404,7 +3467,7 @@ "path": "aot/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu": "4014ebb24011a4213e8e4c1d6419bb506f9d8d62e1a9a6a227e7777c6bce023e" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu": "8fe448b51f64bd64e70f613b30e7097ade23c52ea865e519b8410d63c5616bf0" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu", @@ -3527,7 +3590,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "7e85c72360273a9cd0d3839b31a87a69474d18bc759af5b57a4f1f4206dd589c", + "closure_sha256": "108b2fcc95352ac03b1a859fbbcdc2a164a22961f663d2bbcd36d7ba325f6587", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -3545,7 +3608,7 @@ "path": "aot/cake_dense_projection_det_gemm_9aa9aa60b0c544bfab06_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_31be7faa868eabcedada_binding.cu": "e0291fb2f60cc929950a6843b13ae4adff71dea1058317c4f41f95bb84942ae1", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9aa9aa60b0c544bfab06_kernel.cu": "b9f994e70a6626b88ece6f006343764b912297a5927cc850e84451c3e7804976" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9aa9aa60b0c544bfab06_kernel.cu": "abe374112569bf94653388e4e119e37b1ba06ae5da25f8259c03244b34964ff4" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9aa9aa60b0c544bfab06_kernel.cu", @@ -3554,7 +3617,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "7285ed4045051fbccd608cd389062821c282e320e3180b406f45db21460b7bca", + "closure_sha256": "dc307c7c8c23b702dbfe8b3a0f159169b90782478989966017b6c23a89564cfa", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -3572,7 +3635,7 @@ "path": "aot/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu": "c7933ee9713b9f2b2d2d25c1b134db34af14b39a231d1eca351dc52de08cefad" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu": "b7f61c7d089c2c76008da9d73dbee5a0451b89898cea0822629a68bdb9c0dd82" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu", @@ -3581,7 +3644,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "b396dfd3ca06fc2fb06be16dfa93ad29c01d87c40dd4362eff4ff13258afac27", + "closure_sha256": "ad17a4e73c45688bf92d09cdd169547b50286eff399769740e61b796c5106219", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -3599,7 +3662,7 @@ "path": "aot/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu": "c7933ee9713b9f2b2d2d25c1b134db34af14b39a231d1eca351dc52de08cefad" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu": "b7f61c7d089c2c76008da9d73dbee5a0451b89898cea0822629a68bdb9c0dd82" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu", @@ -3856,7 +3919,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "2c42ea0ecb861bf087f1de9877f91b7f00998c455b6050525fbdbb6528025759", + "closure_sha256": "02401191c1530c8cfec0d1d353c4563ea06195a7e4750f2b366daf77c74a05c1", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -3874,7 +3937,7 @@ "path": "aot/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_kernel.cu": "b9b0965b67a5f02d8ad4a16a33cc776e0bc75520fdac0bec4fca7bdef8db7f5b" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_kernel.cu": "18948661e442aa47878daf3c0e2bac374b52a87a6eaa045c70cb9751da50d857" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_kernel.cu", @@ -3883,7 +3946,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "ff42e4d26ec6585c7f423a6d8459374d9bc11fa8dab3ab6a82c17a100c886758", + "closure_sha256": "5b138c8ffdcd28ee6fb96d2ca4ac8b7f5a7a0bbe799e5fa184f4f11eb1618dfd", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -3901,7 +3964,7 @@ "path": "aot/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_kernel.cu": "b9b0965b67a5f02d8ad4a16a33cc776e0bc75520fdac0bec4fca7bdef8db7f5b" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_kernel.cu": "18948661e442aa47878daf3c0e2bac374b52a87a6eaa045c70cb9751da50d857" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_kernel.cu", @@ -3967,7 +4030,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "84db04041bac3c67a907881719edcd52fce88ed59953a6494d47f36f86ff60de", + "closure_sha256": "0d8060766ee203162d3f3a96b715499a7a8c5b6bce6a3ba484324f1d26c50491", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -3986,7 +4049,7 @@ "path": "aot/cake_dense_projection_det_gemm_b43a23d219b0196769dc_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu": "4014ebb24011a4213e8e4c1d6419bb506f9d8d62e1a9a6a227e7777c6bce023e" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu": "8fe448b51f64bd64e70f613b30e7097ade23c52ea865e519b8410d63c5616bf0" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu", @@ -3995,7 +4058,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "6e64f4f7b392b8f32b95632b7628c1a33c9b3cb60084af3032613df788aa497a", + "closure_sha256": "1d15f61b5a858132c43461e177eb12bd0a278aa22dc820b54d5693cba06e70c3", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -4014,7 +4077,7 @@ "path": "aot/cake_dense_projection_det_gemm_b43a23d219b0196769dc_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu": "4014ebb24011a4213e8e4c1d6419bb506f9d8d62e1a9a6a227e7777c6bce023e" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu": "8fe448b51f64bd64e70f613b30e7097ade23c52ea865e519b8410d63c5616bf0" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu", @@ -4023,7 +4086,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "2bbb318679330bf55b09155e1961e06d4911dfc991d21530e7d9a8ceeaeaad90", + "closure_sha256": "4a72445cb28befc20043bcfb39e6d3c8c673e0a80db9593c238f08d45f67a676", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4042,7 +4105,7 @@ "path": "aot/cake_dense_projection_det_gemm_b58324007fd26df8cf2c_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "6cfbd8ec7854b6933dc238c3b7114cf64f7bd47a00c4c32007f87965a9fe7a14" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "30b0f5a712f5426c8829b78ba88ea841000fabaa8525f96bec060d10fc79d3c8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu", @@ -4051,7 +4114,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "af5a1d2927191154d67e6ddc8c920b33a69cf41f7ce8d37ca19bf36a1e17d827", + "closure_sha256": "648464c6f50bbec1842dd588dfc2678c58827147309faf2d1dab5e335dbce1ea", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4069,7 +4132,7 @@ "path": "aot/cake_dense_projection_det_gemm_b58b50619ef5ed46e109_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b58b50619ef5ed46e109_kernel.cu": "b32d0b9fb4b33f0aa70ea3fe8c58f675ce897ef2bb40eb1e822fdd3947a67c6f" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b58b50619ef5ed46e109_kernel.cu": "621034476995a1c4cd72ac8416c70c6e2612792f1df36ee79edb428e48095ce7" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b58b50619ef5ed46e109_kernel.cu", @@ -4135,7 +4198,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "f89de504a2a9971934469f4ef3c1e5ff8b1638f2d8fa12484d3906487e403896", + "closure_sha256": "99b21c27d35e9bac1fe9c28739d90bb3aa1b03abcb09c280badafd84228c293a", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4154,7 +4217,7 @@ "path": "aot/cake_dense_projection_det_gemm_b9c3f4eb125b638d7c87_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu": "bede085ad48300d03d11e41cffe483cccc7fac4c3bfddeafff683f4f6f89b1af" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu": "17a8acc279e2ebc7769d2dd35d8e1946662b3717dc0946aa1459e8fc50fb3999" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu", @@ -4277,7 +4340,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "d96191675aea1557bee51dcb6355c4cb9bc3cc72aeb2c4d89e2d7b8097cb19e7", + "closure_sha256": "ca905ceb8ab0816d99c3add74e5ff3b82f9212d5e9f712909caa5e36063e9d99", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -4296,7 +4359,7 @@ "path": "aot/cake_dense_projection_det_gemm_c15429b8d7d808a760ea_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "6cfbd8ec7854b6933dc238c3b7114cf64f7bd47a00c4c32007f87965a9fe7a14" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "30b0f5a712f5426c8829b78ba88ea841000fabaa8525f96bec060d10fc79d3c8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu", @@ -4305,7 +4368,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "66c8ed884b3d734dfac2a6604fa4f2e61b855e7baa3cd6bfa197af9384943e23", + "closure_sha256": "9906aae9405c2bb6b53f5e48b07a882bf0a354430c87be9417dbfac9f7b94ab5", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -4324,7 +4387,7 @@ "path": "aot/cake_dense_projection_det_gemm_c15429b8d7d808a760ea_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "6cfbd8ec7854b6933dc238c3b7114cf64f7bd47a00c4c32007f87965a9fe7a14" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "30b0f5a712f5426c8829b78ba88ea841000fabaa8525f96bec060d10fc79d3c8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu", @@ -4333,7 +4396,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "5cbfae9d664eeb8260beb9d8f7ea37c7076c477bd8fd3330f376c2c5aea500d8", + "closure_sha256": "990c40f862cada54c3493e605051a65ff576668718d1b4406f3faf565cf51fcc", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4352,7 +4415,7 @@ "path": "aot/cake_dense_projection_det_gemm_c15429b8d7d808a760ea_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "6cfbd8ec7854b6933dc238c3b7114cf64f7bd47a00c4c32007f87965a9fe7a14" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "30b0f5a712f5426c8829b78ba88ea841000fabaa8525f96bec060d10fc79d3c8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu", @@ -4539,7 +4602,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "773db93b6d116aaab5d46e6dec52aa730c672ce4d0f067b890892c4a6393fc40", + "closure_sha256": "df6b815135c8c8b0d4f46ae08833ce7e72fb9863da85678e666b59c3ff4bbcf0", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -4557,7 +4620,7 @@ "path": "aot/cake_dense_projection_det_gemm_ca9a846738b1443f7318_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ca9a846738b1443f7318_kernel.cu": "b45cc448b5fe4a595a7f533fcc0179708f486ecdfb3ff3b7e19809702e0d7358" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ca9a846738b1443f7318_kernel.cu": "1c6aef6a9aafccc33a523a7a448e39f51d007f23456a2c41c3014d23f7908239" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ca9a846738b1443f7318_kernel.cu", @@ -4566,7 +4629,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "0785e1e9ab67de5739a866c06ace9618722018e6d2c8a3837f2123440f91d90a", + "closure_sha256": "67b86742adfa2a51a1731c936fb0028dd96c22d2f6a733afd83643211d939391", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -4584,7 +4647,7 @@ "path": "aot/cake_dense_projection_det_gemm_ca9a846738b1443f7318_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ca9a846738b1443f7318_kernel.cu": "b45cc448b5fe4a595a7f533fcc0179708f486ecdfb3ff3b7e19809702e0d7358" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ca9a846738b1443f7318_kernel.cu": "1c6aef6a9aafccc33a523a7a448e39f51d007f23456a2c41c3014d23f7908239" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ca9a846738b1443f7318_kernel.cu", @@ -4689,7 +4752,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "dfea0f8cbe61b18a01cf50b7c39e455a4a901ea2a3cff451618203e019e28d40", + "closure_sha256": "f9fbab671ab7c11b86141adec3bf2611a6611e866b5c883119e6a12c55dafa18", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -4707,7 +4770,7 @@ "path": "aot/cake_dense_projection_det_gemm_d2a5e70bd49083888179_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu": "0bbf4cabf147c218077a81518a9d5e0071bf3dfc0e48399fbd08d724c77fd053" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu": "875753f32f4c59964a225c3a5bc668d0132d0ce49502b19b9f5e16a6013703db" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu", @@ -4716,7 +4779,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "1c7cde2abcee57d44f51c8cbd92d8f55f16cdc10d1cf76c49aceb4ed16b8518a", + "closure_sha256": "96572183aa7f4be5be5455fa2ec6bd6ab7615e701387d0ec62608db215ef45a7", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -4734,7 +4797,7 @@ "path": "aot/cake_dense_projection_det_gemm_d2a5e70bd49083888179_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu": "0bbf4cabf147c218077a81518a9d5e0071bf3dfc0e48399fbd08d724c77fd053" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu": "875753f32f4c59964a225c3a5bc668d0132d0ce49502b19b9f5e16a6013703db" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu", @@ -4743,7 +4806,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "9d31993ac9392dcac7de427d8a9f71301a40c9befc3a0daeded210922dc1e99a", + "closure_sha256": "a63f9ab66eedfd21fcd28028e94cc1944d78079331bbb29cac8e98db7e91cc70", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4761,7 +4824,7 @@ "path": "aot/cake_dense_projection_det_gemm_d2a5e70bd49083888179_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu": "0bbf4cabf147c218077a81518a9d5e0071bf3dfc0e48399fbd08d724c77fd053" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu": "875753f32f4c59964a225c3a5bc668d0132d0ce49502b19b9f5e16a6013703db" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu", @@ -4827,7 +4890,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "33b9a6c5b14004f5044aeda3b3f82de68cb92e0e3d74da1bd411469282bd5f55", + "closure_sha256": "32eba002fcf092026e436067a6b8c55578f5cbbf3961c1101510ac5bbefd18f4", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4845,7 +4908,7 @@ "path": "aot/cake_dense_projection_det_gemm_d6ebaab2415ea7b00de9_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d6ebaab2415ea7b00de9_kernel.cu": "13cf7ebc080eb72fe078948f9a6f53e1797e2dca99fcc24f7a898b258942b6ab" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d6ebaab2415ea7b00de9_kernel.cu": "84f3e993efba4837398cb4abec7aa40294005bdc668102cceb31bb76a89bdcee" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d6ebaab2415ea7b00de9_kernel.cu", @@ -4854,7 +4917,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "f2e9b7cfc00b228e1b802276f45a517d5fdb709b73570c44b945b801918d6b99", + "closure_sha256": "0d61e7494048beda5ee7724db062c0195befc625c464258be9c4d4b773904d2f", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4872,7 +4935,7 @@ "path": "aot/cake_dense_projection_det_gemm_ddff2b911137449c1dfc_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ddff2b911137449c1dfc_kernel.cu": "b46d3808a12f0477ba901a388222328fb1ceaf77893a8a3754226e3f2105e1c4" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ddff2b911137449c1dfc_kernel.cu": "784892bdd210aaa822deb4a8393c3a105bea18f4d04071d8402c4c2f5f724c46" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ddff2b911137449c1dfc_kernel.cu", @@ -4881,7 +4944,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "4be228d987360c961609b4dc5e9fac763bd5c1ff6b93e1f6acee5328817096fe", + "closure_sha256": "83f5b91473fc8e798ccee7e4beb81c47c35f8eb7bed046d013a05ebd22961997", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -4899,7 +4962,7 @@ "path": "aot/cake_dense_projection_det_gemm_de7492eeb6e24b31687a_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_de7492eeb6e24b31687a_kernel.cu": "7e34563bd07d3b3ca3b3fcdcac7ae42996f521f8485044336494765462e609f4" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_de7492eeb6e24b31687a_kernel.cu": "75e7218ee9671a591942b9efdcab1357fab5d9e5da1abd0791dc16f229da91fb" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_de7492eeb6e24b31687a_kernel.cu", @@ -5086,7 +5149,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "9327531d239b57337617f5c68f7e357a7178f11cb533dccd6d55bad1b722c6c7", + "closure_sha256": "ad76a3956166dcc4f44cd134a80fef7edaae3b6eb96fc5bd6da073910afc48c6", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -5105,7 +5168,7 @@ "path": "aot/cake_dense_projection_det_gemm_e2692c71bc236502ba68_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu": "85ab847a189bc83ccbb9aa99f77c2ad0a85e841602b7b378bc92a68462fd65a0" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu": "1da60114141530b5a0c3706ef1886c87dc65bbac7d7d8e3b344b0b11acdedca1" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu", @@ -5152,7 +5215,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "efacb37a546bca13a2551c284cc29024aa557b6d6e6c740ea063db11df5c4ae1", + "closure_sha256": "1e10fc741126f7ffc52b968d0125d8e5407da1a765e95f19859972dedf56dcd7", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -5171,7 +5234,7 @@ "path": "aot/cake_dense_projection_det_gemm_e51cf65dcafa642e2c51_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "6cfbd8ec7854b6933dc238c3b7114cf64f7bd47a00c4c32007f87965a9fe7a14" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "30b0f5a712f5426c8829b78ba88ea841000fabaa8525f96bec060d10fc79d3c8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu", @@ -5180,7 +5243,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "04a942f891a7005aeeef8e5e500ae96e107d0e21eac27baf45465a0fedd0827e", + "closure_sha256": "a471256765e9edfc521be0a98f5702f99105c25d2a698f3a9099ef48b725f7c0", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -5199,7 +5262,7 @@ "path": "aot/cake_dense_projection_det_gemm_e51cf65dcafa642e2c51_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "6cfbd8ec7854b6933dc238c3b7114cf64f7bd47a00c4c32007f87965a9fe7a14" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "30b0f5a712f5426c8829b78ba88ea841000fabaa8525f96bec060d10fc79d3c8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu", @@ -5208,7 +5271,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "6837710b551b8f3928cb562724030a6bb59dee5b14a0a01dbed74862c7ec7e27", + "closure_sha256": "1f4f3a56e97f936ea8d0d6500043c527755dd96a701bfd73f45b2ddfcddad7e0", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -5227,7 +5290,7 @@ "path": "aot/cake_dense_projection_det_gemm_e51cf65dcafa642e2c51_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "6cfbd8ec7854b6933dc238c3b7114cf64f7bd47a00c4c32007f87965a9fe7a14" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu": "30b0f5a712f5426c8829b78ba88ea841000fabaa8525f96bec060d10fc79d3c8" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu", @@ -5461,7 +5524,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "f27aa8b550815afa00feca83440dbb5a054c7d06864078bb9104fc9ac338dc76", + "closure_sha256": "168c9418dca9569531bdd5b68ca5c1384f2561c5b84ed835e4ed87354cfa45c8", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -5480,13 +5543,76 @@ "path": "aot/cake_dense_projection_det_gemm_ed6ef54b238c5da2f5a7_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu": "c7933ee9713b9f2b2d2d25c1b134db34af14b39a231d1eca351dc52de08cefad" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu": "b7f61c7d089c2c76008da9d73dbee5a0451b89898cea0822629a68bdb9c0dd82" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu", "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu" ] }, + { + "arch": "sm_100a", + "closure_sha256": "618f0c9c0890656e893aaab41c7781866f9948d0ed6111faab0ff8dc561bee63", + "entry": "run", + "gencode": "-gencode=arch=compute_100a,code=sm_100a", + "min_cuda": "12.8", + "name": "cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_sm_100a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_sm_100a.so", + "sha256": { + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_binding.cu": "e6b7848df3e2e3f8745ceb0138c21594e775a601a6440b2b0ba64bd1bad49374", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_kernel.cu": "c1a3259e49f8f36a09aadaded9ec05a7731233fa1958550061b961f6e4c9805d" + }, + "sources": [ + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_kernel.cu", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_binding.cu" + ] + }, + { + "arch": "sm_103a", + "closure_sha256": "72d0d9f7fabd5a84e801144df8a3904165ff3455f04e3d04b93ab2a58e535dec", + "entry": "run", + "gencode": "-gencode=arch=compute_103a,code=sm_103a", + "min_cuda": "12.9", + "name": "cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_sm_103a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_sm_103a.so", + "sha256": { + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_binding.cu": "e6b7848df3e2e3f8745ceb0138c21594e775a601a6440b2b0ba64bd1bad49374", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_kernel.cu": "c1a3259e49f8f36a09aadaded9ec05a7731233fa1958550061b961f6e4c9805d" + }, + "sources": [ + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_kernel.cu", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "2fb98eca4619e9d9c191b3d257a764bc02da8ad31734e2cc371838d6d0c70c63", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_sm_107a", + "nvcc_flags": [ + "--use_fast_math" + ], + "nvcc_flags_if": [], + "path": "aot/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_sm_107a.so", + "sha256": { + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_binding.cu": "e6b7848df3e2e3f8745ceb0138c21594e775a601a6440b2b0ba64bd1bad49374", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_kernel.cu": "c1a3259e49f8f36a09aadaded9ec05a7731233fa1958550061b961f6e4c9805d" + }, + "sources": [ + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_kernel.cu", + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_binding.cu" + ] + }, { "arch": "sm_100a", "closure_sha256": "7fb694b808a0500f2f1dade1533070a6c9145ffb391e73cbb65e6b7df763a8a9", @@ -5706,7 +5832,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "6f2afa7d2347222840b133429ca6f24d157444fb5c152296c5cbe2a26eace61c", + "closure_sha256": "38d9b79c843dc5301ad0f842531af44cb5b5096e07cc658a2a3d9be2b1a66622", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -5724,7 +5850,7 @@ "path": "aot/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu": "be1b79f7b5d001a0d5d4383c3dffc782ea44080400fd515a0d6d2864beda6a43" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu": "ef62ded66c461910e8cc6efc431489705fd66aa5ecef1dc4748a975baf1ec887" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu", @@ -5733,7 +5859,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "114a4577c48255721ff39e8b1a1577fc08465430a7b558e9e836f6ef8e24451c", + "closure_sha256": "11ccc287ecf3b77b177d8d4f2f2b2e9cac12b0e88c3a0bd2bdf9e1056882bc23", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -5751,7 +5877,7 @@ "path": "aot/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu": "be1b79f7b5d001a0d5d4383c3dffc782ea44080400fd515a0d6d2864beda6a43" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu": "ef62ded66c461910e8cc6efc431489705fd66aa5ecef1dc4748a975baf1ec887" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu", @@ -5760,7 +5886,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "9adbdb4748e1443157a42bb05451f2eeb4e1c7fcec0fd51894eb02be8e484a82", + "closure_sha256": "ff2ee1056cd5fb03589f9c5f547cd1a401dbe3e5141e9bab83546d56f9a5e9bc", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -5778,7 +5904,7 @@ "path": "aot/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu": "be1b79f7b5d001a0d5d4383c3dffc782ea44080400fd515a0d6d2864beda6a43" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu": "ef62ded66c461910e8cc6efc431489705fd66aa5ecef1dc4748a975baf1ec887" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu", @@ -5787,7 +5913,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "25ad0805ae784673b4180e6da3ca65240c5d031f10b7c6261f7a4b514d41b49d", + "closure_sha256": "fdc088446850a9bb8f8701ba2774240906d5d1c2375a0f3af3c3cb9e3ab61a17", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -5805,7 +5931,7 @@ "path": "aot/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_sm_100a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_kernel.cu": "6ca7d7b9e15bfa6243a0ab38d7f1e3edea78a240326cb2cbbd21a4376627397b" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_kernel.cu": "2335bb6489582197097a3ccf5fa12f04afab918b31cb33436c96e07e17f8e844" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_kernel.cu", @@ -5814,7 +5940,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "f04a8eafe07d8a766a6923e39a7a9e8912c6dfbe1d07e5031c5fa60eca68163f", + "closure_sha256": "a6998a5516319c221234ee3dc1f96e8777aac8d91b97cbc3e6763328c9462867", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -5832,7 +5958,7 @@ "path": "aot/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_sm_103a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_kernel.cu": "6ca7d7b9e15bfa6243a0ab38d7f1e3edea78a240326cb2cbbd21a4376627397b" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_kernel.cu": "2335bb6489582197097a3ccf5fa12f04afab918b31cb33436c96e07e17f8e844" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_kernel.cu", @@ -6076,7 +6202,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "172dfa28ee565df4d2db979ecf0151ac35205be8493886dfd89eb661b4e2a2ca", + "closure_sha256": "5b2ccb89874b68f7429c06dedae87fbbf125349dd3da68fa3cd46bb3583ec657", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -6094,7 +6220,7 @@ "path": "aot/cake_dense_projection_det_gemm_fb4bcdf13915dfb96c59_sm_107a.so", "sha256": { "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_binding.cu": "0fb4890331d7801e97838f86d6590df317cde6106208d5975216f2676053472f", - "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_fb4bcdf13915dfb96c59_kernel.cu": "a80980d337b242cdde1b05c602013c474191d9cf1f93f89538b1a43517435b3b" + "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_fb4bcdf13915dfb96c59_kernel.cu": "3fde3897ea5821bb387b33af3c4783e1fe387705ff52a992dbfe351466af02c1" }, "sources": [ "csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_fb4bcdf13915dfb96c59_kernel.cu", diff --git a/flashinfer/experimental/dense_projection_det_gemm/cake_backend.py b/flashinfer/experimental/dense_projection_det_gemm/cake_backend.py index 983523bf6ba..430773a57bf 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/cake_backend.py +++ b/flashinfer/experimental/dense_projection_det_gemm/cake_backend.py @@ -53,6 +53,10 @@ multi-wave problems stay data-parallel) and binds the argument plan of the registered module (``cake_jit.MODULES`` / ``cake_jit.KERNELS``). The result is bitwise identical to the Cake launcher's on the same device. +The convenience path runs a batched forward of at most 256 tokens on the +swapped program instead (the weight rows in the MMA's M, at most two waves of +single-CTA tiles over the caller's strided views; :func:`swap_ab_template`), +which takes ``enable_pdl`` per call (:func:`dense_projection_det_gemm`). K2 -- FP32 router GEMM through split-BF16x3 emulation ---------------------------------------------------- @@ -4285,11 +4289,194 @@ def _launch_cached_gemm( return out +# --- the swapped small-token forward ------------------------------------------------------- +# +# A batched forward row the planner would swap (:func:`swap_small_m`: few tokens against many weight rows) runs on a +# program that puts the weight rows in the MMA's M and the tokens in its N (``swap_ab_`` templates): the +# weights stream through a deep TMA ring, at most two waves of single-CTA tiles, and every output element is the one FP32 sum +# of its whole K, rounded once (no split, so the bits do not depend on the tile). The program reads ``A`` and the +# K-major ``B`` through tensor maps over the caller's views and stores into ``out`` at +# ``l * out_l + m * ldo + n``, so token-major views run without a copy. Its weight loads run ahead of the PDL wait +# (only the token loads wait): with ``enable_pdl`` the call overlaps its weight stream with the predecessor's tail, +# which requires that no kernel still in flight on the stream writes ``B``. +SWAP_TILE_T = 64 # tokens per CTA of the swapped programs +#: Weight rows per CTA -> the template serving them, in preference order: the narrow tile while its CTAs fit one +#: wave (more CTAs streaming the weights), the wide one otherwise. +SWAP_TEMPLATES: Tuple[Tuple[int, str], ...] = ( + (64, "swap_ab_w64_t64_s12"), + (128, "swap_ab_w128_t64_s8"), +) +#: The most waves of single-CTA tiles a swapped forward runs in. Its CTAs each stream their weight rows once; past +#: two waves the planner's pair tiles stream them faster (GB300: 8 x 4096 x 1024 at 64 tokens, 256 CTAs, 17.0 us +#: against 20.5 us; at 128 tokens, 512 CTAs, 25.1 us against 21.7 us). +SWAP_MAX_WAVES = 2 + + +@dataclass(frozen=True) +class _SwapLaunch: + """A routed swapped forward: the module entry, its grid and the scalars of its argument plan.""" + + module_name: str + entry: Callable[..., Any] = field(repr=False) + grid: Tuple[int, int, int] + M: int + N: int + K: int + L: int + ldo: int + out_l: int + + +def _aligned_view(t: torch.Tensor) -> bool: + """A ``[L, rows, K]`` K-major view the swapped program's tensor maps take: unit K stride, row and batch strides + in whole 16-byte units, a 16-byte aligned start.""" + return ( + t.stride(2) == 1 + and t.stride(1) % 8 == 0 + and t.stride(0) % 8 == 0 + and t.data_ptr() % 16 == 0 + ) + + +def swap_ab_template(L: int, M: int, N: int, sm_count: int, arch: str) -> Optional[str]: + """The ``swap_ab_`` template a swapped forward of ``L`` batches of ``M`` tokens against ``N`` weight rows + runs on ``arch``: the first of :data:`SWAP_TEMPLATES` whose CTAs fit one wave of the device's SMs, else the + widest; ``None`` when that one needs more than :data:`SWAP_MAX_WAVES` waves (the planner's program is faster + there) or ``arch`` registers none of them.""" + table = KERNELS.get(arch, {}) + registered = [(w, t) for w, t in SWAP_TEMPLATES if t in table] + if not registered: + return None + m_tiles = -(-M // SWAP_TILE_T) + for tile_w, template in registered: + if -(-N // tile_w) * m_tiles * L <= sm_count: + return template + tile_w, template = registered[-1] + if -(-N // tile_w) * m_tiles * L > SWAP_MAX_WAVES * sm_count: + return None + return template + + +def _swap_ab_launch( + A: torch.Tensor, B: torch.Tensor, out: torch.Tensor, transposed_out: bool +) -> Optional[_SwapLaunch]: + """The swapped launch of a K1 call, or ``None`` when the call stays on the planner: a batched forward + (``A`` K-major, ``B`` K-major, BF16 out, no transposed store) that :func:`swap_small_m` selects, with ``K`` a + multiple of 64, the views' strides and starts in 16-byte units and a registered swapped program.""" + if ( + transposed_out + or out.dtype != torch.bfloat16 + or A.dtype != torch.bfloat16 + or B.dtype != torch.bfloat16 + or A.dim() != 3 + or B.dim() != 3 + or out.dim() != 3 + ): + return None + L, M, K = (int(s) for s in A.shape) + N = int(B.shape[2]) + if ( + tuple(B.shape) != (L, K, N) + or tuple(out.shape) != (L, M, N) + or not swap_small_m(L, M, N, transposed_out) + or K % BLOCK_K != 0 + or N % 8 != 0 + ): + return None + W = B.transpose(1, 2) + if not (_aligned_view(A) and _aligned_view(W)): + return None + ldo = int(out.stride(1)) if M > 1 else N + out_l = int(out.stride(0)) + if ( + out.stride(2) != 1 + or ldo % 8 != 0 + or ldo < N + or out_l % 8 != 0 + or out.data_ptr() % 16 != 0 + or max(ldo, out_l) > 2**31 - 1 + ): + return None + device = _same_cuda_device(A, B, out) + facts = device_facts(device) + if facts.arch is None: + return None + template = swap_ab_template(L, M, N, facts.sm_count, facts.arch) + if template is None: + return None + module_name = select_module(facts.arch, template) + tile_w = dict((t, w) for w, t in SWAP_TEMPLATES)[template] + grid = (-(-N // tile_w), -(-M // SWAP_TILE_T), L) + module = load_cake_dense_projection_det_gemm_module(module_name, facts.arch) + entry = getattr(module, MODULES[module_name]["ffi_entry"]) + return _SwapLaunch(module_name, entry, grid, M, N, K, L, ldo, out_l) + + +_SWAP_CACHE: "OrderedDict[tuple, Union[_SwapLaunch, bool]]" = OrderedDict() + + +def _swap_ab_cached( + A: torch.Tensor, B: torch.Tensor, out: torch.Tensor, transposed_out: bool +) -> Optional[_SwapLaunch]: + """:func:`_swap_ab_launch` cached by the device and the geometry of the views (a routed launch, or ``False`` + for a row that stays on the planner) in a least-recently-used cache of ``CONVENIENCE_CACHE_SIZE`` entries.""" + if not (A.is_cuda and B.is_cuda and out.is_cuda): + return None + key = convenience_key( + _cuda_index(out.device), 0, (bool(transposed_out),), A, B, out + ) + hit = _cache_get(_SWAP_CACHE, key) + if hit is None: + launch = _swap_ab_launch(A, B, out, transposed_out) + with _PLAN_LOCK: + hit = _cache_put(_SWAP_CACHE, key, launch if launch is not None else False) + return hit or None + + +def _run_swap_ab( + launch: _SwapLaunch, + A: torch.Tensor, + B: torch.Tensor, + out: torch.Tensor, + enable_pdl: bool, +) -> torch.Tensor: + index = _cuda_index(out.device) + with _ffi_stream_context(index): + launch.entry( + B.transpose(1, 2), + A, + flat_alias(out).view(torch.uint8), + launch.M, + launch.N, + launch.K, + launch.L, + launch.ldo, + launch.out_l, + int(bool(enable_pdl)), + *launch.grid, + ) + return out + + def dense_projection_det_gemm( - A: torch.Tensor, B: torch.Tensor, out: torch.Tensor, *, transposed_out: bool = False + A: torch.Tensor, + B: torch.Tensor, + out: torch.Tensor, + *, + transposed_out: bool = False, + enable_pdl: bool = False, ) -> torch.Tensor: """``out[l, m, n] = sum_k A[l, m, k] B[l, k, n]`` for strided views (the cached convenience path: - planned once per device, stream and view geometry, then rebound and launched).""" + planned once per device, stream and view geometry, then rebound and launched). + + A batched forward row of few tokens (:func:`swap_small_m`) runs the swapped program (:func:`swap_ab_template`), + every other row the planner's program. ``enable_pdl`` launches the swapped program as a programmatic dependent + launch: its weight loads start before the predecessor grid completes and its token loads wait for it, so ``B`` + must not be written by a kernel still in flight on the stream. The planner's programs take no PDL wait and + always launch without the attribute.""" + swap = _swap_ab_cached(A, B, out, transposed_out) + if swap is not None: + return _run_swap_ab(swap, A, B, out, enable_pdl) return _launch_cached_gemm(A, B, out, transposed_out=transposed_out) @@ -4880,6 +5067,7 @@ def clear_convenience_caches() -> None: with _PLAN_LOCK: _K1_CACHE.clear() _K2_CACHE.clear() + _SWAP_CACHE.clear() _ARENAS.clear() diff --git a/flashinfer/experimental/dense_projection_det_gemm/cake_jit.py b/flashinfer/experimental/dense_projection_det_gemm/cake_jit.py index d0cdfe40ac9..6b26c764c9b 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/cake_jit.py +++ b/flashinfer/experimental/dense_projection_det_gemm/cake_jit.py @@ -56,12 +56,14 @@ "775c109309ff": [["tma_buffer", "A"], ["tma_buffer", "B"], ["tma_buffer", "OUT32"], ["tma_buffer", "OUT16"], ["buffer", "out"], ["buffer", "out32"], ["buffer", "ws"], ["buffer", "counters"], ["parameter", "M"], ["parameter", "N"], ["parameter", "m_tiles"], ["parameter", "n_tiles"], ["parameter", "group_m"], ["parameter", "promo_code"], ["parameter", "k_iters"], ["parameter", "ldo"], ["parameter", "out_l"], ["parameter", "num_cluster_tiles"], ["parameter", "num_l"], ["parameter", "num_full"], ["parameter", "iters_per_unit"], ["parameter", "sk_iters"], ["grid", "grid_x"], ["grid", "grid_y"], ["grid", "grid_z"]], "d898389c4086": [["tma_buffer", "A"], ["tma_buffer", "B"], ["buffer", "out32"], ["parameter", "M"], ["parameter", "N"], ["parameter", "m_tiles"], ["parameter", "n_tiles"], ["parameter", "k_iters_split"], ["parameter", "ldo"], ["parameter", "out_l"], ["parameter", "num_cluster_tiles"], ["grid", "grid_x"], ["grid", "grid_y"], ["grid", "grid_z"]], "1fdda92a51e1": [["tma_buffer", "A"], ["tma_buffer", "B"], ["tma_buffer", "OUT32"], ["tma_buffer", "OUT16"], ["buffer", "out"], ["buffer", "out32"], ["buffer", "ws"], ["buffer", "counters"], ["parameter", "M"], ["parameter", "N"], ["parameter", "m_tiles"], ["parameter", "n_tiles"], ["parameter", "group_m"], ["parameter", "promo_code"], ["parameter", "k_iters"], ["parameter", "ldo"], ["parameter", "out_l"], ["parameter", "num_cluster_tiles"], ["parameter", "num_l"], ["parameter", "num_full"], ["parameter", "iters_per_unit"], ["parameter", "sk_iters"], ["parameter", "fd_tiles_l"], ["parameter", "fd_tiles_g"], ["parameter", "fd_group"], ["parameter", "fd_last"], ["grid", "grid_x"], ["grid", "grid_y"], ["grid", "grid_z"]], + "48d32744aae2": [["tma_buffer", "W"], ["tma_buffer", "A"], ["buffer", "out"], ["parameter", "M"], ["parameter", "N"], ["parameter", "K"], ["parameter", "B"], ["parameter", "ldo"], ["parameter", "out_l"], ["launch", "enable_pdl"], ["grid", "grid_x"], ["grid", "grid_y"], ["grid", "grid_z"]], } # fmt: on # fmt: off MODULES: dict[str, dict[str, Any]] = { "cake_dense_projection_det_gemm_011c85b820ec0cf6863d": {"template": "dense_proj_det_gemm_kn_n128_f32_tma1_c1", "kernel": "cake::dense_projection_det_gemm::kn_n128_f32_tma1_c1__pd2", "library": "cake_dense_projection_det_gemm_011c85b820ec0cf6863d", "ffi_entry": "run", "arg_plan": "775c109309ff", "launch": {"block": [320, 1, 1], "cluster": [1, 1, 1]}, "arches": ["sm_107a"]}, + "cake_dense_projection_det_gemm_02803e1d2393113c0f6c": {"template": "swap_ab_w64_t64_s12", "kernel": "kernel_cake_dense_projection_det_gemm_02803e1d2393113c0f6c", "library": "cake_dense_projection_det_gemm_02803e1d2393113c0f6c", "ffi_entry": "run", "arg_plan": "48d32744aae2", "launch": {"block": [256, 1, 1], "cluster": [1, 1, 1]}, "arches": ["sm_100a", "sm_103a", "sm_107a"], "build": {"module": "tart.bmm_bf16.sm100_sm103_sm107.swap_ab_bmm", "kwargs": {"tile_w": 64, "tile_t": 64, "stages": 12, "w_evict_first": True, "strided": True, "weights_early": True}}}, "cake_dense_projection_det_gemm_0618ae0aec1af120afd0": {"template": "dense_proj_det_gemm_kn_n256_tma1_pd2_pro15", "kernel": "kernel_cake_dense_projection_det_gemm_0618ae0aec1af120afd0", "library": "cake_dense_projection_det_gemm_0618ae0aec1af120afd0", "ffi_entry": "run", "arg_plan": "775c109309ff", "launch": {"block": [320, 1, 1], "cluster": [2, 1, 1]}, "arches": ["sm_100a", "sm_103a"]}, "cake_dense_projection_det_gemm_087ba4f157914949c26a": {"template": "dense_proj_det_gemm_kk_n256_ht_pro15", "kernel": "kernel_cake_dense_projection_det_gemm_087ba4f157914949c26a", "library": "cake_dense_projection_det_gemm_087ba4f157914949c26a", "ffi_entry": "run", "arg_plan": "775c109309ff", "launch": {"block": [320, 1, 1], "cluster": [2, 1, 1]}, "arches": ["sm_107a"]}, "cake_dense_projection_det_gemm_0a71a49da71c7d34ab37": {"template": "dense_proj_det_gemm_kk_n128_hen_pro15", "kernel": "kernel_cake_dense_projection_det_gemm_0a71a49da71c7d34ab37", "library": "cake_dense_projection_det_gemm_0a71a49da71c7d34ab37", "ffi_entry": "run", "arg_plan": "775c109309ff", "launch": {"block": [320, 1, 1], "cluster": [2, 1, 1]}, "arches": ["sm_107a"]}, @@ -176,6 +178,7 @@ "cake_dense_projection_det_gemm_ec7277a904a7211d67ce": {"template": "dense_proj_det_gemm_kk_n256_ht_cg_pro15", "kernel": "kernel_cake_dense_projection_det_gemm_ec7277a904a7211d67ce", "library": "cake_dense_projection_det_gemm_ec7277a904a7211d67ce", "ffi_entry": "run", "arg_plan": "775c109309ff", "launch": {"block": [320, 1, 1], "cluster": [2, 1, 1]}, "arches": ["sm_100a", "sm_103a", "sm_107a"], "build": {"module": "tart.bmm_bf16.sm100_sm103_sm107.dense_projection_det_gemm", "kwargs": {"cgrp": True, "htail": True, "pro": 12}}}, "cake_dense_projection_det_gemm_ed1f3ab86ddc55345e69": {"template": "dense_proj_det_gemm_kk_n192_m256_hen_s5_pro15", "kernel": "kernel_cake_dense_projection_det_gemm_ed1f3ab86ddc55345e69", "library": "cake_dense_projection_det_gemm_ed1f3ab86ddc55345e69", "ffi_entry": "run", "arg_plan": "775c109309ff", "launch": {"block": [320, 1, 1], "cluster": [2, 1, 1]}, "arches": ["sm_107a"], "build": {"module": "tart.bmm_bf16.sm100_sm103_sm107.dense_projection_det_gemm", "kwargs": {"a_hint": "evict_first", "b_hint": "none", "block_n": 192, "cta_rows": 256, "pro": 12, "stages": 5}}}, "cake_dense_projection_det_gemm_ed6ef54b238c5da2f5a7": {"template": "dense_proj_det_gemm_kn_n256_m256_ov_ht_pro15", "kernel": "kernel_cake_dense_projection_det_gemm_ed6ef54b238c5da2f5a7", "library": "cake_dense_projection_det_gemm_ed6ef54b238c5da2f5a7", "ffi_entry": "run", "arg_plan": "775c109309ff", "launch": {"block": [320, 1, 1], "cluster": [2, 1, 1]}, "arches": ["sm_107a"]}, + "cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981": {"template": "swap_ab_w128_t64_s8", "kernel": "kernel_cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981", "library": "cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981", "ffi_entry": "run", "arg_plan": "48d32744aae2", "launch": {"block": [256, 1, 1], "cluster": [1, 1, 1]}, "arches": ["sm_100a", "sm_103a", "sm_107a"], "build": {"module": "tart.bmm_bf16.sm100_sm103_sm107.swap_ab_bmm", "kwargs": {"tile_w": 128, "tile_t": 64, "stages": 8, "w_evict_first": True, "strided": True, "weights_early": True}}}, "cake_dense_projection_det_gemm_eec6f87f8b7f25eb6aa2": {"template": "dense_proj_det_gemm_kn_n256_ht_cg_pro15", "kernel": "kernel_cake_dense_projection_det_gemm_eec6f87f8b7f25eb6aa2", "library": "cake_dense_projection_det_gemm_eec6f87f8b7f25eb6aa2", "ffi_entry": "run", "arg_plan": "775c109309ff", "launch": {"block": [320, 1, 1], "cluster": [2, 1, 1]}, "arches": ["sm_100a", "sm_103a", "sm_107a"], "build": {"module": "tart.bmm_bf16.sm100_sm103_sm107.dense_projection_det_gemm", "kwargs": {"b_mn": True, "cgrp": True, "htail": True, "pro": 12}}}, "cake_dense_projection_det_gemm_f052eb094121b03461ab": {"template": "dense_proj_det_gemm_nn_n256_m256_ht_pro7", "kernel": "cake::dense_projection_det_gemm::nn_n256_m256_pro7__ht", "library": "cake_dense_projection_det_gemm_f052eb094121b03461ab", "ffi_entry": "run", "arg_plan": "775c109309ff", "launch": {"block": [320, 1, 1], "cluster": [2, 1, 1]}, "arches": ["sm_100a", "sm_103a", "sm_107a"]}, "cake_dense_projection_det_gemm_f258d45b7e98001e0c09": {"template": "dense_proj_det_gemm_nn_n128_m64_f32_t_pro7", "kernel": "cake::dense_projection_det_gemm::nn_n128_m64_t_pro7__f32", "library": "cake_dense_projection_det_gemm_f258d45b7e98001e0c09", "ffi_entry": "run", "arg_plan": "775c109309ff", "launch": {"block": [320, 1, 1], "cluster": [2, 1, 1]}, "arches": ["sm_100a", "sm_103a"]}, @@ -193,9 +196,9 @@ # fmt: off KERNELS: dict[str, dict[str, str]] = { - "sm_100a": {"dense_proj_det_gemm_kk_n128_c1": "cake_dense_projection_det_gemm_cc2c62bd9132f8273125", "dense_proj_det_gemm_kk_n128_hen_c1": "cake_dense_projection_det_gemm_c195846d649ef9968762", "dense_proj_det_gemm_kk_n128_hen_pro15": "cake_dense_projection_det_gemm_20a2e5c04427e93572da", "dense_proj_det_gemm_kk_n192_hee_s8_pro15": "cake_dense_projection_det_gemm_56be8af43d5d0822da87", "dense_proj_det_gemm_kk_n192_pro7": "cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e", "dense_proj_det_gemm_kk_n256_cg_pro15": "cake_dense_projection_det_gemm_e33ac44bfff66daaee64", "dense_proj_det_gemm_kk_n256_cg_pro7": "cake_dense_projection_det_gemm_73a471b3db6b5aaf0c32", "dense_proj_det_gemm_kk_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_ec7277a904a7211d67ce", "dense_proj_det_gemm_kk_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_3d7b66f61c710f893b29", "dense_proj_det_gemm_kk_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_295aef2f2e0332de2372", "dense_proj_det_gemm_kk_n256_pro15": "cake_dense_projection_det_gemm_e51cf65dcafa642e2c51", "dense_proj_det_gemm_kk_n256_pro7": "cake_dense_projection_det_gemm_a6ff52c845eac9296b34", "dense_proj_det_gemm_kk_n256_q_pro7": "cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d", "dense_proj_det_gemm_kn_n128_f32_tma1_pro7": "cake_dense_projection_det_gemm_5a06c5e2f33772a33174", "dense_proj_det_gemm_kn_n128_f32_tma2_s4_c1": "cake_dense_projection_det_gemm_29d013dfb72e932eaee5", "dense_proj_det_gemm_kn_n256": "cake_dense_projection_det_gemm_e98b3ee7b774c63d7544", "dense_proj_det_gemm_kn_n256_cg_pro15": "cake_dense_projection_det_gemm_3a87cc189b4a7c1a8d7d", "dense_proj_det_gemm_kn_n256_f32_tma1_pro7": "cake_dense_projection_det_gemm_1350861ef426bd310e0f", "dense_proj_det_gemm_kn_n256_f32_tma2_pd2_pro15": "cake_dense_projection_det_gemm_624eb3d7232071ddaf91", "dense_proj_det_gemm_kn_n256_f32_v8_ef_pro7": "cake_dense_projection_det_gemm_895f9ba7159bea07aedd", "dense_proj_det_gemm_kn_n256_f32_v8_pro7": "cake_dense_projection_det_gemm_7d403fa7b167188ccc24", "dense_proj_det_gemm_kn_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_eec6f87f8b7f25eb6aa2", "dense_proj_det_gemm_kn_n256_ht_tma2_lw_pro15": "cake_dense_projection_det_gemm_7ef28d420de555e7a28d", "dense_proj_det_gemm_kn_n256_m256_f32_v8_ef_ov_ht_pro7": "cake_dense_projection_det_gemm_765ece06955b70a95b29", "dense_proj_det_gemm_kn_n256_m256_ht_or_pro7": "cake_dense_projection_det_gemm_d2ea37e4d394bf40922a", "dense_proj_det_gemm_kn_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_9ffc37c198f2d2f42c1f", "dense_proj_det_gemm_kn_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_9cc6363566760a38b4fe", "dense_proj_det_gemm_kn_n256_m256_tma1_bg4_pro7": "cake_dense_projection_det_gemm_5226b6c854d7db395d7e", "dense_proj_det_gemm_kn_n256_pro15": "cake_dense_projection_det_gemm_1c1874c560375d29237d", "dense_proj_det_gemm_kn_n256_pro7": "cake_dense_projection_det_gemm_c15429b8d7d808a760ea", "dense_proj_det_gemm_kn_n256_tma1_pd2_lw_pro15": "cake_dense_projection_det_gemm_0d487a8f71dac54321ef", "dense_proj_det_gemm_kn_n256_tma1_pd2_pro15": "cake_dense_projection_det_gemm_0618ae0aec1af120afd0", "dense_proj_det_gemm_kn_n256_tma1_pro7": "cake_dense_projection_det_gemm_8053ebc84a22444c52af", "dense_proj_det_gemm_kn_n256_tma1_w8": "cake_dense_projection_det_gemm_0b91f29d88de00658fc5", "dense_proj_det_gemm_kn_n256_tma1_w8r": "cake_dense_projection_det_gemm_a24a3d3fd3ceac3e1cdf", "dense_proj_det_gemm_kn_n256_tma1_w8rs": "cake_dense_projection_det_gemm_bab00fb2b95c93758095", "dense_proj_det_gemm_kn_n256_tma2_pd2_lw_pro15": "cake_dense_projection_det_gemm_4de304a13db1f2a0e850", "dense_proj_det_gemm_kn_n256_tma2_pd2_pro15": "cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c", "dense_proj_det_gemm_kn_n256_tma2_pd2_pro7": "cake_dense_projection_det_gemm_39200edabebcd9eb2edd", "dense_proj_det_gemm_kn_n256_tma2_pd3_c1": "cake_dense_projection_det_gemm_eb2bb538645fe303bcca", "dense_proj_det_gemm_nk_n256_t_pro7": "cake_dense_projection_det_gemm_d2a5e70bd49083888179", "dense_proj_det_gemm_nn_n128_hen_f32_t_pro15": "cake_dense_projection_det_gemm_59fb66ccddd67565e592", "dense_proj_det_gemm_nn_n128_hen_f32_t_sks_pro7": "cake_dense_projection_det_gemm_f993d8de6f29f9e8a1ac", "dense_proj_det_gemm_nn_n128_hen_t_pro15": "cake_dense_projection_det_gemm_51c81f11e3b490b5aebb", "dense_proj_det_gemm_nn_n128_hen_t_sks_pro7": "cake_dense_projection_det_gemm_f589a6aff5e88318c36c", "dense_proj_det_gemm_nn_n128_m64_f32_t_pro7": "cake_dense_projection_det_gemm_f258d45b7e98001e0c09", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_b2a9665ee12fd6afea2b", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_pro7": "cake_dense_projection_det_gemm_3507ceddbf4e88a13231", "dense_proj_det_gemm_nn_n128_m64_hen_t_pro15": "cake_dense_projection_det_gemm_e2555d533680e08880ae", "dense_proj_det_gemm_nn_n128_m64_hen_t_pro7": "cake_dense_projection_det_gemm_f861bf1312b9c7fe78c1", "dense_proj_det_gemm_nn_n128_m64_t_pro7": "cake_dense_projection_det_gemm_0c72e864a4afb5347da6", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_cg_pro15": "cake_dense_projection_det_gemm_29de51d5e9cdd6d22491", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_cg_pro15": "cake_dense_projection_det_gemm_0dd5420b570cabc144ab", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_0bbd5d9f1d72a90ef3ea", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_pro15": "cake_dense_projection_det_gemm_23305bdb787b7c6d7378", "dense_proj_det_gemm_nn_n192_bz64_hne_f32_v8_ef_sks_sb2_pro15": "cake_dense_projection_det_gemm_f6c9b40184336f49390d", "dense_proj_det_gemm_nn_n192_bz64_hne_sks_sb2_pro15": "cake_dense_projection_det_gemm_6579637169bec3c2f1d5", "dense_proj_det_gemm_nn_n192_hee_f32_t_tma1_pro7": "cake_dense_projection_det_gemm_36e10fde6497f82171d4", "dense_proj_det_gemm_nn_n192_hee_t_tma1_pro7": "cake_dense_projection_det_gemm_df649506b6ba6f6e47ef", "dense_proj_det_gemm_nn_n192_m256_t_tma1_pro7": "cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052", "dense_proj_det_gemm_nn_n224_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_13790f025541d7d162a7", "dense_proj_det_gemm_nn_n224_hne_cg_pro15": "cake_dense_projection_det_gemm_aa4f7c2fbd0b9a99e535", "dense_proj_det_gemm_nn_n224_m256_pro7": "cake_dense_projection_det_gemm_c63d1400e64208d7865c", "dense_proj_det_gemm_nn_n256_f32_tma1_cg_pro15": "cake_dense_projection_det_gemm_dfd65a75564d04d3826f", "dense_proj_det_gemm_nn_n256_f32_v8_ef_sks_sb2_pro7": "cake_dense_projection_det_gemm_9475731f174aaefb2e44", "dense_proj_det_gemm_nn_n256_f32_v8_pro7": "cake_dense_projection_det_gemm_b43a23d219b0196769dc", "dense_proj_det_gemm_nn_n256_m256_f32_ht_tma1_pro7": "cake_dense_projection_det_gemm_6235599f21b10d4ab9f2", "dense_proj_det_gemm_nn_n256_m256_f32_tma1_pro7": "cake_dense_projection_det_gemm_6e17162acb2711a87d39", "dense_proj_det_gemm_nn_n256_m256_f32_tma1_sks_sb3_pro7": "cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c", "dense_proj_det_gemm_nn_n256_m256_ht_pro7": "cake_dense_projection_det_gemm_f052eb094121b03461ab", "dense_proj_det_gemm_nn_n256_m256_ht_tma1_pro7": "cake_dense_projection_det_gemm_806927d101fe72fc1317", "dense_proj_det_gemm_nn_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_5568657a526e2040f0bc", "dense_proj_det_gemm_nn_n256_m256_pro7": "cake_dense_projection_det_gemm_367fc93d3342f82a1c9f", "dense_proj_det_gemm_nn_n256_m256_sks_sb2_pro7": "cake_dense_projection_det_gemm_736ad30e83d88796883f", "dense_proj_det_gemm_nn_n256_m256_skx_skf_pro7": "cake_dense_projection_det_gemm_b773cc0e3894fd0ef2e0", "dense_proj_det_gemm_nn_n256_m256_t_tma1_pro7": "cake_dense_projection_det_gemm_ca9a846738b1443f7318", "dense_proj_det_gemm_nn_n256_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_fb766b1be68412e6fda7", "dense_proj_det_gemm_nn_n256_m64_hen_t_pro15": "cake_dense_projection_det_gemm_bab9e9dfd7c6ca5e17c0", "dense_proj_det_gemm_nn_n256_sks_sb2_pro7": "cake_dense_projection_det_gemm_c363601373e2dfb8e9af", "router_fp32_det_gemm_kk": "cake_dense_projection_det_gemm_ebbb505a3077e8da0a50", "router_fp32_det_gemm_kn": "cake_dense_projection_det_gemm_7a0ce55fd7221e354322", "router_fp32_det_gemm_nn_t": "cake_dense_projection_det_gemm_9269f6841d7aeb58471c", "split_fp32_det_bf16x3": "cake_dense_projection_det_gemm_5b4e87b523f799e09384"}, - "sm_103a": {"dense_proj_det_gemm_kk_n128_c1": "cake_dense_projection_det_gemm_cc2c62bd9132f8273125", "dense_proj_det_gemm_kk_n128_hen_c1": "cake_dense_projection_det_gemm_c195846d649ef9968762", "dense_proj_det_gemm_kk_n128_hen_pro15": "cake_dense_projection_det_gemm_20a2e5c04427e93572da", "dense_proj_det_gemm_kk_n192_hee_s8_pro15": "cake_dense_projection_det_gemm_56be8af43d5d0822da87", "dense_proj_det_gemm_kk_n192_pro7": "cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e", "dense_proj_det_gemm_kk_n256_cg_pro15": "cake_dense_projection_det_gemm_e33ac44bfff66daaee64", "dense_proj_det_gemm_kk_n256_cg_pro7": "cake_dense_projection_det_gemm_73a471b3db6b5aaf0c32", "dense_proj_det_gemm_kk_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_ec7277a904a7211d67ce", "dense_proj_det_gemm_kk_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_3d7b66f61c710f893b29", "dense_proj_det_gemm_kk_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_295aef2f2e0332de2372", "dense_proj_det_gemm_kk_n256_pro15": "cake_dense_projection_det_gemm_e51cf65dcafa642e2c51", "dense_proj_det_gemm_kk_n256_pro7": "cake_dense_projection_det_gemm_a6ff52c845eac9296b34", "dense_proj_det_gemm_kk_n256_q_pro7": "cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d", "dense_proj_det_gemm_kn_n128_f32_tma1_pro7": "cake_dense_projection_det_gemm_5a06c5e2f33772a33174", "dense_proj_det_gemm_kn_n128_f32_tma2_s4_c1": "cake_dense_projection_det_gemm_29d013dfb72e932eaee5", "dense_proj_det_gemm_kn_n256": "cake_dense_projection_det_gemm_e98b3ee7b774c63d7544", "dense_proj_det_gemm_kn_n256_cg_pro15": "cake_dense_projection_det_gemm_3a87cc189b4a7c1a8d7d", "dense_proj_det_gemm_kn_n256_f32_tma1_pro7": "cake_dense_projection_det_gemm_1350861ef426bd310e0f", "dense_proj_det_gemm_kn_n256_f32_tma2_pd2_pro15": "cake_dense_projection_det_gemm_624eb3d7232071ddaf91", "dense_proj_det_gemm_kn_n256_f32_v8_ef_pro7": "cake_dense_projection_det_gemm_895f9ba7159bea07aedd", "dense_proj_det_gemm_kn_n256_f32_v8_pro7": "cake_dense_projection_det_gemm_7d403fa7b167188ccc24", "dense_proj_det_gemm_kn_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_eec6f87f8b7f25eb6aa2", "dense_proj_det_gemm_kn_n256_ht_tma2_lw_pro15": "cake_dense_projection_det_gemm_7ef28d420de555e7a28d", "dense_proj_det_gemm_kn_n256_m256_f32_v8_ef_ov_ht_pro7": "cake_dense_projection_det_gemm_765ece06955b70a95b29", "dense_proj_det_gemm_kn_n256_m256_ht_or_pro7": "cake_dense_projection_det_gemm_d2ea37e4d394bf40922a", "dense_proj_det_gemm_kn_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_9ffc37c198f2d2f42c1f", "dense_proj_det_gemm_kn_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_9cc6363566760a38b4fe", "dense_proj_det_gemm_kn_n256_m256_tma1_bg4_pro7": "cake_dense_projection_det_gemm_5226b6c854d7db395d7e", "dense_proj_det_gemm_kn_n256_pro15": "cake_dense_projection_det_gemm_1c1874c560375d29237d", "dense_proj_det_gemm_kn_n256_pro7": "cake_dense_projection_det_gemm_c15429b8d7d808a760ea", "dense_proj_det_gemm_kn_n256_tma1_pd2_lw_mg_pro15": "cake_dense_projection_det_gemm_bb0e7457db664d6f0f6a", "dense_proj_det_gemm_kn_n256_tma1_pd2_lw_pro15": "cake_dense_projection_det_gemm_0d487a8f71dac54321ef", "dense_proj_det_gemm_kn_n256_tma1_pd2_mg_pro15": "cake_dense_projection_det_gemm_4eb96fb2eb1aaa14c7f7", "dense_proj_det_gemm_kn_n256_tma1_pd2_mg_sw25_pro15": "cake_dense_projection_det_gemm_75b1dcdaef42319f5ed4", "dense_proj_det_gemm_kn_n256_tma1_pd2_pro15": "cake_dense_projection_det_gemm_0618ae0aec1af120afd0", "dense_proj_det_gemm_kn_n256_tma1_pd2_sw25_pro15": "cake_dense_projection_det_gemm_552070c21b97faa67a71", "dense_proj_det_gemm_kn_n256_tma1_pro7": "cake_dense_projection_det_gemm_8053ebc84a22444c52af", "dense_proj_det_gemm_kn_n256_tma1_w8r": "cake_dense_projection_det_gemm_a24a3d3fd3ceac3e1cdf", "dense_proj_det_gemm_kn_n256_tma1_w8rs": "cake_dense_projection_det_gemm_bab00fb2b95c93758095", "dense_proj_det_gemm_kn_n256_tma2_pd2_lw_pro15": "cake_dense_projection_det_gemm_4de304a13db1f2a0e850", "dense_proj_det_gemm_kn_n256_tma2_pd2_pro15": "cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c", "dense_proj_det_gemm_kn_n256_tma2_pd2_pro7": "cake_dense_projection_det_gemm_39200edabebcd9eb2edd", "dense_proj_det_gemm_kn_n256_tma2_pd3_c1": "cake_dense_projection_det_gemm_eb2bb538645fe303bcca", "dense_proj_det_gemm_nk_n256_t_pro7": "cake_dense_projection_det_gemm_d2a5e70bd49083888179", "dense_proj_det_gemm_nn_n128_hen_f32_t_pro15": "cake_dense_projection_det_gemm_59fb66ccddd67565e592", "dense_proj_det_gemm_nn_n128_hen_f32_t_sks_pro7": "cake_dense_projection_det_gemm_f993d8de6f29f9e8a1ac", "dense_proj_det_gemm_nn_n128_hen_t_pro15": "cake_dense_projection_det_gemm_51c81f11e3b490b5aebb", "dense_proj_det_gemm_nn_n128_hen_t_sks_pro7": "cake_dense_projection_det_gemm_f589a6aff5e88318c36c", "dense_proj_det_gemm_nn_n128_m64_f32_t_pro7": "cake_dense_projection_det_gemm_f258d45b7e98001e0c09", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_b2a9665ee12fd6afea2b", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_pro7": "cake_dense_projection_det_gemm_3507ceddbf4e88a13231", "dense_proj_det_gemm_nn_n128_m64_hen_t_pro15": "cake_dense_projection_det_gemm_e2555d533680e08880ae", "dense_proj_det_gemm_nn_n128_m64_hen_t_pro7": "cake_dense_projection_det_gemm_f861bf1312b9c7fe78c1", "dense_proj_det_gemm_nn_n128_m64_t_pro7": "cake_dense_projection_det_gemm_0c72e864a4afb5347da6", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_cg_pro15": "cake_dense_projection_det_gemm_29de51d5e9cdd6d22491", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_cg_pro15": "cake_dense_projection_det_gemm_0dd5420b570cabc144ab", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_0bbd5d9f1d72a90ef3ea", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_pro15": "cake_dense_projection_det_gemm_23305bdb787b7c6d7378", "dense_proj_det_gemm_nn_n192_bz64_hne_f32_v8_ef_sks_sb2_pro15": "cake_dense_projection_det_gemm_f6c9b40184336f49390d", "dense_proj_det_gemm_nn_n192_bz64_hne_sks_sb2_pro15": "cake_dense_projection_det_gemm_6579637169bec3c2f1d5", "dense_proj_det_gemm_nn_n192_hee_f32_t_tma1_pro7": "cake_dense_projection_det_gemm_36e10fde6497f82171d4", "dense_proj_det_gemm_nn_n192_hee_t_tma1_pro7": "cake_dense_projection_det_gemm_df649506b6ba6f6e47ef", "dense_proj_det_gemm_nn_n192_m256_t_tma1_pro7": "cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052", "dense_proj_det_gemm_nn_n224_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_13790f025541d7d162a7", "dense_proj_det_gemm_nn_n224_hne_cg_pro15": "cake_dense_projection_det_gemm_aa4f7c2fbd0b9a99e535", "dense_proj_det_gemm_nn_n224_m256_pro7": "cake_dense_projection_det_gemm_c63d1400e64208d7865c", "dense_proj_det_gemm_nn_n256_f32_tma1_cg_pro15": "cake_dense_projection_det_gemm_dfd65a75564d04d3826f", "dense_proj_det_gemm_nn_n256_f32_v8_ef_sks_sb2_pro7": "cake_dense_projection_det_gemm_9475731f174aaefb2e44", "dense_proj_det_gemm_nn_n256_f32_v8_pro7": "cake_dense_projection_det_gemm_b43a23d219b0196769dc", "dense_proj_det_gemm_nn_n256_m256_f32_ht_tma1_pro7": "cake_dense_projection_det_gemm_6235599f21b10d4ab9f2", "dense_proj_det_gemm_nn_n256_m256_f32_tma1_pro7": "cake_dense_projection_det_gemm_6e17162acb2711a87d39", "dense_proj_det_gemm_nn_n256_m256_f32_tma1_sks_sb3_pro7": "cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c", "dense_proj_det_gemm_nn_n256_m256_ht_pro7": "cake_dense_projection_det_gemm_f052eb094121b03461ab", "dense_proj_det_gemm_nn_n256_m256_ht_tma1_pro7": "cake_dense_projection_det_gemm_806927d101fe72fc1317", "dense_proj_det_gemm_nn_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_5568657a526e2040f0bc", "dense_proj_det_gemm_nn_n256_m256_pro7": "cake_dense_projection_det_gemm_367fc93d3342f82a1c9f", "dense_proj_det_gemm_nn_n256_m256_sks_sb2_pro7": "cake_dense_projection_det_gemm_736ad30e83d88796883f", "dense_proj_det_gemm_nn_n256_m256_skx_skf_pro7": "cake_dense_projection_det_gemm_b773cc0e3894fd0ef2e0", "dense_proj_det_gemm_nn_n256_m256_t_tma1_pro7": "cake_dense_projection_det_gemm_ca9a846738b1443f7318", "dense_proj_det_gemm_nn_n256_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_fb766b1be68412e6fda7", "dense_proj_det_gemm_nn_n256_m64_hen_t_pro15": "cake_dense_projection_det_gemm_bab9e9dfd7c6ca5e17c0", "dense_proj_det_gemm_nn_n256_sks_sb2_pro7": "cake_dense_projection_det_gemm_c363601373e2dfb8e9af", "router_fp32_det_gemm_kk": "cake_dense_projection_det_gemm_ebbb505a3077e8da0a50", "router_fp32_det_gemm_kn": "cake_dense_projection_det_gemm_7a0ce55fd7221e354322", "router_fp32_det_gemm_nn_t": "cake_dense_projection_det_gemm_9269f6841d7aeb58471c", "split_fp32_det_bf16x3": "cake_dense_projection_det_gemm_5b4e87b523f799e09384"}, - "sm_107a": {"dense_proj_det_gemm_kk_n128_hen_pro15": "cake_dense_projection_det_gemm_0a71a49da71c7d34ab37", "dense_proj_det_gemm_kk_n128_hen_pro7": "cake_dense_projection_det_gemm_7ad01af3a13775199254", "dense_proj_det_gemm_kk_n128_pro7": "cake_dense_projection_det_gemm_451a0f92e10205e345eb", "dense_proj_det_gemm_kk_n192_m256_hen_s5_pro15": "cake_dense_projection_det_gemm_ed1f3ab86ddc55345e69", "dense_proj_det_gemm_kk_n192_pro7": "cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e", "dense_proj_det_gemm_kk_n256_cg_pro15": "cake_dense_projection_det_gemm_438841f42a0743b63fb5", "dense_proj_det_gemm_kk_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_ec7277a904a7211d67ce", "dense_proj_det_gemm_kk_n256_ht_pro15": "cake_dense_projection_det_gemm_087ba4f157914949c26a", "dense_proj_det_gemm_kk_n256_m256_cg_or_pro7": "cake_dense_projection_det_gemm_4af009e00a1d5c0ad794", "dense_proj_det_gemm_kk_n256_m256_ht_cg_or_pro7": "cake_dense_projection_det_gemm_53a63652be2155219d76", "dense_proj_det_gemm_kk_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_3d7b66f61c710f893b29", "dense_proj_det_gemm_kk_n256_pro15": "cake_dense_projection_det_gemm_e51cf65dcafa642e2c51", "dense_proj_det_gemm_kk_n256_q_pro7": "cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d", "dense_proj_det_gemm_kn_n128_f32_tma1_c1": "cake_dense_projection_det_gemm_011c85b820ec0cf6863d", "dense_proj_det_gemm_kn_n128_f32_tma1_pd2_c1": "cake_dense_projection_det_gemm_55c1b40f71e3fe4a892c", "dense_proj_det_gemm_kn_n256_cg_pro15": "cake_dense_projection_det_gemm_3a87cc189b4a7c1a8d7d", "dense_proj_det_gemm_kn_n256_cg_pro7": "cake_dense_projection_det_gemm_14192244441c1003db74", "dense_proj_det_gemm_kn_n256_f32_tma1_pro15": "cake_dense_projection_det_gemm_b58b50619ef5ed46e109", "dense_proj_det_gemm_kn_n256_f32_tma2_pd2_pro15": "cake_dense_projection_det_gemm_624eb3d7232071ddaf91", "dense_proj_det_gemm_kn_n256_f32_v8_ef_s9_pro15": "cake_dense_projection_det_gemm_31be7faa868eabcedada", "dense_proj_det_gemm_kn_n256_f32_v8_ht_pro7": "cake_dense_projection_det_gemm_b9c3f4eb125b638d7c87", "dense_proj_det_gemm_kn_n256_f32_v8_pro7": "cake_dense_projection_det_gemm_7d403fa7b167188ccc24", "dense_proj_det_gemm_kn_n256_hne_pro7": "cake_dense_projection_det_gemm_66df6446dd2cca3106d3", "dense_proj_det_gemm_kn_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_eec6f87f8b7f25eb6aa2", "dense_proj_det_gemm_kn_n256_ht_pro15": "cake_dense_projection_det_gemm_b58324007fd26df8cf2c", "dense_proj_det_gemm_kn_n256_ht_tma2_lw_pro15": "cake_dense_projection_det_gemm_7ef28d420de555e7a28d", "dense_proj_det_gemm_kn_n256_m256_cg_or_pro7": "cake_dense_projection_det_gemm_13d9310f37868a18eace", "dense_proj_det_gemm_kn_n256_m256_f32_tma1_pro7": "cake_dense_projection_det_gemm_de7492eeb6e24b31687a", "dense_proj_det_gemm_kn_n256_m256_ht_or_pro7": "cake_dense_projection_det_gemm_d2ea37e4d394bf40922a", "dense_proj_det_gemm_kn_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_9ffc37c198f2d2f42c1f", "dense_proj_det_gemm_kn_n256_m256_ov_ht_pro15": "cake_dense_projection_det_gemm_ed6ef54b238c5da2f5a7", "dense_proj_det_gemm_kn_n256_pro15": "cake_dense_projection_det_gemm_1c1874c560375d29237d", "dense_proj_det_gemm_kn_n256_pro7": "cake_dense_projection_det_gemm_c15429b8d7d808a760ea", "dense_proj_det_gemm_kn_n256_q_pd1_pro7": "cake_dense_projection_det_gemm_fb4bcdf13915dfb96c59", "dense_proj_det_gemm_kn_n256_tma1_bg8_pro7": "cake_dense_projection_det_gemm_cb45dbd62c7edd4410f1", "dense_proj_det_gemm_kn_n256_tma1_bg8_sol_pro7": "cake_dense_projection_det_gemm_332b601bf82c22b05ef2", "dense_proj_det_gemm_kn_n256_tma1_pd2_lw_pro15": "cake_dense_projection_det_gemm_0d487a8f71dac54321ef", "dense_proj_det_gemm_kn_n256_tma2_pd2_lw_pro15": "cake_dense_projection_det_gemm_4de304a13db1f2a0e850", "dense_proj_det_gemm_kn_n256_tma2_pd2_pro15": "cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c", "dense_proj_det_gemm_kn_n256_tma2_pd3_c1": "cake_dense_projection_det_gemm_49acb91191bbcced9051", "dense_proj_det_gemm_nk_n256_t_pro7": "cake_dense_projection_det_gemm_d2a5e70bd49083888179", "dense_proj_det_gemm_nn_n128_f32_t_pro7": "cake_dense_projection_det_gemm_563dd919de0e74c71bde", "dense_proj_det_gemm_nn_n128_hen_f32_t_pro15": "cake_dense_projection_det_gemm_59fb66ccddd67565e592", "dense_proj_det_gemm_nn_n128_hen_f32_t_sks_pro7": "cake_dense_projection_det_gemm_f993d8de6f29f9e8a1ac", "dense_proj_det_gemm_nn_n128_hen_f32_t_skx_pro7": "cake_dense_projection_det_gemm_b09222ffcfa7a7e4ead7", "dense_proj_det_gemm_nn_n128_hen_t_pro15": "cake_dense_projection_det_gemm_51c81f11e3b490b5aebb", "dense_proj_det_gemm_nn_n128_hen_t_sks_pro7": "cake_dense_projection_det_gemm_f589a6aff5e88318c36c", "dense_proj_det_gemm_nn_n128_hen_t_skx_pro7": "cake_dense_projection_det_gemm_6f17e515b2e1b7e2fde9", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_b2a9665ee12fd6afea2b", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_skx_pro15": "cake_dense_projection_det_gemm_56c1924c362457a1c3fe", "dense_proj_det_gemm_nn_n128_m64_hen_t_pro15": "cake_dense_projection_det_gemm_e2555d533680e08880ae", "dense_proj_det_gemm_nn_n128_m64_hen_t_skx_pro15": "cake_dense_projection_det_gemm_4f31f3828842f20176d6", "dense_proj_det_gemm_nn_n128_t_pro7": "cake_dense_projection_det_gemm_36d0401698df21c9425f", "dense_proj_det_gemm_nn_n160_m256_bz64_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_5e77ef909e88a99c4357", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_cg_pro15": "cake_dense_projection_det_gemm_29de51d5e9cdd6d22491", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_cg_pro15": "cake_dense_projection_det_gemm_0dd5420b570cabc144ab", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_0bbd5d9f1d72a90ef3ea", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_pro15": "cake_dense_projection_det_gemm_23305bdb787b7c6d7378", "dense_proj_det_gemm_nn_n160_m256_bz64_pro15": "cake_dense_projection_det_gemm_505fbe05ce8b62f0a589", "dense_proj_det_gemm_nn_n192_bz64_hee_f32_t_pro7": "cake_dense_projection_det_gemm_d6ebaab2415ea7b00de9", "dense_proj_det_gemm_nn_n192_bz64_hee_f32_t_sks_sb2_pro7": "cake_dense_projection_det_gemm_fdb106e6e73897521cbb", "dense_proj_det_gemm_nn_n192_bz64_hee_t_sks_stt_pro15": "cake_dense_projection_det_gemm_ddff2b911137449c1dfc", "dense_proj_det_gemm_nn_n192_bz64_hne_f32_v8_ef_sks_sb2_pro15": "cake_dense_projection_det_gemm_f6c9b40184336f49390d", "dense_proj_det_gemm_nn_n192_bz64_hne_sks_sb2_pro15": "cake_dense_projection_det_gemm_6579637169bec3c2f1d5", "dense_proj_det_gemm_nn_n192_m256_bz64_hee_t_tma1_s6_pro7": "cake_dense_projection_det_gemm_9aa9aa60b0c544bfab06", "dense_proj_det_gemm_nn_n224_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_13790f025541d7d162a7", "dense_proj_det_gemm_nn_n224_hne_cg_pro15": "cake_dense_projection_det_gemm_aa4f7c2fbd0b9a99e535", "dense_proj_det_gemm_nn_n224_m256_pro7": "cake_dense_projection_det_gemm_c63d1400e64208d7865c", "dense_proj_det_gemm_nn_n256_f32_tma1_cg_pro15": "cake_dense_projection_det_gemm_dfd65a75564d04d3826f", "dense_proj_det_gemm_nn_n256_f32_v8_ef_sks_sb2_pro7": "cake_dense_projection_det_gemm_9475731f174aaefb2e44", "dense_proj_det_gemm_nn_n256_f32_v8_pro15": "cake_dense_projection_det_gemm_8e1f6805342f620cca8c", "dense_proj_det_gemm_nn_n256_m256_f32_ht_tma1_pro7": "cake_dense_projection_det_gemm_6235599f21b10d4ab9f2", "dense_proj_det_gemm_nn_n256_m256_f32_tma1_pro7": "cake_dense_projection_det_gemm_6e17162acb2711a87d39", "dense_proj_det_gemm_nn_n256_m256_f32_v8_ef_ht_pro7": "cake_dense_projection_det_gemm_5445e0f16ad84868fa01", "dense_proj_det_gemm_nn_n256_m256_hee_t_tma1_s6_pro7": "cake_dense_projection_det_gemm_446497e750658dfb057b", "dense_proj_det_gemm_nn_n256_m256_ht_pro7": "cake_dense_projection_det_gemm_f052eb094121b03461ab", "dense_proj_det_gemm_nn_n256_m256_pro7": "cake_dense_projection_det_gemm_367fc93d3342f82a1c9f", "dense_proj_det_gemm_nn_n256_m256_skx_skf_pro7": "cake_dense_projection_det_gemm_b773cc0e3894fd0ef2e0", "dense_proj_det_gemm_nn_n256_m256_tma1_pro7": "cake_dense_projection_det_gemm_e2692c71bc236502ba68", "dense_proj_det_gemm_nn_n256_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_fb766b1be68412e6fda7", "dense_proj_det_gemm_nn_n256_m64_hen_t_pro15": "cake_dense_projection_det_gemm_bab9e9dfd7c6ca5e17c0", "dense_proj_det_gemm_nn_n256_pro15": "cake_dense_projection_det_gemm_82f698f67a1cca6e4291", "dense_proj_det_gemm_nn_n256_sks_sb2_pro7": "cake_dense_projection_det_gemm_c363601373e2dfb8e9af", "router_fp32_det_gemm_kk": "cake_dense_projection_det_gemm_ebbb505a3077e8da0a50", "router_fp32_det_gemm_kn": "cake_dense_projection_det_gemm_7a0ce55fd7221e354322", "router_fp32_det_gemm_nn_t": "cake_dense_projection_det_gemm_9269f6841d7aeb58471c", "split_fp32_det_bf16x3": "cake_dense_projection_det_gemm_5b4e87b523f799e09384"}, + "sm_100a": {"dense_proj_det_gemm_kk_n128_c1": "cake_dense_projection_det_gemm_cc2c62bd9132f8273125", "dense_proj_det_gemm_kk_n128_hen_c1": "cake_dense_projection_det_gemm_c195846d649ef9968762", "dense_proj_det_gemm_kk_n128_hen_pro15": "cake_dense_projection_det_gemm_20a2e5c04427e93572da", "dense_proj_det_gemm_kk_n192_hee_s8_pro15": "cake_dense_projection_det_gemm_56be8af43d5d0822da87", "dense_proj_det_gemm_kk_n192_pro7": "cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e", "dense_proj_det_gemm_kk_n256_cg_pro15": "cake_dense_projection_det_gemm_e33ac44bfff66daaee64", "dense_proj_det_gemm_kk_n256_cg_pro7": "cake_dense_projection_det_gemm_73a471b3db6b5aaf0c32", "dense_proj_det_gemm_kk_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_ec7277a904a7211d67ce", "dense_proj_det_gemm_kk_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_3d7b66f61c710f893b29", "dense_proj_det_gemm_kk_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_295aef2f2e0332de2372", "dense_proj_det_gemm_kk_n256_pro15": "cake_dense_projection_det_gemm_e51cf65dcafa642e2c51", "dense_proj_det_gemm_kk_n256_pro7": "cake_dense_projection_det_gemm_a6ff52c845eac9296b34", "dense_proj_det_gemm_kk_n256_q_pro7": "cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d", "dense_proj_det_gemm_kn_n128_f32_tma1_pro7": "cake_dense_projection_det_gemm_5a06c5e2f33772a33174", "dense_proj_det_gemm_kn_n128_f32_tma2_s4_c1": "cake_dense_projection_det_gemm_29d013dfb72e932eaee5", "dense_proj_det_gemm_kn_n256": "cake_dense_projection_det_gemm_e98b3ee7b774c63d7544", "dense_proj_det_gemm_kn_n256_cg_pro15": "cake_dense_projection_det_gemm_3a87cc189b4a7c1a8d7d", "dense_proj_det_gemm_kn_n256_f32_tma1_pro7": "cake_dense_projection_det_gemm_1350861ef426bd310e0f", "dense_proj_det_gemm_kn_n256_f32_tma2_pd2_pro15": "cake_dense_projection_det_gemm_624eb3d7232071ddaf91", "dense_proj_det_gemm_kn_n256_f32_v8_ef_pro7": "cake_dense_projection_det_gemm_895f9ba7159bea07aedd", "dense_proj_det_gemm_kn_n256_f32_v8_pro7": "cake_dense_projection_det_gemm_7d403fa7b167188ccc24", "dense_proj_det_gemm_kn_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_eec6f87f8b7f25eb6aa2", "dense_proj_det_gemm_kn_n256_ht_tma2_lw_pro15": "cake_dense_projection_det_gemm_7ef28d420de555e7a28d", "dense_proj_det_gemm_kn_n256_m256_f32_v8_ef_ov_ht_pro7": "cake_dense_projection_det_gemm_765ece06955b70a95b29", "dense_proj_det_gemm_kn_n256_m256_ht_or_pro7": "cake_dense_projection_det_gemm_d2ea37e4d394bf40922a", "dense_proj_det_gemm_kn_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_9ffc37c198f2d2f42c1f", "dense_proj_det_gemm_kn_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_9cc6363566760a38b4fe", "dense_proj_det_gemm_kn_n256_m256_tma1_bg4_pro7": "cake_dense_projection_det_gemm_5226b6c854d7db395d7e", "dense_proj_det_gemm_kn_n256_pro15": "cake_dense_projection_det_gemm_1c1874c560375d29237d", "dense_proj_det_gemm_kn_n256_pro7": "cake_dense_projection_det_gemm_c15429b8d7d808a760ea", "dense_proj_det_gemm_kn_n256_tma1_pd2_lw_pro15": "cake_dense_projection_det_gemm_0d487a8f71dac54321ef", "dense_proj_det_gemm_kn_n256_tma1_pd2_pro15": "cake_dense_projection_det_gemm_0618ae0aec1af120afd0", "dense_proj_det_gemm_kn_n256_tma1_pro7": "cake_dense_projection_det_gemm_8053ebc84a22444c52af", "dense_proj_det_gemm_kn_n256_tma1_w8": "cake_dense_projection_det_gemm_0b91f29d88de00658fc5", "dense_proj_det_gemm_kn_n256_tma1_w8r": "cake_dense_projection_det_gemm_a24a3d3fd3ceac3e1cdf", "dense_proj_det_gemm_kn_n256_tma1_w8rs": "cake_dense_projection_det_gemm_bab00fb2b95c93758095", "dense_proj_det_gemm_kn_n256_tma2_pd2_lw_pro15": "cake_dense_projection_det_gemm_4de304a13db1f2a0e850", "dense_proj_det_gemm_kn_n256_tma2_pd2_pro15": "cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c", "dense_proj_det_gemm_kn_n256_tma2_pd2_pro7": "cake_dense_projection_det_gemm_39200edabebcd9eb2edd", "dense_proj_det_gemm_kn_n256_tma2_pd3_c1": "cake_dense_projection_det_gemm_eb2bb538645fe303bcca", "dense_proj_det_gemm_nk_n256_t_pro7": "cake_dense_projection_det_gemm_d2a5e70bd49083888179", "dense_proj_det_gemm_nn_n128_hen_f32_t_pro15": "cake_dense_projection_det_gemm_59fb66ccddd67565e592", "dense_proj_det_gemm_nn_n128_hen_f32_t_sks_pro7": "cake_dense_projection_det_gemm_f993d8de6f29f9e8a1ac", "dense_proj_det_gemm_nn_n128_hen_t_pro15": "cake_dense_projection_det_gemm_51c81f11e3b490b5aebb", "dense_proj_det_gemm_nn_n128_hen_t_sks_pro7": "cake_dense_projection_det_gemm_f589a6aff5e88318c36c", "dense_proj_det_gemm_nn_n128_m64_f32_t_pro7": "cake_dense_projection_det_gemm_f258d45b7e98001e0c09", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_b2a9665ee12fd6afea2b", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_pro7": "cake_dense_projection_det_gemm_3507ceddbf4e88a13231", "dense_proj_det_gemm_nn_n128_m64_hen_t_pro15": "cake_dense_projection_det_gemm_e2555d533680e08880ae", "dense_proj_det_gemm_nn_n128_m64_hen_t_pro7": "cake_dense_projection_det_gemm_f861bf1312b9c7fe78c1", "dense_proj_det_gemm_nn_n128_m64_t_pro7": "cake_dense_projection_det_gemm_0c72e864a4afb5347da6", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_cg_pro15": "cake_dense_projection_det_gemm_29de51d5e9cdd6d22491", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_cg_pro15": "cake_dense_projection_det_gemm_0dd5420b570cabc144ab", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_0bbd5d9f1d72a90ef3ea", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_pro15": "cake_dense_projection_det_gemm_23305bdb787b7c6d7378", "dense_proj_det_gemm_nn_n192_bz64_hne_f32_v8_ef_sks_sb2_pro15": "cake_dense_projection_det_gemm_f6c9b40184336f49390d", "dense_proj_det_gemm_nn_n192_bz64_hne_sks_sb2_pro15": "cake_dense_projection_det_gemm_6579637169bec3c2f1d5", "dense_proj_det_gemm_nn_n192_hee_f32_t_tma1_pro7": "cake_dense_projection_det_gemm_36e10fde6497f82171d4", "dense_proj_det_gemm_nn_n192_hee_t_tma1_pro7": "cake_dense_projection_det_gemm_df649506b6ba6f6e47ef", "dense_proj_det_gemm_nn_n192_m256_t_tma1_pro7": "cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052", "dense_proj_det_gemm_nn_n224_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_13790f025541d7d162a7", "dense_proj_det_gemm_nn_n224_hne_cg_pro15": "cake_dense_projection_det_gemm_aa4f7c2fbd0b9a99e535", "dense_proj_det_gemm_nn_n224_m256_pro7": "cake_dense_projection_det_gemm_c63d1400e64208d7865c", "dense_proj_det_gemm_nn_n256_f32_tma1_cg_pro15": "cake_dense_projection_det_gemm_dfd65a75564d04d3826f", "dense_proj_det_gemm_nn_n256_f32_v8_ef_sks_sb2_pro7": "cake_dense_projection_det_gemm_9475731f174aaefb2e44", "dense_proj_det_gemm_nn_n256_f32_v8_pro7": "cake_dense_projection_det_gemm_b43a23d219b0196769dc", "dense_proj_det_gemm_nn_n256_m256_f32_ht_tma1_pro7": "cake_dense_projection_det_gemm_6235599f21b10d4ab9f2", "dense_proj_det_gemm_nn_n256_m256_f32_tma1_pro7": "cake_dense_projection_det_gemm_6e17162acb2711a87d39", "dense_proj_det_gemm_nn_n256_m256_f32_tma1_sks_sb3_pro7": "cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c", "dense_proj_det_gemm_nn_n256_m256_ht_pro7": "cake_dense_projection_det_gemm_f052eb094121b03461ab", "dense_proj_det_gemm_nn_n256_m256_ht_tma1_pro7": "cake_dense_projection_det_gemm_806927d101fe72fc1317", "dense_proj_det_gemm_nn_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_5568657a526e2040f0bc", "dense_proj_det_gemm_nn_n256_m256_pro7": "cake_dense_projection_det_gemm_367fc93d3342f82a1c9f", "dense_proj_det_gemm_nn_n256_m256_sks_sb2_pro7": "cake_dense_projection_det_gemm_736ad30e83d88796883f", "dense_proj_det_gemm_nn_n256_m256_skx_skf_pro7": "cake_dense_projection_det_gemm_b773cc0e3894fd0ef2e0", "dense_proj_det_gemm_nn_n256_m256_t_tma1_pro7": "cake_dense_projection_det_gemm_ca9a846738b1443f7318", "dense_proj_det_gemm_nn_n256_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_fb766b1be68412e6fda7", "dense_proj_det_gemm_nn_n256_m64_hen_t_pro15": "cake_dense_projection_det_gemm_bab9e9dfd7c6ca5e17c0", "dense_proj_det_gemm_nn_n256_sks_sb2_pro7": "cake_dense_projection_det_gemm_c363601373e2dfb8e9af", "router_fp32_det_gemm_kk": "cake_dense_projection_det_gemm_ebbb505a3077e8da0a50", "router_fp32_det_gemm_kn": "cake_dense_projection_det_gemm_7a0ce55fd7221e354322", "router_fp32_det_gemm_nn_t": "cake_dense_projection_det_gemm_9269f6841d7aeb58471c", "split_fp32_det_bf16x3": "cake_dense_projection_det_gemm_5b4e87b523f799e09384", "swap_ab_w128_t64_s8": "cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981", "swap_ab_w64_t64_s12": "cake_dense_projection_det_gemm_02803e1d2393113c0f6c"}, + "sm_103a": {"dense_proj_det_gemm_kk_n128_c1": "cake_dense_projection_det_gemm_cc2c62bd9132f8273125", "dense_proj_det_gemm_kk_n128_hen_c1": "cake_dense_projection_det_gemm_c195846d649ef9968762", "dense_proj_det_gemm_kk_n128_hen_pro15": "cake_dense_projection_det_gemm_20a2e5c04427e93572da", "dense_proj_det_gemm_kk_n192_hee_s8_pro15": "cake_dense_projection_det_gemm_56be8af43d5d0822da87", "dense_proj_det_gemm_kk_n192_pro7": "cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e", "dense_proj_det_gemm_kk_n256_cg_pro15": "cake_dense_projection_det_gemm_e33ac44bfff66daaee64", "dense_proj_det_gemm_kk_n256_cg_pro7": "cake_dense_projection_det_gemm_73a471b3db6b5aaf0c32", "dense_proj_det_gemm_kk_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_ec7277a904a7211d67ce", "dense_proj_det_gemm_kk_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_3d7b66f61c710f893b29", "dense_proj_det_gemm_kk_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_295aef2f2e0332de2372", "dense_proj_det_gemm_kk_n256_pro15": "cake_dense_projection_det_gemm_e51cf65dcafa642e2c51", "dense_proj_det_gemm_kk_n256_pro7": "cake_dense_projection_det_gemm_a6ff52c845eac9296b34", "dense_proj_det_gemm_kk_n256_q_pro7": "cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d", "dense_proj_det_gemm_kn_n128_f32_tma1_pro7": "cake_dense_projection_det_gemm_5a06c5e2f33772a33174", "dense_proj_det_gemm_kn_n128_f32_tma2_s4_c1": "cake_dense_projection_det_gemm_29d013dfb72e932eaee5", "dense_proj_det_gemm_kn_n256": "cake_dense_projection_det_gemm_e98b3ee7b774c63d7544", "dense_proj_det_gemm_kn_n256_cg_pro15": "cake_dense_projection_det_gemm_3a87cc189b4a7c1a8d7d", "dense_proj_det_gemm_kn_n256_f32_tma1_pro7": "cake_dense_projection_det_gemm_1350861ef426bd310e0f", "dense_proj_det_gemm_kn_n256_f32_tma2_pd2_pro15": "cake_dense_projection_det_gemm_624eb3d7232071ddaf91", "dense_proj_det_gemm_kn_n256_f32_v8_ef_pro7": "cake_dense_projection_det_gemm_895f9ba7159bea07aedd", "dense_proj_det_gemm_kn_n256_f32_v8_pro7": "cake_dense_projection_det_gemm_7d403fa7b167188ccc24", "dense_proj_det_gemm_kn_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_eec6f87f8b7f25eb6aa2", "dense_proj_det_gemm_kn_n256_ht_tma2_lw_pro15": "cake_dense_projection_det_gemm_7ef28d420de555e7a28d", "dense_proj_det_gemm_kn_n256_m256_f32_v8_ef_ov_ht_pro7": "cake_dense_projection_det_gemm_765ece06955b70a95b29", "dense_proj_det_gemm_kn_n256_m256_ht_or_pro7": "cake_dense_projection_det_gemm_d2ea37e4d394bf40922a", "dense_proj_det_gemm_kn_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_9ffc37c198f2d2f42c1f", "dense_proj_det_gemm_kn_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_9cc6363566760a38b4fe", "dense_proj_det_gemm_kn_n256_m256_tma1_bg4_pro7": "cake_dense_projection_det_gemm_5226b6c854d7db395d7e", "dense_proj_det_gemm_kn_n256_pro15": "cake_dense_projection_det_gemm_1c1874c560375d29237d", "dense_proj_det_gemm_kn_n256_pro7": "cake_dense_projection_det_gemm_c15429b8d7d808a760ea", "dense_proj_det_gemm_kn_n256_tma1_pd2_lw_mg_pro15": "cake_dense_projection_det_gemm_bb0e7457db664d6f0f6a", "dense_proj_det_gemm_kn_n256_tma1_pd2_lw_pro15": "cake_dense_projection_det_gemm_0d487a8f71dac54321ef", "dense_proj_det_gemm_kn_n256_tma1_pd2_mg_pro15": "cake_dense_projection_det_gemm_4eb96fb2eb1aaa14c7f7", "dense_proj_det_gemm_kn_n256_tma1_pd2_mg_sw25_pro15": "cake_dense_projection_det_gemm_75b1dcdaef42319f5ed4", "dense_proj_det_gemm_kn_n256_tma1_pd2_pro15": "cake_dense_projection_det_gemm_0618ae0aec1af120afd0", "dense_proj_det_gemm_kn_n256_tma1_pd2_sw25_pro15": "cake_dense_projection_det_gemm_552070c21b97faa67a71", "dense_proj_det_gemm_kn_n256_tma1_pro7": "cake_dense_projection_det_gemm_8053ebc84a22444c52af", "dense_proj_det_gemm_kn_n256_tma1_w8r": "cake_dense_projection_det_gemm_a24a3d3fd3ceac3e1cdf", "dense_proj_det_gemm_kn_n256_tma1_w8rs": "cake_dense_projection_det_gemm_bab00fb2b95c93758095", "dense_proj_det_gemm_kn_n256_tma2_pd2_lw_pro15": "cake_dense_projection_det_gemm_4de304a13db1f2a0e850", "dense_proj_det_gemm_kn_n256_tma2_pd2_pro15": "cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c", "dense_proj_det_gemm_kn_n256_tma2_pd2_pro7": "cake_dense_projection_det_gemm_39200edabebcd9eb2edd", "dense_proj_det_gemm_kn_n256_tma2_pd3_c1": "cake_dense_projection_det_gemm_eb2bb538645fe303bcca", "dense_proj_det_gemm_nk_n256_t_pro7": "cake_dense_projection_det_gemm_d2a5e70bd49083888179", "dense_proj_det_gemm_nn_n128_hen_f32_t_pro15": "cake_dense_projection_det_gemm_59fb66ccddd67565e592", "dense_proj_det_gemm_nn_n128_hen_f32_t_sks_pro7": "cake_dense_projection_det_gemm_f993d8de6f29f9e8a1ac", "dense_proj_det_gemm_nn_n128_hen_t_pro15": "cake_dense_projection_det_gemm_51c81f11e3b490b5aebb", "dense_proj_det_gemm_nn_n128_hen_t_sks_pro7": "cake_dense_projection_det_gemm_f589a6aff5e88318c36c", "dense_proj_det_gemm_nn_n128_m64_f32_t_pro7": "cake_dense_projection_det_gemm_f258d45b7e98001e0c09", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_b2a9665ee12fd6afea2b", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_pro7": "cake_dense_projection_det_gemm_3507ceddbf4e88a13231", "dense_proj_det_gemm_nn_n128_m64_hen_t_pro15": "cake_dense_projection_det_gemm_e2555d533680e08880ae", "dense_proj_det_gemm_nn_n128_m64_hen_t_pro7": "cake_dense_projection_det_gemm_f861bf1312b9c7fe78c1", "dense_proj_det_gemm_nn_n128_m64_t_pro7": "cake_dense_projection_det_gemm_0c72e864a4afb5347da6", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_cg_pro15": "cake_dense_projection_det_gemm_29de51d5e9cdd6d22491", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_cg_pro15": "cake_dense_projection_det_gemm_0dd5420b570cabc144ab", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_0bbd5d9f1d72a90ef3ea", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_pro15": "cake_dense_projection_det_gemm_23305bdb787b7c6d7378", "dense_proj_det_gemm_nn_n192_bz64_hne_f32_v8_ef_sks_sb2_pro15": "cake_dense_projection_det_gemm_f6c9b40184336f49390d", "dense_proj_det_gemm_nn_n192_bz64_hne_sks_sb2_pro15": "cake_dense_projection_det_gemm_6579637169bec3c2f1d5", "dense_proj_det_gemm_nn_n192_hee_f32_t_tma1_pro7": "cake_dense_projection_det_gemm_36e10fde6497f82171d4", "dense_proj_det_gemm_nn_n192_hee_t_tma1_pro7": "cake_dense_projection_det_gemm_df649506b6ba6f6e47ef", "dense_proj_det_gemm_nn_n192_m256_t_tma1_pro7": "cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052", "dense_proj_det_gemm_nn_n224_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_13790f025541d7d162a7", "dense_proj_det_gemm_nn_n224_hne_cg_pro15": "cake_dense_projection_det_gemm_aa4f7c2fbd0b9a99e535", "dense_proj_det_gemm_nn_n224_m256_pro7": "cake_dense_projection_det_gemm_c63d1400e64208d7865c", "dense_proj_det_gemm_nn_n256_f32_tma1_cg_pro15": "cake_dense_projection_det_gemm_dfd65a75564d04d3826f", "dense_proj_det_gemm_nn_n256_f32_v8_ef_sks_sb2_pro7": "cake_dense_projection_det_gemm_9475731f174aaefb2e44", "dense_proj_det_gemm_nn_n256_f32_v8_pro7": "cake_dense_projection_det_gemm_b43a23d219b0196769dc", "dense_proj_det_gemm_nn_n256_m256_f32_ht_tma1_pro7": "cake_dense_projection_det_gemm_6235599f21b10d4ab9f2", "dense_proj_det_gemm_nn_n256_m256_f32_tma1_pro7": "cake_dense_projection_det_gemm_6e17162acb2711a87d39", "dense_proj_det_gemm_nn_n256_m256_f32_tma1_sks_sb3_pro7": "cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c", "dense_proj_det_gemm_nn_n256_m256_ht_pro7": "cake_dense_projection_det_gemm_f052eb094121b03461ab", "dense_proj_det_gemm_nn_n256_m256_ht_tma1_pro7": "cake_dense_projection_det_gemm_806927d101fe72fc1317", "dense_proj_det_gemm_nn_n256_m256_ov_ht_pro7": "cake_dense_projection_det_gemm_5568657a526e2040f0bc", "dense_proj_det_gemm_nn_n256_m256_pro7": "cake_dense_projection_det_gemm_367fc93d3342f82a1c9f", "dense_proj_det_gemm_nn_n256_m256_sks_sb2_pro7": "cake_dense_projection_det_gemm_736ad30e83d88796883f", "dense_proj_det_gemm_nn_n256_m256_skx_skf_pro7": "cake_dense_projection_det_gemm_b773cc0e3894fd0ef2e0", "dense_proj_det_gemm_nn_n256_m256_t_tma1_pro7": "cake_dense_projection_det_gemm_ca9a846738b1443f7318", "dense_proj_det_gemm_nn_n256_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_fb766b1be68412e6fda7", "dense_proj_det_gemm_nn_n256_m64_hen_t_pro15": "cake_dense_projection_det_gemm_bab9e9dfd7c6ca5e17c0", "dense_proj_det_gemm_nn_n256_sks_sb2_pro7": "cake_dense_projection_det_gemm_c363601373e2dfb8e9af", "router_fp32_det_gemm_kk": "cake_dense_projection_det_gemm_ebbb505a3077e8da0a50", "router_fp32_det_gemm_kn": "cake_dense_projection_det_gemm_7a0ce55fd7221e354322", "router_fp32_det_gemm_nn_t": "cake_dense_projection_det_gemm_9269f6841d7aeb58471c", "split_fp32_det_bf16x3": "cake_dense_projection_det_gemm_5b4e87b523f799e09384", "swap_ab_w128_t64_s8": "cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981", "swap_ab_w64_t64_s12": "cake_dense_projection_det_gemm_02803e1d2393113c0f6c"}, + "sm_107a": {"dense_proj_det_gemm_kk_n128_hen_pro15": "cake_dense_projection_det_gemm_0a71a49da71c7d34ab37", "dense_proj_det_gemm_kk_n128_hen_pro7": "cake_dense_projection_det_gemm_7ad01af3a13775199254", "dense_proj_det_gemm_kk_n128_pro7": "cake_dense_projection_det_gemm_451a0f92e10205e345eb", "dense_proj_det_gemm_kk_n192_m256_hen_s5_pro15": "cake_dense_projection_det_gemm_ed1f3ab86ddc55345e69", "dense_proj_det_gemm_kk_n192_pro7": "cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e", "dense_proj_det_gemm_kk_n256_cg_pro15": "cake_dense_projection_det_gemm_438841f42a0743b63fb5", "dense_proj_det_gemm_kk_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_ec7277a904a7211d67ce", "dense_proj_det_gemm_kk_n256_ht_pro15": "cake_dense_projection_det_gemm_087ba4f157914949c26a", "dense_proj_det_gemm_kk_n256_m256_cg_or_pro7": "cake_dense_projection_det_gemm_4af009e00a1d5c0ad794", "dense_proj_det_gemm_kk_n256_m256_ht_cg_or_pro7": "cake_dense_projection_det_gemm_53a63652be2155219d76", "dense_proj_det_gemm_kk_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_3d7b66f61c710f893b29", "dense_proj_det_gemm_kk_n256_pro15": "cake_dense_projection_det_gemm_e51cf65dcafa642e2c51", "dense_proj_det_gemm_kk_n256_q_pro7": "cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d", "dense_proj_det_gemm_kn_n128_f32_tma1_c1": "cake_dense_projection_det_gemm_011c85b820ec0cf6863d", "dense_proj_det_gemm_kn_n128_f32_tma1_pd2_c1": "cake_dense_projection_det_gemm_55c1b40f71e3fe4a892c", "dense_proj_det_gemm_kn_n256_cg_pro15": "cake_dense_projection_det_gemm_3a87cc189b4a7c1a8d7d", "dense_proj_det_gemm_kn_n256_cg_pro7": "cake_dense_projection_det_gemm_14192244441c1003db74", "dense_proj_det_gemm_kn_n256_f32_tma1_pro15": "cake_dense_projection_det_gemm_b58b50619ef5ed46e109", "dense_proj_det_gemm_kn_n256_f32_tma2_pd2_pro15": "cake_dense_projection_det_gemm_624eb3d7232071ddaf91", "dense_proj_det_gemm_kn_n256_f32_v8_ef_s9_pro15": "cake_dense_projection_det_gemm_31be7faa868eabcedada", "dense_proj_det_gemm_kn_n256_f32_v8_ht_pro7": "cake_dense_projection_det_gemm_b9c3f4eb125b638d7c87", "dense_proj_det_gemm_kn_n256_f32_v8_pro7": "cake_dense_projection_det_gemm_7d403fa7b167188ccc24", "dense_proj_det_gemm_kn_n256_hne_pro7": "cake_dense_projection_det_gemm_66df6446dd2cca3106d3", "dense_proj_det_gemm_kn_n256_ht_cg_pro15": "cake_dense_projection_det_gemm_eec6f87f8b7f25eb6aa2", "dense_proj_det_gemm_kn_n256_ht_pro15": "cake_dense_projection_det_gemm_b58324007fd26df8cf2c", "dense_proj_det_gemm_kn_n256_ht_tma2_lw_pro15": "cake_dense_projection_det_gemm_7ef28d420de555e7a28d", "dense_proj_det_gemm_kn_n256_m256_cg_or_pro7": "cake_dense_projection_det_gemm_13d9310f37868a18eace", "dense_proj_det_gemm_kn_n256_m256_f32_tma1_pro7": "cake_dense_projection_det_gemm_de7492eeb6e24b31687a", "dense_proj_det_gemm_kn_n256_m256_ht_or_pro7": "cake_dense_projection_det_gemm_d2ea37e4d394bf40922a", "dense_proj_det_gemm_kn_n256_m256_ov_ht_cg_pro7": "cake_dense_projection_det_gemm_9ffc37c198f2d2f42c1f", "dense_proj_det_gemm_kn_n256_m256_ov_ht_pro15": "cake_dense_projection_det_gemm_ed6ef54b238c5da2f5a7", "dense_proj_det_gemm_kn_n256_pro15": "cake_dense_projection_det_gemm_1c1874c560375d29237d", "dense_proj_det_gemm_kn_n256_pro7": "cake_dense_projection_det_gemm_c15429b8d7d808a760ea", "dense_proj_det_gemm_kn_n256_q_pd1_pro7": "cake_dense_projection_det_gemm_fb4bcdf13915dfb96c59", "dense_proj_det_gemm_kn_n256_tma1_bg8_pro7": "cake_dense_projection_det_gemm_cb45dbd62c7edd4410f1", "dense_proj_det_gemm_kn_n256_tma1_bg8_sol_pro7": "cake_dense_projection_det_gemm_332b601bf82c22b05ef2", "dense_proj_det_gemm_kn_n256_tma1_pd2_lw_pro15": "cake_dense_projection_det_gemm_0d487a8f71dac54321ef", "dense_proj_det_gemm_kn_n256_tma2_pd2_lw_pro15": "cake_dense_projection_det_gemm_4de304a13db1f2a0e850", "dense_proj_det_gemm_kn_n256_tma2_pd2_pro15": "cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c", "dense_proj_det_gemm_kn_n256_tma2_pd3_c1": "cake_dense_projection_det_gemm_49acb91191bbcced9051", "dense_proj_det_gemm_nk_n256_t_pro7": "cake_dense_projection_det_gemm_d2a5e70bd49083888179", "dense_proj_det_gemm_nn_n128_f32_t_pro7": "cake_dense_projection_det_gemm_563dd919de0e74c71bde", "dense_proj_det_gemm_nn_n128_hen_f32_t_pro15": "cake_dense_projection_det_gemm_59fb66ccddd67565e592", "dense_proj_det_gemm_nn_n128_hen_f32_t_sks_pro7": "cake_dense_projection_det_gemm_f993d8de6f29f9e8a1ac", "dense_proj_det_gemm_nn_n128_hen_f32_t_skx_pro7": "cake_dense_projection_det_gemm_b09222ffcfa7a7e4ead7", "dense_proj_det_gemm_nn_n128_hen_t_pro15": "cake_dense_projection_det_gemm_51c81f11e3b490b5aebb", "dense_proj_det_gemm_nn_n128_hen_t_sks_pro7": "cake_dense_projection_det_gemm_f589a6aff5e88318c36c", "dense_proj_det_gemm_nn_n128_hen_t_skx_pro7": "cake_dense_projection_det_gemm_6f17e515b2e1b7e2fde9", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_b2a9665ee12fd6afea2b", "dense_proj_det_gemm_nn_n128_m64_hen_f32_t_skx_pro15": "cake_dense_projection_det_gemm_56c1924c362457a1c3fe", "dense_proj_det_gemm_nn_n128_m64_hen_t_pro15": "cake_dense_projection_det_gemm_e2555d533680e08880ae", "dense_proj_det_gemm_nn_n128_m64_hen_t_skx_pro15": "cake_dense_projection_det_gemm_4f31f3828842f20176d6", "dense_proj_det_gemm_nn_n128_t_pro7": "cake_dense_projection_det_gemm_36d0401698df21c9425f", "dense_proj_det_gemm_nn_n160_m256_bz64_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_5e77ef909e88a99c4357", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_cg_pro15": "cake_dense_projection_det_gemm_29de51d5e9cdd6d22491", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_cg_pro15": "cake_dense_projection_det_gemm_0dd5420b570cabc144ab", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_0bbd5d9f1d72a90ef3ea", "dense_proj_det_gemm_nn_n160_m256_bz64_hne_pro15": "cake_dense_projection_det_gemm_23305bdb787b7c6d7378", "dense_proj_det_gemm_nn_n160_m256_bz64_pro15": "cake_dense_projection_det_gemm_505fbe05ce8b62f0a589", "dense_proj_det_gemm_nn_n192_bz64_hee_f32_t_pro7": "cake_dense_projection_det_gemm_d6ebaab2415ea7b00de9", "dense_proj_det_gemm_nn_n192_bz64_hee_f32_t_sks_sb2_pro7": "cake_dense_projection_det_gemm_fdb106e6e73897521cbb", "dense_proj_det_gemm_nn_n192_bz64_hee_t_sks_stt_pro15": "cake_dense_projection_det_gemm_ddff2b911137449c1dfc", "dense_proj_det_gemm_nn_n192_bz64_hne_f32_v8_ef_sks_sb2_pro15": "cake_dense_projection_det_gemm_f6c9b40184336f49390d", "dense_proj_det_gemm_nn_n192_bz64_hne_sks_sb2_pro15": "cake_dense_projection_det_gemm_6579637169bec3c2f1d5", "dense_proj_det_gemm_nn_n192_m256_bz64_hee_t_tma1_s6_pro7": "cake_dense_projection_det_gemm_9aa9aa60b0c544bfab06", "dense_proj_det_gemm_nn_n224_f32_v8_ef_pro15": "cake_dense_projection_det_gemm_13790f025541d7d162a7", "dense_proj_det_gemm_nn_n224_hne_cg_pro15": "cake_dense_projection_det_gemm_aa4f7c2fbd0b9a99e535", "dense_proj_det_gemm_nn_n224_m256_pro7": "cake_dense_projection_det_gemm_c63d1400e64208d7865c", "dense_proj_det_gemm_nn_n256_f32_tma1_cg_pro15": "cake_dense_projection_det_gemm_dfd65a75564d04d3826f", "dense_proj_det_gemm_nn_n256_f32_v8_ef_sks_sb2_pro7": "cake_dense_projection_det_gemm_9475731f174aaefb2e44", "dense_proj_det_gemm_nn_n256_f32_v8_pro15": "cake_dense_projection_det_gemm_8e1f6805342f620cca8c", "dense_proj_det_gemm_nn_n256_m256_f32_ht_tma1_pro7": "cake_dense_projection_det_gemm_6235599f21b10d4ab9f2", "dense_proj_det_gemm_nn_n256_m256_f32_tma1_pro7": "cake_dense_projection_det_gemm_6e17162acb2711a87d39", "dense_proj_det_gemm_nn_n256_m256_f32_v8_ef_ht_pro7": "cake_dense_projection_det_gemm_5445e0f16ad84868fa01", "dense_proj_det_gemm_nn_n256_m256_hee_t_tma1_s6_pro7": "cake_dense_projection_det_gemm_446497e750658dfb057b", "dense_proj_det_gemm_nn_n256_m256_ht_pro7": "cake_dense_projection_det_gemm_f052eb094121b03461ab", "dense_proj_det_gemm_nn_n256_m256_pro7": "cake_dense_projection_det_gemm_367fc93d3342f82a1c9f", "dense_proj_det_gemm_nn_n256_m256_skx_skf_pro7": "cake_dense_projection_det_gemm_b773cc0e3894fd0ef2e0", "dense_proj_det_gemm_nn_n256_m256_tma1_pro7": "cake_dense_projection_det_gemm_e2692c71bc236502ba68", "dense_proj_det_gemm_nn_n256_m64_hen_f32_t_pro15": "cake_dense_projection_det_gemm_fb766b1be68412e6fda7", "dense_proj_det_gemm_nn_n256_m64_hen_t_pro15": "cake_dense_projection_det_gemm_bab9e9dfd7c6ca5e17c0", "dense_proj_det_gemm_nn_n256_pro15": "cake_dense_projection_det_gemm_82f698f67a1cca6e4291", "dense_proj_det_gemm_nn_n256_sks_sb2_pro7": "cake_dense_projection_det_gemm_c363601373e2dfb8e9af", "router_fp32_det_gemm_kk": "cake_dense_projection_det_gemm_ebbb505a3077e8da0a50", "router_fp32_det_gemm_kn": "cake_dense_projection_det_gemm_7a0ce55fd7221e354322", "router_fp32_det_gemm_nn_t": "cake_dense_projection_det_gemm_9269f6841d7aeb58471c", "split_fp32_det_bf16x3": "cake_dense_projection_det_gemm_5b4e87b523f799e09384", "swap_ab_w128_t64_s8": "cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981", "swap_ab_w64_t64_s12": "cake_dense_projection_det_gemm_02803e1d2393113c0f6c"}, } # fmt: on diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_binding.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_binding.cu new file mode 100644 index 00000000000..313730077e5 --- /dev/null +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_binding.cu @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Generated by Cake: the tvm-ffi host binding of kernel_cake_dense_projection_det_gemm_02803e1d2393113c0f6c, for the device source that defines it. Do not edit. +#include +#include + +#include "cake_ffi_checks.h" + +#include + +extern "C" __global__ void kernel_cake_dense_projection_det_gemm_02803e1d2393113c0f6c(const __grid_constant__ CUtensorMap W, const __grid_constant__ CUtensorMap A, uint8_t* __restrict__ out, int M, int N, int K, int B, int ldo, int out_l); + +namespace cake_binding_kernel_cake_dense_projection_det_gemm_02803e1d2393113c0f6c { + +using tvm::ffi::TensorView; + +constexpr const char* KERNEL_NAME = "kernel_cake_dense_projection_det_gemm_02803e1d2393113c0f6c"; + +void CheckCuda(cudaError_t status, const char* what) { + TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) + << KERNEL_NAME << ": " << what << " failed: " << cudaGetErrorString(status); +} + +void CheckCu(CUresult result, const char* what) { + if (result != CUDA_SUCCESS) { + const char* name = nullptr; + const char* message = nullptr; + cuGetErrorName(result, &name); + cuGetErrorString(result, &message); + TVM_FFI_THROW(RuntimeError) << KERNEL_NAME << ": " << what << " failed with " + << (name ? name : "an unknown CUresult") << " (" << static_cast(result) + << "): " << (message ? message : ""); + } +} + +void* DataPtr(const TensorView& t) { return static_cast(t.data_ptr()) + t.byte_offset(); } + +void CheckUnitInnerStride(const TensorView& t, const char* name) { + TVM_FFI_CHECK(t.stride(-1) == 1, ValueError) + << KERNEL_NAME << ": '" << name << "' needs a unit innermost stride, got " << t.stride(-1); +} + +// The product of the extents above the trailing `trailing` dims. +int64_t NumelAbove(const TensorView& t, int trailing) { + int64_t product = 1; + for (int axis = 0; axis < t.ndim() - trailing; ++axis) product *= t.size(axis); + return product; +} + +[[noreturn]] void NoInt64(int64_t a, const char* op, int64_t b) { + TVM_FFI_THROW(ValueError) << KERNEL_NAME << ": " << a << " " << op << " " << b + << " has no int64 value in a host check"; +} + +int64_t CDiv(int64_t a, int64_t b) { + if (b == 0 || (a == INT64_MIN && b == -1)) NoInt64(a, "/", b); + return a / b; +} + +int64_t CMod(int64_t a, int64_t b) { + if (b == 0) NoInt64(a, "%", b); + return b == -1 ? 0 : a % b; +} + +cuuint64_t Extent(int64_t value, const char* descriptor) { + TVM_FFI_CHECK(value > 0, ValueError) + << KERNEL_NAME << ": an extent of descriptor '" << descriptor << "' resolves to " << value; + return static_cast(value); +} + +cuuint64_t Stride(int64_t value, const char* descriptor) { + TVM_FFI_CHECK(value >= 0, ValueError) + << KERNEL_NAME << ": a stride of descriptor '" << descriptor << "' resolves to " << value; + return static_cast(value); +} + +CUtensorMap EncodeTma_W(const TensorView& t) { + TVM_FFI_CHECK(t.ndim() >= 3, ValueError) + << KERNEL_NAME << ": descriptor 'W' takes a source of at least 3 dims, got " << t.ndim(); + CheckUnitInnerStride(t, "W"); + int64_t d1 = t.size(-1); + int64_t d2 = t.size(-2); + int64_t outer2 = NumelAbove(t, 2); + check_dense_leading_dims(t, 2, "W"); + int64_t s2 = t.stride(-2) * 2; + int64_t s3 = t.stride(-3) * 2; + TVM_FFI_CHECK((CMod(d1, 64LL) == 0LL), ValueError) + << KERNEL_NAME << ": descriptor 'W' requires (CMod(d1, 64) == 0)"; + cuuint64_t global_dim[4] = {Extent(64LL, "W"), Extent(d2, "W"), Extent(CDiv(d1, 64LL), "W"), Extent(outer2, "W")}; + cuuint64_t global_strides[3] = {Stride(s2, "W"), Stride(128LL, "W"), Stride(s3, "W")}; + cuuint32_t box_dim[4] = {64u, 64u, 1u, 1u}; + cuuint32_t elem_strides[4] = {1u, 1u, 1u, 1u}; + const bool allow_oob_box[4] = {false, true, false, false}; + for (int axis = 0; axis < 4; ++axis) { + TVM_FFI_CHECK(allow_oob_box[axis] || box_dim[axis] <= global_dim[axis], ValueError) + << KERNEL_NAME << ": descriptor 'W' axis " << axis << " (0 is the innermost) has a TMA box of " + << box_dim[axis] << " elements but the source extent along it is " << global_dim[axis] + << ". A box may not exceed its extent unless the axis is listed in the descriptor's allow_oob_box (CK.describe_tma(..., allow_oob_box=(axis,))). List it when the box is a tile that may run past the end of the axis, a tail tile: a load fills the part past the extent with zeros (its bytes still count toward the barrier's transaction) and a store or reduce skips it, so that is safe when the kernel ignores or masks those rows and columns. If the box was meant to fit, box_dim or the extent is wrong: check box_dim, global_dim and the tensor's shape"; + } + TVM_FFI_CHECK(global_strides[0] % 16 == 0 && global_strides[1] % 16 == 0 && global_strides[2] % 16 == 0, ValueError) + << KERNEL_NAME << ": the strides of descriptor 'W' must be multiples of 16 bytes"; + void* address = DataPtr(t); + TVM_FFI_CHECK(reinterpret_cast(address) % 16 == 0, ValueError) + << KERNEL_NAME << ": the address of descriptor 'W' must be 16-byte aligned"; + CUtensorMap map{}; + CheckCu(cuTensorMapEncodeTiled(&map, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 4, address, global_dim, global_strides, + box_dim, elem_strides, CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_L2_PROMOTION_NONE, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE), + "cuTensorMapEncodeTiled of W"); + return map; +} + +CUtensorMap EncodeTma_A(const TensorView& t) { + TVM_FFI_CHECK(t.ndim() >= 3, ValueError) + << KERNEL_NAME << ": descriptor 'A' takes a source of at least 3 dims, got " << t.ndim(); + CheckUnitInnerStride(t, "A"); + int64_t d1 = t.size(-1); + int64_t d2 = t.size(-2); + int64_t outer2 = NumelAbove(t, 2); + check_dense_leading_dims(t, 2, "A"); + int64_t s2 = t.stride(-2) * 2; + int64_t s3 = t.stride(-3) * 2; + TVM_FFI_CHECK((CMod(d1, 64LL) == 0LL), ValueError) + << KERNEL_NAME << ": descriptor 'A' requires (CMod(d1, 64) == 0)"; + cuuint64_t global_dim[4] = {Extent(64LL, "A"), Extent(d2, "A"), Extent(CDiv(d1, 64LL), "A"), Extent(outer2, "A")}; + cuuint64_t global_strides[3] = {Stride(s2, "A"), Stride(128LL, "A"), Stride(s3, "A")}; + cuuint32_t box_dim[4] = {64u, 64u, 1u, 1u}; + cuuint32_t elem_strides[4] = {1u, 1u, 1u, 1u}; + const bool allow_oob_box[4] = {false, true, false, false}; + for (int axis = 0; axis < 4; ++axis) { + TVM_FFI_CHECK(allow_oob_box[axis] || box_dim[axis] <= global_dim[axis], ValueError) + << KERNEL_NAME << ": descriptor 'A' axis " << axis << " (0 is the innermost) has a TMA box of " + << box_dim[axis] << " elements but the source extent along it is " << global_dim[axis] + << ". A box may not exceed its extent unless the axis is listed in the descriptor's allow_oob_box (CK.describe_tma(..., allow_oob_box=(axis,))). List it when the box is a tile that may run past the end of the axis, a tail tile: a load fills the part past the extent with zeros (its bytes still count toward the barrier's transaction) and a store or reduce skips it, so that is safe when the kernel ignores or masks those rows and columns. If the box was meant to fit, box_dim or the extent is wrong: check box_dim, global_dim and the tensor's shape"; + } + TVM_FFI_CHECK(global_strides[0] % 16 == 0 && global_strides[1] % 16 == 0 && global_strides[2] % 16 == 0, ValueError) + << KERNEL_NAME << ": the strides of descriptor 'A' must be multiples of 16 bytes"; + void* address = DataPtr(t); + TVM_FFI_CHECK(reinterpret_cast(address) % 16 == 0, ValueError) + << KERNEL_NAME << ": the address of descriptor 'A' must be 16-byte aligned"; + CUtensorMap map{}; + CheckCu(cuTensorMapEncodeTiled(&map, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 4, address, global_dim, global_strides, + box_dim, elem_strides, CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_L2_PROMOTION_NONE, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE), + "cuTensorMapEncodeTiled of A"); + return map; +} + +void Run(TensorView a_W, TensorView a_A, TensorView a_out, int64_t a_M, int64_t a_N, int64_t a_K, int64_t a_B, int64_t a_ldo, int64_t a_out_l, int64_t a_enable_pdl, int64_t grid_x, int64_t grid_y, int64_t grid_z) { + DLDevice dev = a_W.device(); + tvm::ffi::CUDADeviceGuard device_guard(dev.device_id); + check_cuda_tensor(a_W, "W"); + check_dtype(a_W, DLDataType{kDLBfloat, 16, 1}, "W"); + check_cuda_tensor(a_A, "A"); + check_dtype(a_A, DLDataType{kDLBfloat, 16, 1}, "A"); + check_cuda_tensor(a_out, "out"); + check_dtype(a_out, DLDataType{kDLUInt, 8, 1}, "out"); + TVM_FFI_CHECK(a_M >= -2147483648LL && a_M <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'M' = " << a_M + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_N >= -2147483648LL && a_N <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'N' = " << a_N + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_K >= -2147483648LL && a_K <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'K' = " << a_K + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_B >= -2147483648LL && a_B <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'B' = " << a_B + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_ldo >= -2147483648LL && a_ldo <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'ldo' = " << a_ldo + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_out_l >= -2147483648LL && a_out_l <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'out_l' = " << a_out_l + << " is outside the i32 range [-2147483648, 2147483647]"; + check_same_device(a_A, a_W, "A", "W"); + check_contiguous(a_out, "out"); + check_same_device(a_out, a_W, "out", "W"); + TVM_FFI_CHECK((CMod(a_K, 64LL) == 0LL), ValueError) + << KERNEL_NAME << ": requires (CMod(K, 64) == 0)"; + TVM_FFI_CHECK((CMod(a_N, 8LL) == 0LL), ValueError) + << KERNEL_NAME << ": requires (CMod(N, 8) == 0)"; + TVM_FFI_CHECK((CMod(a_ldo, 8LL) == 0LL), ValueError) + << KERNEL_NAME << ": requires (CMod(ldo, 8) == 0)"; + TVM_FFI_CHECK((CMod(a_out_l, 8LL) == 0LL), ValueError) + << KERNEL_NAME << ": requires (CMod(out_l, 8) == 0)"; + TVM_FFI_CHECK((a_ldo >= a_N), ValueError) + << KERNEL_NAME << ": requires (ldo >= N)"; + TVM_FFI_CHECK(a_enable_pdl == 0LL || a_enable_pdl == 1LL, ValueError) + << KERNEL_NAME << ": 'enable_pdl' = " << a_enable_pdl << " is not 0 or 1"; + TVM_FFI_CHECK(grid_x > 0 && grid_y > 0 && grid_z > 0, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") must be positive"; + TVM_FFI_CHECK(grid_x <= 2147483647 && grid_y <= 65535 && grid_z <= 65535, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") exceeds CUDA's limits"; + cudaStream_t stream = static_cast(TVMFFIEnvGetStream(dev.device_type, dev.device_id)); + { + // Once per thread per device: the kernel's function attributes hold per device; the tensor-map encoders need this thread's context of the device bound. + static thread_local uint64_t prepared_devices = 0; + const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; + if (device_bit == 0 || (prepared_devices & device_bit) == 0) { + CheckCuda(cudaSetDevice(dev.device_id), "cudaSetDevice"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_dense_projection_det_gemm_02803e1d2393113c0f6c), cudaFuncAttributeMaxDynamicSharedMemorySize, 205824), "cudaFuncSetAttribute"); + prepared_devices |= device_bit; + } + } + CUtensorMap v_W = EncodeTma_W(a_W); + CUtensorMap v_A = EncodeTma_A(a_A); + void* v_out = DataPtr(a_out); + int32_t v_M = static_cast(a_M); + int32_t v_N = static_cast(a_N); + int32_t v_K = static_cast(a_K); + int32_t v_B = static_cast(a_B); + int32_t v_ldo = static_cast(a_ldo); + int32_t v_out_l = static_cast(a_out_l); + void* kernel_args[] = {&v_W, &v_A, &v_out, &v_M, &v_N, &v_K, &v_B, &v_ldo, &v_out_l}; + cudaLaunchConfig_t config{}; + config.gridDim = dim3(static_cast(grid_x), static_cast(grid_y), static_cast(grid_z)); + config.blockDim = dim3(256, 1, 1); + config.dynamicSmemBytes = 205824; + config.stream = stream; + cudaLaunchAttribute attrs[1] = {}; + int n = 0; + if (a_enable_pdl) { + attrs[n].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[n].val.programmaticStreamSerializationAllowed = 1; + ++n; + } + config.attrs = attrs; + config.numAttrs = n; + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_dense_projection_det_gemm_02803e1d2393113c0f6c), kernel_args), + "cudaLaunchKernelExC"); +} + +} // namespace cake_binding_kernel_cake_dense_projection_det_gemm_02803e1d2393113c0f6c + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_dense_projection_det_gemm_02803e1d2393113c0f6c::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_kernel.cu new file mode 100644 index 00000000000..293567319c1 --- /dev/null +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_02803e1d2393113c0f6c_kernel.cu @@ -0,0 +1,635 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +#if defined(__CUDACC_RTC__) +typedef unsigned long long uint64_t; +#else +typedef unsigned long uint64_t; +#endif +static_assert(sizeof(uint64_t) == 8, "Cake requires an LP64 CUDA host ABI"); +typedef signed int int32_t; +typedef short int int16_t; +struct __align__(64) CakeTensorMap64 { uint64_t opaque[16]; }; +static_assert(sizeof(CakeTensorMap64) == 128, "64-aligned tensor-map ABI size"); +static_assert(alignof(CakeTensorMap64) == 64, "64-aligned tensor-map ABI alignment"); + +#if defined(__CUDACC_RTC__) +typedef struct __align__(128) { uint64_t opaque[16]; } CUtensorMap; +#else +#include +#endif + +static_assert(sizeof(CUtensorMap) == 128, "CUtensorMap CUDA ABI must be 128 bytes"); +#include +#include + +#define CAKE_INF CUDART_INF_F +#define TMEM_NCOLS 64 +#define TMEM_ACCUM_OFFSET 0 +#define NUM_TMA_PIPE_STAGES 12 +#define NUM_EPI_PIPE_STAGES 1 +#define SMEM_SMEM_W_OFF 1024 +#define SMEM_SMEM_W_STAGE_BYTES 8192 +#define SMEM_SMEM_W_STRIDE 8192 +#define SMEM_SMEM_A_OFF 99328 +#define SMEM_SMEM_A_STAGE_BYTES 8192 +#define SMEM_SMEM_A_STRIDE 8192 +#define SMEM_STAGING_OFF 197632 +#define SMEM_STAGING_STAGE_BYTES 8192 +#define SMEM_STAGING_STRIDE 8192 +#define SMEM_TOTAL 205824 +#define THREADS 256 + +#include + +__device__ __forceinline__ uint32_t elect_sync() { + uint32_t pred = 0; + asm volatile( + "{\n\t" + ".reg .pred %%px;\n\t" + "elect.sync _|%%px, %1;\n\t" + "@%%px mov.s32 %0, 1;\n\t" + "}\n" + : "+r"(pred) + : "r"(0xFFFFFFFF)); + return pred; +} + + +__device__ __forceinline__ void mbarrier_init(int mbar_addr, int count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" + :: "r"(mbar_addr), "r"(count) : "memory"); +} + + + +// CTA-local pipelines have short, resident producer/consumer edges. Omitting +// suspendTimeHint keeps a miss on the lightweight TRYWAIT retry path; the +// explicit loop still makes this helper blocking until acquire succeeds. +__device__ __forceinline__ void mbarrier_wait(int mbar_addr, int phase) { + asm volatile( + "{\n\t" + ".reg .pred P1;\n\t" + "LAB_WAIT:\n\t" + "mbarrier.try_wait.parity.acquire.cta.shared::cta.b64" + " P1, [%0], %1;\n\t" + "@P1 bra.uni DONE;\n\t" + "bra.uni LAB_WAIT;\n\t" + "DONE:\n\t" + "}\n" + :: "r"(mbar_addr), "r"(phase) : "memory"); +} + + +__device__ __forceinline__ void tcgen05_mma_f16( + int taddr, uint64_t a_desc, uint64_t b_desc, + uint32_t i_desc, int enable_input_d) { + asm volatile( + "{\n\t" + ".reg .pred p;\n\t" + "setp.ne.b32 p, %4, 0;\n\t" + "tcgen05.mma.cta_group::1.kind::f16 [%0], %1, %2, %3, p;\n\t" + "}\n" + :: "r"(taddr), "l"(a_desc), "l"(b_desc), + "r"(i_desc), "r"(enable_input_d) + : "memory"); +} + + + +union MmaSmemDesc { + uint64_t u64; + uint32_t u32[2]; +}; + +__device__ __forceinline__ void incr_smem_desc_lo(uint64_t& smem_desc, uint32_t offset) { + MmaSmemDesc tmp; + tmp.u64 = smem_desc; + tmp.u32[0] += offset; + smem_desc = tmp.u64; +} + + +__device__ __forceinline__ void elect_commit(int mbar_addr) { + asm volatile( + "{\n\t" + ".reg .pred leader;\n\t" + "elect.sync _|leader, 0xFFFFFFFF;\n\t" + "@leader tcgen05.commit.cta_group::1.mbarrier::arrive::one" + ".shared::cluster.b64 [%0];\n\t" + "}\n" + :: "r"(mbar_addr)); +} + + + +__device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_t bytes) { + asm volatile( + "mbarrier.arrive.expect_tx.release.cta.shared::cta.b64 _, [%0], %1;" + :: "r"(mbar_addr), "r"(bytes) : "memory"); +} + + + + + +__device__ __forceinline__ void tma_4d_gmem2smem( + int dst, const void *tmap_ptr, int x, int y, int z, int w, int mbar_addr) { + asm volatile( + "cp.async.bulk.tensor.4d.shared::cta.global" + ".mbarrier::complete_tx::bytes" + " [%0], [%1, {%2, %3, %4, %5}], [%6];" + :: "r"(dst), "l"(tmap_ptr), "r"(x), "r"(y), "r"(z), "r"(w), + "r"(mbar_addr) : "memory"); +} + + + +__device__ __forceinline__ uint32_t make_warp_uniform(uint32_t val) { + uint32_t result; + asm volatile("shfl.sync.idx.b32 %0, %1, 0, 0x1f, 0xffffffff;" + : "=r"(result) : "r"(val)); + return result; +} + +extern "C" { + +__global__ __launch_bounds__(256) void +kernel_cake_dense_projection_det_gemm_02803e1d2393113c0f6c(const __grid_constant__ CUtensorMap W, const __grid_constant__ CUtensorMap A, uint8_t* __restrict__ out, int M, int N, int K, int B, int ldo, int out_l) +{ + const int tid = threadIdx.x; + const int warp = make_warp_uniform(tid / 32); + const int lane = tid % 32; + + extern __shared__ __align__(1024) char smem_raw[]; + int smem; +#if __CUDA_ARCH__ == 1000 + asm volatile("{ .reg .u64 smem_ptr; cvta.to.shared.u64 smem_ptr, %1; cvt.u32.u64 %0, smem_ptr; }" : "=r"(smem) : "l"(smem_raw)); + smem = make_warp_uniform(smem); +#else + smem = (int)(unsigned long long)__cvta_generic_to_shared(smem_raw); +#endif + + const int mbar_base = smem; + #define tma_full_addr (mbar_base + 0) + #define tma_empty_addr (mbar_base + 96) + #define mma_epi_full_addr (mbar_base + 192) + + const int bid = blockIdx.x; + const int num_bids = gridDim.x; + + const int cta_rank = 0; + + // Kernel setup ops + __nv_bfloat16* smem_w = reinterpret_cast<__nv_bfloat16*>(smem_raw + 1024); + const int smem_w_addr = smem + 1024; + __nv_bfloat16* smem_a = reinterpret_cast<__nv_bfloat16*>(smem_raw + 99328); + const int smem_a_addr = smem + 99328; + unsigned int* staging = reinterpret_cast(smem_raw + 197632); + const int staging_addr = smem + 197632; + asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); + if (warp == 0) { asm volatile("prefetch.tensormap [%0];" :: "l"((uint64_t)((&W))) : "memory"); } + if (warp == 1) { asm volatile("prefetch.tensormap [%0];" :: "l"((uint64_t)((&A))) : "memory"); } + + // Mbarrier init (3 pipeline groups, 0 ordered-sequence groups, 25 barriers) + // Mbarriers at smem_raw[0..200) + + if (warp == 0) { + uint32_t leader = elect_sync(); + if (leader) { + // --- pipeline 'tma_pipe' --- + // tma_full: 12 barriers, init_count=2 + mbarrier_init(smem + 0, 2); + mbarrier_init(smem + 8, 2); + mbarrier_init(smem + 16, 2); + mbarrier_init(smem + 24, 2); + mbarrier_init(smem + 32, 2); + mbarrier_init(smem + 40, 2); + mbarrier_init(smem + 48, 2); + mbarrier_init(smem + 56, 2); + mbarrier_init(smem + 64, 2); + mbarrier_init(smem + 72, 2); + mbarrier_init(smem + 80, 2); + mbarrier_init(smem + 88, 2); + // tma_empty: 12 barriers, init_count=1 + mbarrier_init(smem + 96, 1); + mbarrier_init(smem + 104, 1); + mbarrier_init(smem + 112, 1); + mbarrier_init(smem + 120, 1); + mbarrier_init(smem + 128, 1); + mbarrier_init(smem + 136, 1); + mbarrier_init(smem + 144, 1); + mbarrier_init(smem + 152, 1); + mbarrier_init(smem + 160, 1); + mbarrier_init(smem + 168, 1); + mbarrier_init(smem + 176, 1); + mbarrier_init(smem + 184, 1); + // --- pipeline 'epi_pipe' --- + // mma_epi_full: 1 barriers, init_count=1 + mbarrier_init(smem + 192, 1); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + } + } + + __syncthreads(); + + // TMEM alloc (64 columns, 64 used) + volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 200); + if (warp == 2) { + int _tmem_hold = smem + 200; + asm volatile("tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" :: "r"(_tmem_hold), "r"(64) : "memory"); + __syncwarp(); + asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;"); + } + + if (warp == 2 || warp == 4 || warp == 5 || warp == 6 || warp == 7) { + asm volatile("barrier.sync.aligned %0, %1;" :: "r"(1), "r"(160) : "memory"); + asm volatile("tcgen05.fence::after_thread_sync;"); + } + const int taddr = (warp == 2 || warp == 4 || warp == 5 || warp == 6 || warp == 7) ? tmem_addr_storage[0] : 0; + + // Kernel post-init ops + const int tmem_accum = taddr; + + // ---- Role: dma_w ---- + if (warp == 0) { + { // dma_w_main + unsigned int w_stage = 0; + int w_idx = blockIdx.x * 64; + int b_idx = blockIdx.z; + int k_tiles = K / 64; + int k_begin = 0; + unsigned int _phase_tma_empty = 1; + if (elect_sync()) { + #pragma unroll 4 + for (int k_tile = k_begin; k_tile < k_begin + k_tiles; k_tile++) { + mbarrier_wait(tma_empty_addr + (w_stage) * 8, _phase_tma_empty); + asm volatile( + "cp.async.bulk.tensor.4d.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint" + " [%0], [%1, {%2, %3, %4, %5}], [%6], %7;" + :: "r"(smem_w_addr + w_stage * 8192), "l"((&W)), "r"(0), "r"(w_idx), "r"(k_tile), "r"(b_idx), + "r"(tma_full_addr + (w_stage) * 8), "l"(0x12F0000000000000ULL) : "memory"); + mbarrier_arrive_expect_tx(tma_full_addr + (w_stage) * 8, 8192); + w_stage += 1; + if (w_stage == 12) { w_stage = 0; _phase_tma_empty ^= 1; } + } + } + } + } + // ---- Role: dma_a ---- + if (warp == 1) { + { // dma_a_main + unsigned int a_stage = 0; + int t_idx = blockIdx.y * 64; + int b_idx_a = blockIdx.z; + int k_tiles_a = K / 64; + int k_begin_a = 0; + unsigned int _phase_tma_empty_1 = 1; + if (elect_sync()) { + asm volatile("griddepcontrol.wait;" ::: "memory"); + #pragma unroll 4 + for (int k_tile_1 = k_begin_a; k_tile_1 < k_begin_a + k_tiles_a; k_tile_1++) { + mbarrier_wait(tma_empty_addr + (a_stage) * 8, _phase_tma_empty_1); + tma_4d_gmem2smem(smem_a_addr + a_stage * 8192, (&A), 0, t_idx, k_tile_1, b_idx_a, tma_full_addr + (a_stage) * 8); + mbarrier_arrive_expect_tx(tma_full_addr + (a_stage) * 8, 8192); + a_stage += 1; + if (a_stage == 12) { a_stage = 0; _phase_tma_empty_1 ^= 1; } + } + } + } + } + // ---- Role: mma ---- + if (warp == 2) { + { // mma_main + unsigned int mma_stage = 0; + int k_tiles_m = K / 64; + unsigned int _phase_tma_full = 0; + #pragma unroll 2 + for (int k_tile_2 = 0; k_tile_2 < k_tiles_m; k_tile_2++) { + mbarrier_wait(tma_full_addr + (mma_stage) * 8, _phase_tma_full); + asm volatile("tcgen05.fence::after_thread_sync;"); + int init_flag = ((k_tile_2 == 0) ? 1 : 0); +#if __CUDA_ARCH__ == 1070 + int _mma_a_lo_0 = make_warp_uniform((((smem_w_addr) >> 4) & 0x7FFF) + (mma_stage) * 512); + int _mma_b_lo_0 = make_warp_uniform((((smem_a_addr) >> 4) & 0x7FFF) + (mma_stage) * 512); +#else + int _mma_a_lo_0 = make_warp_uniform((((smem_w_addr) >> 4) & 0x3FFF) + (mma_stage) * 512); + int _mma_b_lo_0 = make_warp_uniform((((smem_a_addr) >> 4) & 0x3FFF) + (mma_stage) * 512); +#endif + { + uint64_t _mma_ss_a_desc_0 = (static_cast(0x40004040U) << 32) | static_cast(_mma_a_lo_0); + uint64_t _mma_ss_b_desc_0 = (static_cast(0x40004040U) << 32) | static_cast(_mma_b_lo_0); + if (elect_sync()) { + tcgen05_mma_f16(tmem_accum, _mma_ss_a_desc_0, _mma_ss_b_desc_0, 68158608, ((init_flag) ? 0 : 1)); + } + incr_smem_desc_lo(_mma_ss_a_desc_0, 2U); + incr_smem_desc_lo(_mma_ss_b_desc_0, 2U); + if (elect_sync()) { + tcgen05_mma_f16(tmem_accum, _mma_ss_a_desc_0, _mma_ss_b_desc_0, 68158608, 1); + } + incr_smem_desc_lo(_mma_ss_a_desc_0, 2U); + incr_smem_desc_lo(_mma_ss_b_desc_0, 2U); + if (elect_sync()) { + tcgen05_mma_f16(tmem_accum, _mma_ss_a_desc_0, _mma_ss_b_desc_0, 68158608, 1); + } + incr_smem_desc_lo(_mma_ss_a_desc_0, 2U); + incr_smem_desc_lo(_mma_ss_b_desc_0, 2U); + if (elect_sync()) { + tcgen05_mma_f16(tmem_accum, _mma_ss_a_desc_0, _mma_ss_b_desc_0, 68158608, 1); + } + } + elect_commit(tma_empty_addr + (mma_stage) * 8); + mma_stage += 1; + if (mma_stage == 12) { mma_stage = 0; _phase_tma_full ^= 1; } + } + elect_commit(mma_epi_full_addr); + } + } + // ---- Role: epilogue ---- + if (warp >= 4 && warp <= 7) { + { // epilogue_main + const int epi_warp = warp % 4; + const int epi_tid = epi_warp * 32 + lane; + int row_local = epi_warp * 16 + lane % 16; + int col_half = lane / 16; + int n_row = blockIdx.x * 64 + row_local; + int m_first = blockIdx.y * 64 + col_half * 32; + int b_idx_e = blockIdx.z; + mbarrier_wait(mma_epi_full_addr, 0); + asm volatile("tcgen05.fence::after_thread_sync;"); + unsigned int tmem_addr = taddr + (unsigned int)(epi_warp * 32 << 16); + float _tmem_load_0[32]; + asm volatile( + "tcgen05.ld.sync.aligned.16x32bx2.x32.b32" + " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32], 32;" + : "=r"(*reinterpret_cast(&_tmem_load_0[0])), "=r"(*reinterpret_cast(&_tmem_load_0[1])), "=r"(*reinterpret_cast(&_tmem_load_0[2])), "=r"(*reinterpret_cast(&_tmem_load_0[3])), "=r"(*reinterpret_cast(&_tmem_load_0[4])), "=r"(*reinterpret_cast(&_tmem_load_0[5])), "=r"(*reinterpret_cast(&_tmem_load_0[6])), "=r"(*reinterpret_cast(&_tmem_load_0[7])), "=r"(*reinterpret_cast(&_tmem_load_0[8])), "=r"(*reinterpret_cast(&_tmem_load_0[9])), "=r"(*reinterpret_cast(&_tmem_load_0[10])), "=r"(*reinterpret_cast(&_tmem_load_0[11])), "=r"(*reinterpret_cast(&_tmem_load_0[12])), "=r"(*reinterpret_cast(&_tmem_load_0[13])), "=r"(*reinterpret_cast(&_tmem_load_0[14])), "=r"(*reinterpret_cast(&_tmem_load_0[15])), "=r"(*reinterpret_cast(&_tmem_load_0[16])), "=r"(*reinterpret_cast(&_tmem_load_0[17])), "=r"(*reinterpret_cast(&_tmem_load_0[18])), "=r"(*reinterpret_cast(&_tmem_load_0[19])), "=r"(*reinterpret_cast(&_tmem_load_0[20])), "=r"(*reinterpret_cast(&_tmem_load_0[21])), "=r"(*reinterpret_cast(&_tmem_load_0[22])), "=r"(*reinterpret_cast(&_tmem_load_0[23])), "=r"(*reinterpret_cast(&_tmem_load_0[24])), "=r"(*reinterpret_cast(&_tmem_load_0[25])), "=r"(*reinterpret_cast(&_tmem_load_0[26])), "=r"(*reinterpret_cast(&_tmem_load_0[27])), "=r"(*reinterpret_cast(&_tmem_load_0[28])), "=r"(*reinterpret_cast(&_tmem_load_0[29])), "=r"(*reinterpret_cast(&_tmem_load_0[30])), "=r"(*reinterpret_cast(&_tmem_load_0[31])) + : "r"(tmem_addr)); + asm volatile("tcgen05.wait::ld.sync.aligned;"); + { + __nv_bfloat16 _bval_0 = __float2bfloat16_rn(_tmem_load_0[0]); + uint16_t _bits_0 = *(uint16_t*)&_bval_0; + uint32_t _addr_0 = static_cast(staging_addr + (unsigned int)((col_half * 32 * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_0), "h"(_bits_0)); + } + { + __nv_bfloat16 _bval_1 = __float2bfloat16_rn(_tmem_load_0[1]); + uint16_t _bits_1 = *(uint16_t*)&_bval_1; + uint32_t _addr_1 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 1) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_1), "h"(_bits_1)); + } + { + __nv_bfloat16 _bval_2 = __float2bfloat16_rn(_tmem_load_0[2]); + uint16_t _bits_2 = *(uint16_t*)&_bval_2; + uint32_t _addr_2 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 2) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_2), "h"(_bits_2)); + } + { + __nv_bfloat16 _bval_3 = __float2bfloat16_rn(_tmem_load_0[3]); + uint16_t _bits_3 = *(uint16_t*)&_bval_3; + uint32_t _addr_3 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 3) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_3), "h"(_bits_3)); + } + { + __nv_bfloat16 _bval_4 = __float2bfloat16_rn(_tmem_load_0[4]); + uint16_t _bits_4 = *(uint16_t*)&_bval_4; + uint32_t _addr_4 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 4) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_4), "h"(_bits_4)); + } + { + __nv_bfloat16 _bval_5 = __float2bfloat16_rn(_tmem_load_0[5]); + uint16_t _bits_5 = *(uint16_t*)&_bval_5; + uint32_t _addr_5 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 5) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_5), "h"(_bits_5)); + } + { + __nv_bfloat16 _bval_6 = __float2bfloat16_rn(_tmem_load_0[6]); + uint16_t _bits_6 = *(uint16_t*)&_bval_6; + uint32_t _addr_6 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 6) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_6), "h"(_bits_6)); + } + { + __nv_bfloat16 _bval_7 = __float2bfloat16_rn(_tmem_load_0[7]); + uint16_t _bits_7 = *(uint16_t*)&_bval_7; + uint32_t _addr_7 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 7) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_7), "h"(_bits_7)); + } + { + __nv_bfloat16 _bval_8 = __float2bfloat16_rn(_tmem_load_0[8]); + uint16_t _bits_8 = *(uint16_t*)&_bval_8; + uint32_t _addr_8 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 8) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_8), "h"(_bits_8)); + } + { + __nv_bfloat16 _bval_9 = __float2bfloat16_rn(_tmem_load_0[9]); + uint16_t _bits_9 = *(uint16_t*)&_bval_9; + uint32_t _addr_9 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 9) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_9), "h"(_bits_9)); + } + { + __nv_bfloat16 _bval_10 = __float2bfloat16_rn(_tmem_load_0[10]); + uint16_t _bits_10 = *(uint16_t*)&_bval_10; + uint32_t _addr_10 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 10) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_10), "h"(_bits_10)); + } + { + __nv_bfloat16 _bval_11 = __float2bfloat16_rn(_tmem_load_0[11]); + uint16_t _bits_11 = *(uint16_t*)&_bval_11; + uint32_t _addr_11 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 11) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_11), "h"(_bits_11)); + } + { + __nv_bfloat16 _bval_12 = __float2bfloat16_rn(_tmem_load_0[12]); + uint16_t _bits_12 = *(uint16_t*)&_bval_12; + uint32_t _addr_12 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 12) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_12), "h"(_bits_12)); + } + { + __nv_bfloat16 _bval_13 = __float2bfloat16_rn(_tmem_load_0[13]); + uint16_t _bits_13 = *(uint16_t*)&_bval_13; + uint32_t _addr_13 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 13) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_13), "h"(_bits_13)); + } + { + __nv_bfloat16 _bval_14 = __float2bfloat16_rn(_tmem_load_0[14]); + uint16_t _bits_14 = *(uint16_t*)&_bval_14; + uint32_t _addr_14 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 14) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_14), "h"(_bits_14)); + } + { + __nv_bfloat16 _bval_15 = __float2bfloat16_rn(_tmem_load_0[15]); + uint16_t _bits_15 = *(uint16_t*)&_bval_15; + uint32_t _addr_15 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 15) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_15), "h"(_bits_15)); + } + { + __nv_bfloat16 _bval_16 = __float2bfloat16_rn(_tmem_load_0[16]); + uint16_t _bits_16 = *(uint16_t*)&_bval_16; + uint32_t _addr_16 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 16) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_16), "h"(_bits_16)); + } + { + __nv_bfloat16 _bval_17 = __float2bfloat16_rn(_tmem_load_0[17]); + uint16_t _bits_17 = *(uint16_t*)&_bval_17; + uint32_t _addr_17 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 17) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_17), "h"(_bits_17)); + } + { + __nv_bfloat16 _bval_18 = __float2bfloat16_rn(_tmem_load_0[18]); + uint16_t _bits_18 = *(uint16_t*)&_bval_18; + uint32_t _addr_18 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 18) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_18), "h"(_bits_18)); + } + { + __nv_bfloat16 _bval_19 = __float2bfloat16_rn(_tmem_load_0[19]); + uint16_t _bits_19 = *(uint16_t*)&_bval_19; + uint32_t _addr_19 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 19) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_19), "h"(_bits_19)); + } + { + __nv_bfloat16 _bval_20 = __float2bfloat16_rn(_tmem_load_0[20]); + uint16_t _bits_20 = *(uint16_t*)&_bval_20; + uint32_t _addr_20 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 20) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_20), "h"(_bits_20)); + } + { + __nv_bfloat16 _bval_21 = __float2bfloat16_rn(_tmem_load_0[21]); + uint16_t _bits_21 = *(uint16_t*)&_bval_21; + uint32_t _addr_21 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 21) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_21), "h"(_bits_21)); + } + { + __nv_bfloat16 _bval_22 = __float2bfloat16_rn(_tmem_load_0[22]); + uint16_t _bits_22 = *(uint16_t*)&_bval_22; + uint32_t _addr_22 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 22) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_22), "h"(_bits_22)); + } + { + __nv_bfloat16 _bval_23 = __float2bfloat16_rn(_tmem_load_0[23]); + uint16_t _bits_23 = *(uint16_t*)&_bval_23; + uint32_t _addr_23 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 23) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_23), "h"(_bits_23)); + } + { + __nv_bfloat16 _bval_24 = __float2bfloat16_rn(_tmem_load_0[24]); + uint16_t _bits_24 = *(uint16_t*)&_bval_24; + uint32_t _addr_24 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 24) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_24), "h"(_bits_24)); + } + { + __nv_bfloat16 _bval_25 = __float2bfloat16_rn(_tmem_load_0[25]); + uint16_t _bits_25 = *(uint16_t*)&_bval_25; + uint32_t _addr_25 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 25) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_25), "h"(_bits_25)); + } + { + __nv_bfloat16 _bval_26 = __float2bfloat16_rn(_tmem_load_0[26]); + uint16_t _bits_26 = *(uint16_t*)&_bval_26; + uint32_t _addr_26 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 26) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_26), "h"(_bits_26)); + } + { + __nv_bfloat16 _bval_27 = __float2bfloat16_rn(_tmem_load_0[27]); + uint16_t _bits_27 = *(uint16_t*)&_bval_27; + uint32_t _addr_27 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 27) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_27), "h"(_bits_27)); + } + { + __nv_bfloat16 _bval_28 = __float2bfloat16_rn(_tmem_load_0[28]); + uint16_t _bits_28 = *(uint16_t*)&_bval_28; + uint32_t _addr_28 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 28) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_28), "h"(_bits_28)); + } + { + __nv_bfloat16 _bval_29 = __float2bfloat16_rn(_tmem_load_0[29]); + uint16_t _bits_29 = *(uint16_t*)&_bval_29; + uint32_t _addr_29 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 29) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_29), "h"(_bits_29)); + } + { + __nv_bfloat16 _bval_30 = __float2bfloat16_rn(_tmem_load_0[30]); + uint16_t _bits_30 = *(uint16_t*)&_bval_30; + uint32_t _addr_30 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 30) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_30), "h"(_bits_30)); + } + { + __nv_bfloat16 _bval_31 = __float2bfloat16_rn(_tmem_load_0[31]); + uint16_t _bits_31 = *(uint16_t*)&_bval_31; + uint32_t _addr_31 = static_cast(staging_addr + (unsigned int)(((col_half * 32 + 31) * 64 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_31), "h"(_bits_31)); + } + asm volatile("tcgen05.fence::before_thread_sync;"); + asm volatile("bar.sync 2, 128;" ::: "memory"); + int c = epi_tid; + int t_row = c / 8; + int m_row = blockIdx.y * 64 + t_row; + int n_col = blockIdx.x * 64 + c % 8 * 8; + if (m_row < M && n_col < N) { + unsigned int words[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row * (long long)ldo + (long long)n_col) * 2)))[0] = reinterpret_cast(words)[0]; + } + int c_0 = epi_tid + 128; + int t_row_0 = c_0 / 8; + int m_row_0 = blockIdx.y * 64 + t_row_0; + int n_col_0 = blockIdx.x * 64 + c_0 % 8 * 8; + if (m_row_0 < M && n_col_0 < N) { + unsigned int words_1[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words_1[0])), "=r"(*reinterpret_cast(&words_1[(0) + 1])), "=r"(*reinterpret_cast(&words_1[(0) + 2])), "=r"(*reinterpret_cast(&words_1[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c_0 * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row_0 * (long long)ldo + (long long)n_col_0) * 2)))[0] = reinterpret_cast(words_1)[0]; + } + int c_1 = epi_tid + 256; + int t_row_1 = c_1 / 8; + int m_row_1 = blockIdx.y * 64 + t_row_1; + int n_col_1 = blockIdx.x * 64 + c_1 % 8 * 8; + if (m_row_1 < M && n_col_1 < N) { + unsigned int words_2[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words_2[0])), "=r"(*reinterpret_cast(&words_2[(0) + 1])), "=r"(*reinterpret_cast(&words_2[(0) + 2])), "=r"(*reinterpret_cast(&words_2[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c_1 * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row_1 * (long long)ldo + (long long)n_col_1) * 2)))[0] = reinterpret_cast(words_2)[0]; + } + int c_2 = epi_tid + 384; + int t_row_2 = c_2 / 8; + int m_row_2 = blockIdx.y * 64 + t_row_2; + int n_col_2 = blockIdx.x * 64 + c_2 % 8 * 8; + if (m_row_2 < M && n_col_2 < N) { + unsigned int words_3[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words_3[0])), "=r"(*reinterpret_cast(&words_3[(0) + 1])), "=r"(*reinterpret_cast(&words_3[(0) + 2])), "=r"(*reinterpret_cast(&words_3[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c_2 * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row_2 * (long long)ldo + (long long)n_col_2) * 2)))[0] = reinterpret_cast(words_3)[0]; + } + } + } + + // Cleanup + asm volatile("tcgen05.fence::before_thread_sync;" ::: "memory"); + __syncthreads(); // barrier before TMEM dealloc + + if (warp == 2) { + asm volatile("tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" :: "r"(tmem_addr_storage[0]), "r"(64)); + } +} + +} // extern "C" diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_kernel.cu index 8f5c9ff0b6f..26efc1db307 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0618ae0aec1af120afd0_kernel.cu @@ -130,9 +130,9 @@ kernel_cake_dense_projection_det_gemm_0618ae0aec1af120afd0(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 200); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu index aec2627059c..a3b49d69c15 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_087ba4f157914949c26a_kernel.cu @@ -182,9 +182,9 @@ CAKE_PROGRAM_KERNEL(const __grid_constant__ CUtensorMap A, const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) #if defined(CAKE_DPG_DENSE_PROJ_DET_GEMM_KN_N256_PRO7) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 208); diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0a71a49da71c7d34ab37_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0a71a49da71c7d34ab37_kernel.cu index 76146913d00..bcc59a9f46c 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0a71a49da71c7d34ab37_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0a71a49da71c7d34ab37_kernel.cu @@ -129,9 +129,9 @@ kernel_cake_dense_projection_det_gemm_0a71a49da71c7d34ab37(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 248); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu index c6c1bea64aa..3a790d88d7f 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d_kernel.cu @@ -124,9 +124,9 @@ kernel_cake_dense_projection_det_gemm_0db65f3ddd780ed42d4d(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 208); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_13d9310f37868a18eace_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_13d9310f37868a18eace_kernel.cu index d0d472bb080..b2e72836427 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_13d9310f37868a18eace_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_13d9310f37868a18eace_kernel.cu @@ -122,9 +122,9 @@ kernel_cake_dense_projection_det_gemm_13d9310f37868a18eace(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 152); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_14192244441c1003db74_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_14192244441c1003db74_kernel.cu index 38b788a687b..ccb62afec8b 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_14192244441c1003db74_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_14192244441c1003db74_kernel.cu @@ -122,9 +122,9 @@ kernel_cake_dense_projection_det_gemm_14192244441c1003db74(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 208); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_295aef2f2e0332de2372_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_295aef2f2e0332de2372_kernel.cu index 18cd8ffa8b9..ccf2e1302f9 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_295aef2f2e0332de2372_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_295aef2f2e0332de2372_kernel.cu @@ -134,9 +134,9 @@ kernel_cake_dense_projection_det_gemm_295aef2f2e0332de2372(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 152); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu index 094a159af8e..cc5547ccfd9 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_3088b8056e0b7f9ca32c_kernel.cu @@ -163,9 +163,9 @@ CAKE_PROGRAM_KERNEL(const __grid_constant__ CUtensorMap A, const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) #if !defined(CAKE_DPG_DENSE_PROJ_DET_GEMM_KN_N256_TMA2_PD2_PRO7) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 184); diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_31be7faa868eabcedada_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_31be7faa868eabcedada_kernel.cu index 6b75837490a..5737361ea4f 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_31be7faa868eabcedada_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_31be7faa868eabcedada_kernel.cu @@ -129,9 +129,9 @@ kernel_cake_dense_projection_det_gemm_31be7faa868eabcedada(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 248); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_438841f42a0743b63fb5_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_438841f42a0743b63fb5_kernel.cu index 944c067d67b..ab756c59646 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_438841f42a0743b63fb5_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_438841f42a0743b63fb5_kernel.cu @@ -125,9 +125,9 @@ kernel_cake_dense_projection_det_gemm_438841f42a0743b63fb5(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 216); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_446497e750658dfb057b_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_446497e750658dfb057b_kernel.cu index f76c58f0c8a..6f2be0601c1 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_446497e750658dfb057b_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_446497e750658dfb057b_kernel.cu @@ -128,9 +128,9 @@ kernel_cake_dense_projection_det_gemm_446497e750658dfb057b(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 176); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_505fbe05ce8b62f0a589_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_505fbe05ce8b62f0a589_kernel.cu index 63de12867a1..8866c75cbc8 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_505fbe05ce8b62f0a589_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_505fbe05ce8b62f0a589_kernel.cu @@ -124,9 +124,9 @@ kernel_cake_dense_projection_det_gemm_505fbe05ce8b62f0a589(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 168); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_kernel.cu index 5c5837ce4a5..890c362b989 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5226b6c854d7db395d7e_kernel.cu @@ -126,9 +126,9 @@ kernel_cake_dense_projection_det_gemm_5226b6c854d7db395d7e(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 144); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5445e0f16ad84868fa01_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5445e0f16ad84868fa01_kernel.cu index f6bfb5d35f1..6188badd5c1 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5445e0f16ad84868fa01_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5445e0f16ad84868fa01_kernel.cu @@ -119,9 +119,9 @@ kernel_cake_dense_projection_det_gemm_5445e0f16ad84868fa01(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 144); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5568657a526e2040f0bc_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5568657a526e2040f0bc_kernel.cu index 586d62c0aea..47a2dc0c507 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5568657a526e2040f0bc_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5568657a526e2040f0bc_kernel.cu @@ -134,9 +134,9 @@ kernel_cake_dense_projection_det_gemm_5568657a526e2040f0bc(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 152); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_56be8af43d5d0822da87_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_56be8af43d5d0822da87_kernel.cu index dac0edcef0d..b9e5f983e73 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_56be8af43d5d0822da87_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_56be8af43d5d0822da87_kernel.cu @@ -129,9 +129,9 @@ kernel_cake_dense_projection_det_gemm_56be8af43d5d0822da87(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 232); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_kernel.cu index 48370acff98..b116acfc1de 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5a06c5e2f33772a33174_kernel.cu @@ -131,9 +131,9 @@ kernel_cake_dense_projection_det_gemm_5a06c5e2f33772a33174(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 224); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5e77ef909e88a99c4357_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5e77ef909e88a99c4357_kernel.cu index 7a92c463e87..c135b50ff8c 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5e77ef909e88a99c4357_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_5e77ef909e88a99c4357_kernel.cu @@ -124,9 +124,9 @@ kernel_cake_dense_projection_det_gemm_5e77ef909e88a99c4357(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 168); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_66df6446dd2cca3106d3_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_66df6446dd2cca3106d3_kernel.cu index 384a7a37de2..afc4d51be51 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_66df6446dd2cca3106d3_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_66df6446dd2cca3106d3_kernel.cu @@ -122,9 +122,9 @@ kernel_cake_dense_projection_det_gemm_66df6446dd2cca3106d3(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 208); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_765ece06955b70a95b29_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_765ece06955b70a95b29_kernel.cu index 48919755180..5801308c16b 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_765ece06955b70a95b29_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_765ece06955b70a95b29_kernel.cu @@ -134,9 +134,9 @@ kernel_cake_dense_projection_det_gemm_765ece06955b70a95b29(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 152); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu index ebcecd02b82..fa75e762c05 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_7d403fa7b167188ccc24_kernel.cu @@ -146,9 +146,9 @@ CAKE_PROGRAM_KERNEL(const __grid_constant__ CUtensorMap A, const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 208); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu index df446c5b4c0..4b9adee11df 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_806927d101fe72fc1317_kernel.cu @@ -139,9 +139,9 @@ CAKE_PROGRAM_KERNEL(const __grid_constant__ CUtensorMap A, const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 144); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_82f698f67a1cca6e4291_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_82f698f67a1cca6e4291_kernel.cu index 6ef9b8c6932..28ef7e6a599 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_82f698f67a1cca6e4291_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_82f698f67a1cca6e4291_kernel.cu @@ -125,9 +125,9 @@ kernel_cake_dense_projection_det_gemm_82f698f67a1cca6e4291(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 216); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_kernel.cu index fe6b5fa1399..a74bbbd0edd 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_895f9ba7159bea07aedd_kernel.cu @@ -124,9 +124,9 @@ kernel_cake_dense_projection_det_gemm_895f9ba7159bea07aedd(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 208); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu index a40e55ed2df..946217932f2 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_8e1f6805342f620cca8c_kernel.cu @@ -162,9 +162,9 @@ CAKE_PROGRAM_KERNEL(const __grid_constant__ CUtensorMap A, const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) #if !defined(CAKE_DPG_DENSE_PROJ_DET_GEMM_NN_N256_F32_V8_PRO7) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 216); diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9aa9aa60b0c544bfab06_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9aa9aa60b0c544bfab06_kernel.cu index 6a28cb496ee..8f02e727eb6 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9aa9aa60b0c544bfab06_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9aa9aa60b0c544bfab06_kernel.cu @@ -128,9 +128,9 @@ kernel_cake_dense_projection_det_gemm_9aa9aa60b0c544bfab06(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 176); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu index 60acb82f0b0..a5159ce5c2a 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_9cc6363566760a38b4fe_kernel.cu @@ -173,9 +173,9 @@ CAKE_PROGRAM_KERNEL(const __grid_constant__ CUtensorMap A, const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) #if !defined(CAKE_DPG_DENSE_PROJ_DET_GEMM_KN_N256_M256_OV_HT_PRO15) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 152); diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_kernel.cu index 58594f85212..cee53060f25 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052_kernel.cu @@ -126,9 +126,9 @@ kernel_cake_dense_projection_det_gemm_b26ac3f3ed56a78cb052(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 144); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b58b50619ef5ed46e109_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b58b50619ef5ed46e109_kernel.cu index b68f5654222..a4f03324aec 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b58b50619ef5ed46e109_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_b58b50619ef5ed46e109_kernel.cu @@ -128,9 +128,9 @@ kernel_cake_dense_projection_det_gemm_b58b50619ef5ed46e109(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 200); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ca9a846738b1443f7318_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ca9a846738b1443f7318_kernel.cu index 60b286c0daa..bcea232b0c5 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ca9a846738b1443f7318_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ca9a846738b1443f7318_kernel.cu @@ -126,9 +126,9 @@ kernel_cake_dense_projection_det_gemm_ca9a846738b1443f7318(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 144); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu index 40cae4d2eec..c05488aa435 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d2a5e70bd49083888179_kernel.cu @@ -124,9 +124,9 @@ kernel_cake_dense_projection_det_gemm_d2a5e70bd49083888179(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 208); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d6ebaab2415ea7b00de9_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d6ebaab2415ea7b00de9_kernel.cu index 9ae60b0a6e1..a29bd3e6f1c 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d6ebaab2415ea7b00de9_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_d6ebaab2415ea7b00de9_kernel.cu @@ -124,9 +124,9 @@ kernel_cake_dense_projection_det_gemm_d6ebaab2415ea7b00de9(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 224); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ddff2b911137449c1dfc_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ddff2b911137449c1dfc_kernel.cu index 0b57af986d9..e9d0252b472 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ddff2b911137449c1dfc_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_ddff2b911137449c1dfc_kernel.cu @@ -132,9 +132,9 @@ kernel_cake_dense_projection_det_gemm_ddff2b911137449c1dfc(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 232); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_de7492eeb6e24b31687a_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_de7492eeb6e24b31687a_kernel.cu index c8b013bb0ce..7136909e25c 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_de7492eeb6e24b31687a_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_de7492eeb6e24b31687a_kernel.cu @@ -124,9 +124,9 @@ kernel_cake_dense_projection_det_gemm_de7492eeb6e24b31687a(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 144); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_binding.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_binding.cu new file mode 100644 index 00000000000..1d1158d02f0 --- /dev/null +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_binding.cu @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Generated by Cake: the tvm-ffi host binding of kernel_cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981, for the device source that defines it. Do not edit. +#include +#include + +#include "cake_ffi_checks.h" + +#include + +extern "C" __global__ void kernel_cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981(const __grid_constant__ CUtensorMap W, const __grid_constant__ CUtensorMap A, uint8_t* __restrict__ out, int M, int N, int K, int B, int ldo, int out_l); + +namespace cake_binding_kernel_cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981 { + +using tvm::ffi::TensorView; + +constexpr const char* KERNEL_NAME = "kernel_cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981"; + +void CheckCuda(cudaError_t status, const char* what) { + TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) + << KERNEL_NAME << ": " << what << " failed: " << cudaGetErrorString(status); +} + +void CheckCu(CUresult result, const char* what) { + if (result != CUDA_SUCCESS) { + const char* name = nullptr; + const char* message = nullptr; + cuGetErrorName(result, &name); + cuGetErrorString(result, &message); + TVM_FFI_THROW(RuntimeError) << KERNEL_NAME << ": " << what << " failed with " + << (name ? name : "an unknown CUresult") << " (" << static_cast(result) + << "): " << (message ? message : ""); + } +} + +void* DataPtr(const TensorView& t) { return static_cast(t.data_ptr()) + t.byte_offset(); } + +void CheckUnitInnerStride(const TensorView& t, const char* name) { + TVM_FFI_CHECK(t.stride(-1) == 1, ValueError) + << KERNEL_NAME << ": '" << name << "' needs a unit innermost stride, got " << t.stride(-1); +} + +// The product of the extents above the trailing `trailing` dims. +int64_t NumelAbove(const TensorView& t, int trailing) { + int64_t product = 1; + for (int axis = 0; axis < t.ndim() - trailing; ++axis) product *= t.size(axis); + return product; +} + +[[noreturn]] void NoInt64(int64_t a, const char* op, int64_t b) { + TVM_FFI_THROW(ValueError) << KERNEL_NAME << ": " << a << " " << op << " " << b + << " has no int64 value in a host check"; +} + +int64_t CDiv(int64_t a, int64_t b) { + if (b == 0 || (a == INT64_MIN && b == -1)) NoInt64(a, "/", b); + return a / b; +} + +int64_t CMod(int64_t a, int64_t b) { + if (b == 0) NoInt64(a, "%", b); + return b == -1 ? 0 : a % b; +} + +cuuint64_t Extent(int64_t value, const char* descriptor) { + TVM_FFI_CHECK(value > 0, ValueError) + << KERNEL_NAME << ": an extent of descriptor '" << descriptor << "' resolves to " << value; + return static_cast(value); +} + +cuuint64_t Stride(int64_t value, const char* descriptor) { + TVM_FFI_CHECK(value >= 0, ValueError) + << KERNEL_NAME << ": a stride of descriptor '" << descriptor << "' resolves to " << value; + return static_cast(value); +} + +CUtensorMap EncodeTma_W(const TensorView& t) { + TVM_FFI_CHECK(t.ndim() >= 3, ValueError) + << KERNEL_NAME << ": descriptor 'W' takes a source of at least 3 dims, got " << t.ndim(); + CheckUnitInnerStride(t, "W"); + int64_t d1 = t.size(-1); + int64_t d2 = t.size(-2); + int64_t outer2 = NumelAbove(t, 2); + check_dense_leading_dims(t, 2, "W"); + int64_t s2 = t.stride(-2) * 2; + int64_t s3 = t.stride(-3) * 2; + TVM_FFI_CHECK((CMod(d1, 64LL) == 0LL), ValueError) + << KERNEL_NAME << ": descriptor 'W' requires (CMod(d1, 64) == 0)"; + cuuint64_t global_dim[4] = {Extent(64LL, "W"), Extent(d2, "W"), Extent(CDiv(d1, 64LL), "W"), Extent(outer2, "W")}; + cuuint64_t global_strides[3] = {Stride(s2, "W"), Stride(128LL, "W"), Stride(s3, "W")}; + cuuint32_t box_dim[4] = {64u, 128u, 1u, 1u}; + cuuint32_t elem_strides[4] = {1u, 1u, 1u, 1u}; + const bool allow_oob_box[4] = {false, true, false, false}; + for (int axis = 0; axis < 4; ++axis) { + TVM_FFI_CHECK(allow_oob_box[axis] || box_dim[axis] <= global_dim[axis], ValueError) + << KERNEL_NAME << ": descriptor 'W' axis " << axis << " (0 is the innermost) has a TMA box of " + << box_dim[axis] << " elements but the source extent along it is " << global_dim[axis] + << ". A box may not exceed its extent unless the axis is listed in the descriptor's allow_oob_box (CK.describe_tma(..., allow_oob_box=(axis,))). List it when the box is a tile that may run past the end of the axis, a tail tile: a load fills the part past the extent with zeros (its bytes still count toward the barrier's transaction) and a store or reduce skips it, so that is safe when the kernel ignores or masks those rows and columns. If the box was meant to fit, box_dim or the extent is wrong: check box_dim, global_dim and the tensor's shape"; + } + TVM_FFI_CHECK(global_strides[0] % 16 == 0 && global_strides[1] % 16 == 0 && global_strides[2] % 16 == 0, ValueError) + << KERNEL_NAME << ": the strides of descriptor 'W' must be multiples of 16 bytes"; + void* address = DataPtr(t); + TVM_FFI_CHECK(reinterpret_cast(address) % 16 == 0, ValueError) + << KERNEL_NAME << ": the address of descriptor 'W' must be 16-byte aligned"; + CUtensorMap map{}; + CheckCu(cuTensorMapEncodeTiled(&map, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 4, address, global_dim, global_strides, + box_dim, elem_strides, CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_L2_PROMOTION_NONE, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE), + "cuTensorMapEncodeTiled of W"); + return map; +} + +CUtensorMap EncodeTma_A(const TensorView& t) { + TVM_FFI_CHECK(t.ndim() >= 3, ValueError) + << KERNEL_NAME << ": descriptor 'A' takes a source of at least 3 dims, got " << t.ndim(); + CheckUnitInnerStride(t, "A"); + int64_t d1 = t.size(-1); + int64_t d2 = t.size(-2); + int64_t outer2 = NumelAbove(t, 2); + check_dense_leading_dims(t, 2, "A"); + int64_t s2 = t.stride(-2) * 2; + int64_t s3 = t.stride(-3) * 2; + TVM_FFI_CHECK((CMod(d1, 64LL) == 0LL), ValueError) + << KERNEL_NAME << ": descriptor 'A' requires (CMod(d1, 64) == 0)"; + cuuint64_t global_dim[4] = {Extent(64LL, "A"), Extent(d2, "A"), Extent(CDiv(d1, 64LL), "A"), Extent(outer2, "A")}; + cuuint64_t global_strides[3] = {Stride(s2, "A"), Stride(128LL, "A"), Stride(s3, "A")}; + cuuint32_t box_dim[4] = {64u, 64u, 1u, 1u}; + cuuint32_t elem_strides[4] = {1u, 1u, 1u, 1u}; + const bool allow_oob_box[4] = {false, true, false, false}; + for (int axis = 0; axis < 4; ++axis) { + TVM_FFI_CHECK(allow_oob_box[axis] || box_dim[axis] <= global_dim[axis], ValueError) + << KERNEL_NAME << ": descriptor 'A' axis " << axis << " (0 is the innermost) has a TMA box of " + << box_dim[axis] << " elements but the source extent along it is " << global_dim[axis] + << ". A box may not exceed its extent unless the axis is listed in the descriptor's allow_oob_box (CK.describe_tma(..., allow_oob_box=(axis,))). List it when the box is a tile that may run past the end of the axis, a tail tile: a load fills the part past the extent with zeros (its bytes still count toward the barrier's transaction) and a store or reduce skips it, so that is safe when the kernel ignores or masks those rows and columns. If the box was meant to fit, box_dim or the extent is wrong: check box_dim, global_dim and the tensor's shape"; + } + TVM_FFI_CHECK(global_strides[0] % 16 == 0 && global_strides[1] % 16 == 0 && global_strides[2] % 16 == 0, ValueError) + << KERNEL_NAME << ": the strides of descriptor 'A' must be multiples of 16 bytes"; + void* address = DataPtr(t); + TVM_FFI_CHECK(reinterpret_cast(address) % 16 == 0, ValueError) + << KERNEL_NAME << ": the address of descriptor 'A' must be 16-byte aligned"; + CUtensorMap map{}; + CheckCu(cuTensorMapEncodeTiled(&map, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 4, address, global_dim, global_strides, + box_dim, elem_strides, CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_L2_PROMOTION_NONE, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE), + "cuTensorMapEncodeTiled of A"); + return map; +} + +void Run(TensorView a_W, TensorView a_A, TensorView a_out, int64_t a_M, int64_t a_N, int64_t a_K, int64_t a_B, int64_t a_ldo, int64_t a_out_l, int64_t a_enable_pdl, int64_t grid_x, int64_t grid_y, int64_t grid_z) { + DLDevice dev = a_W.device(); + tvm::ffi::CUDADeviceGuard device_guard(dev.device_id); + check_cuda_tensor(a_W, "W"); + check_dtype(a_W, DLDataType{kDLBfloat, 16, 1}, "W"); + check_cuda_tensor(a_A, "A"); + check_dtype(a_A, DLDataType{kDLBfloat, 16, 1}, "A"); + check_cuda_tensor(a_out, "out"); + check_dtype(a_out, DLDataType{kDLUInt, 8, 1}, "out"); + TVM_FFI_CHECK(a_M >= -2147483648LL && a_M <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'M' = " << a_M + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_N >= -2147483648LL && a_N <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'N' = " << a_N + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_K >= -2147483648LL && a_K <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'K' = " << a_K + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_B >= -2147483648LL && a_B <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'B' = " << a_B + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_ldo >= -2147483648LL && a_ldo <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'ldo' = " << a_ldo + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_out_l >= -2147483648LL && a_out_l <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'out_l' = " << a_out_l + << " is outside the i32 range [-2147483648, 2147483647]"; + check_same_device(a_A, a_W, "A", "W"); + check_contiguous(a_out, "out"); + check_same_device(a_out, a_W, "out", "W"); + TVM_FFI_CHECK((CMod(a_K, 64LL) == 0LL), ValueError) + << KERNEL_NAME << ": requires (CMod(K, 64) == 0)"; + TVM_FFI_CHECK((CMod(a_N, 8LL) == 0LL), ValueError) + << KERNEL_NAME << ": requires (CMod(N, 8) == 0)"; + TVM_FFI_CHECK((CMod(a_ldo, 8LL) == 0LL), ValueError) + << KERNEL_NAME << ": requires (CMod(ldo, 8) == 0)"; + TVM_FFI_CHECK((CMod(a_out_l, 8LL) == 0LL), ValueError) + << KERNEL_NAME << ": requires (CMod(out_l, 8) == 0)"; + TVM_FFI_CHECK((a_ldo >= a_N), ValueError) + << KERNEL_NAME << ": requires (ldo >= N)"; + TVM_FFI_CHECK(a_enable_pdl == 0LL || a_enable_pdl == 1LL, ValueError) + << KERNEL_NAME << ": 'enable_pdl' = " << a_enable_pdl << " is not 0 or 1"; + TVM_FFI_CHECK(grid_x > 0 && grid_y > 0 && grid_z > 0, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") must be positive"; + TVM_FFI_CHECK(grid_x <= 2147483647 && grid_y <= 65535 && grid_z <= 65535, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") exceeds CUDA's limits"; + cudaStream_t stream = static_cast(TVMFFIEnvGetStream(dev.device_type, dev.device_id)); + { + // Once per thread per device: the kernel's function attributes hold per device; the tensor-map encoders need this thread's context of the device bound. + static thread_local uint64_t prepared_devices = 0; + const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; + if (device_bit == 0 || (prepared_devices & device_bit) == 0) { + CheckCuda(cudaSetDevice(dev.device_id), "cudaSetDevice"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981), cudaFuncAttributeMaxDynamicSharedMemorySize, 214016), "cudaFuncSetAttribute"); + prepared_devices |= device_bit; + } + } + CUtensorMap v_W = EncodeTma_W(a_W); + CUtensorMap v_A = EncodeTma_A(a_A); + void* v_out = DataPtr(a_out); + int32_t v_M = static_cast(a_M); + int32_t v_N = static_cast(a_N); + int32_t v_K = static_cast(a_K); + int32_t v_B = static_cast(a_B); + int32_t v_ldo = static_cast(a_ldo); + int32_t v_out_l = static_cast(a_out_l); + void* kernel_args[] = {&v_W, &v_A, &v_out, &v_M, &v_N, &v_K, &v_B, &v_ldo, &v_out_l}; + cudaLaunchConfig_t config{}; + config.gridDim = dim3(static_cast(grid_x), static_cast(grid_y), static_cast(grid_z)); + config.blockDim = dim3(256, 1, 1); + config.dynamicSmemBytes = 214016; + config.stream = stream; + cudaLaunchAttribute attrs[1] = {}; + int n = 0; + if (a_enable_pdl) { + attrs[n].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[n].val.programmaticStreamSerializationAllowed = 1; + ++n; + } + config.attrs = attrs; + config.numAttrs = n; + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981), kernel_args), + "cudaLaunchKernelExC"); +} + +} // namespace cake_binding_kernel_cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981 + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_kernel.cu new file mode 100644 index 00000000000..ef21981f866 --- /dev/null +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981_kernel.cu @@ -0,0 +1,869 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +#if defined(__CUDACC_RTC__) +typedef unsigned long long uint64_t; +#else +typedef unsigned long uint64_t; +#endif +static_assert(sizeof(uint64_t) == 8, "Cake requires an LP64 CUDA host ABI"); +typedef signed int int32_t; +typedef short int int16_t; +struct __align__(64) CakeTensorMap64 { uint64_t opaque[16]; }; +static_assert(sizeof(CakeTensorMap64) == 128, "64-aligned tensor-map ABI size"); +static_assert(alignof(CakeTensorMap64) == 64, "64-aligned tensor-map ABI alignment"); + +#if defined(__CUDACC_RTC__) +typedef struct __align__(128) { uint64_t opaque[16]; } CUtensorMap; +#else +#include +#endif + +static_assert(sizeof(CUtensorMap) == 128, "CUtensorMap CUDA ABI must be 128 bytes"); +#include +#include + +#define CAKE_INF CUDART_INF_F +#define TMEM_NCOLS 64 +#define TMEM_ACCUM_OFFSET 0 +#define NUM_TMA_PIPE_STAGES 8 +#define NUM_EPI_PIPE_STAGES 1 +#define SMEM_SMEM_W_OFF 1024 +#define SMEM_SMEM_W_STAGE_BYTES 16384 +#define SMEM_SMEM_W_STRIDE 16384 +#define SMEM_SMEM_A_OFF 132096 +#define SMEM_SMEM_A_STAGE_BYTES 8192 +#define SMEM_SMEM_A_STRIDE 8192 +#define SMEM_STAGING_OFF 197632 +#define SMEM_STAGING_STAGE_BYTES 16384 +#define SMEM_STAGING_STRIDE 16384 +#define SMEM_TOTAL 214016 +#define THREADS 256 + +#include + +__device__ __forceinline__ uint32_t elect_sync() { + uint32_t pred = 0; + asm volatile( + "{\n\t" + ".reg .pred %%px;\n\t" + "elect.sync _|%%px, %1;\n\t" + "@%%px mov.s32 %0, 1;\n\t" + "}\n" + : "+r"(pred) + : "r"(0xFFFFFFFF)); + return pred; +} + + +__device__ __forceinline__ void mbarrier_init(int mbar_addr, int count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" + :: "r"(mbar_addr), "r"(count) : "memory"); +} + + + +// CTA-local pipelines have short, resident producer/consumer edges. Omitting +// suspendTimeHint keeps a miss on the lightweight TRYWAIT retry path; the +// explicit loop still makes this helper blocking until acquire succeeds. +__device__ __forceinline__ void mbarrier_wait(int mbar_addr, int phase) { + asm volatile( + "{\n\t" + ".reg .pred P1;\n\t" + "LAB_WAIT:\n\t" + "mbarrier.try_wait.parity.acquire.cta.shared::cta.b64" + " P1, [%0], %1;\n\t" + "@P1 bra.uni DONE;\n\t" + "bra.uni LAB_WAIT;\n\t" + "DONE:\n\t" + "}\n" + :: "r"(mbar_addr), "r"(phase) : "memory"); +} + + +__device__ __forceinline__ void tcgen05_mma_f16( + int taddr, uint64_t a_desc, uint64_t b_desc, + uint32_t i_desc, int enable_input_d) { + asm volatile( + "{\n\t" + ".reg .pred p;\n\t" + "setp.ne.b32 p, %4, 0;\n\t" + "tcgen05.mma.cta_group::1.kind::f16 [%0], %1, %2, %3, p;\n\t" + "}\n" + :: "r"(taddr), "l"(a_desc), "l"(b_desc), + "r"(i_desc), "r"(enable_input_d) + : "memory"); +} + + + +union MmaSmemDesc { + uint64_t u64; + uint32_t u32[2]; +}; + +__device__ __forceinline__ void incr_smem_desc_lo(uint64_t& smem_desc, uint32_t offset) { + MmaSmemDesc tmp; + tmp.u64 = smem_desc; + tmp.u32[0] += offset; + smem_desc = tmp.u64; +} + + +__device__ __forceinline__ void elect_commit(int mbar_addr) { + asm volatile( + "{\n\t" + ".reg .pred leader;\n\t" + "elect.sync _|leader, 0xFFFFFFFF;\n\t" + "@leader tcgen05.commit.cta_group::1.mbarrier::arrive::one" + ".shared::cluster.b64 [%0];\n\t" + "}\n" + :: "r"(mbar_addr)); +} + + + +__device__ __forceinline__ void mbarrier_arrive_expect_tx(int mbar_addr, uint32_t bytes) { + asm volatile( + "mbarrier.arrive.expect_tx.release.cta.shared::cta.b64 _, [%0], %1;" + :: "r"(mbar_addr), "r"(bytes) : "memory"); +} + + + + + +__device__ __forceinline__ void tma_4d_gmem2smem( + int dst, const void *tmap_ptr, int x, int y, int z, int w, int mbar_addr) { + asm volatile( + "cp.async.bulk.tensor.4d.shared::cta.global" + ".mbarrier::complete_tx::bytes" + " [%0], [%1, {%2, %3, %4, %5}], [%6];" + :: "r"(dst), "l"(tmap_ptr), "r"(x), "r"(y), "r"(z), "r"(w), + "r"(mbar_addr) : "memory"); +} + + + +__device__ __forceinline__ uint32_t make_warp_uniform(uint32_t val) { + uint32_t result; + asm volatile("shfl.sync.idx.b32 %0, %1, 0, 0x1f, 0xffffffff;" + : "=r"(result) : "r"(val)); + return result; +} + +extern "C" { + +__global__ __launch_bounds__(256) void +kernel_cake_dense_projection_det_gemm_edf7fa4c4f889f0c4981(const __grid_constant__ CUtensorMap W, const __grid_constant__ CUtensorMap A, uint8_t* __restrict__ out, int M, int N, int K, int B, int ldo, int out_l) +{ + const int tid = threadIdx.x; + const int warp = make_warp_uniform(tid / 32); + const int lane = tid % 32; + + extern __shared__ __align__(1024) char smem_raw[]; + int smem; +#if __CUDA_ARCH__ == 1000 + asm volatile("{ .reg .u64 smem_ptr; cvta.to.shared.u64 smem_ptr, %1; cvt.u32.u64 %0, smem_ptr; }" : "=r"(smem) : "l"(smem_raw)); + smem = make_warp_uniform(smem); +#else + smem = (int)(unsigned long long)__cvta_generic_to_shared(smem_raw); +#endif + + const int mbar_base = smem; + #define tma_full_addr (mbar_base + 0) + #define tma_empty_addr (mbar_base + 64) + #define mma_epi_full_addr (mbar_base + 128) + + const int bid = blockIdx.x; + const int num_bids = gridDim.x; + + const int cta_rank = 0; + + // Kernel setup ops + __nv_bfloat16* smem_w = reinterpret_cast<__nv_bfloat16*>(smem_raw + 1024); + const int smem_w_addr = smem + 1024; + __nv_bfloat16* smem_a = reinterpret_cast<__nv_bfloat16*>(smem_raw + 132096); + const int smem_a_addr = smem + 132096; + unsigned int* staging = reinterpret_cast(smem_raw + 197632); + const int staging_addr = smem + 197632; + asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); + if (warp == 0) { asm volatile("prefetch.tensormap [%0];" :: "l"((uint64_t)((&W))) : "memory"); } + if (warp == 1) { asm volatile("prefetch.tensormap [%0];" :: "l"((uint64_t)((&A))) : "memory"); } + + // Mbarrier init (3 pipeline groups, 0 ordered-sequence groups, 17 barriers) + // Mbarriers at smem_raw[0..136) + + if (warp == 0) { + uint32_t leader = elect_sync(); + if (leader) { + // --- pipeline 'tma_pipe' --- + // tma_full: 8 barriers, init_count=2 + mbarrier_init(smem + 0, 2); + mbarrier_init(smem + 8, 2); + mbarrier_init(smem + 16, 2); + mbarrier_init(smem + 24, 2); + mbarrier_init(smem + 32, 2); + mbarrier_init(smem + 40, 2); + mbarrier_init(smem + 48, 2); + mbarrier_init(smem + 56, 2); + // tma_empty: 8 barriers, init_count=1 + mbarrier_init(smem + 64, 1); + mbarrier_init(smem + 72, 1); + mbarrier_init(smem + 80, 1); + mbarrier_init(smem + 88, 1); + mbarrier_init(smem + 96, 1); + mbarrier_init(smem + 104, 1); + mbarrier_init(smem + 112, 1); + mbarrier_init(smem + 120, 1); + // --- pipeline 'epi_pipe' --- + // mma_epi_full: 1 barriers, init_count=1 + mbarrier_init(smem + 128, 1); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + } + } + + __syncthreads(); + + // TMEM alloc (64 columns, 64 used) + volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 136); + if (warp == 2) { + int _tmem_hold = smem + 136; + asm volatile("tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" :: "r"(_tmem_hold), "r"(64) : "memory"); + __syncwarp(); + asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;"); + } + + if (warp == 2 || warp == 4 || warp == 5 || warp == 6 || warp == 7) { + asm volatile("barrier.sync.aligned %0, %1;" :: "r"(1), "r"(160) : "memory"); + asm volatile("tcgen05.fence::after_thread_sync;"); + } + const int taddr = (warp == 2 || warp == 4 || warp == 5 || warp == 6 || warp == 7) ? tmem_addr_storage[0] : 0; + + // Kernel post-init ops + const int tmem_accum = taddr; + + // ---- Role: dma_w ---- + if (warp == 0) { + { // dma_w_main + unsigned int w_stage = 0; + int w_idx = blockIdx.x * 128; + int b_idx = blockIdx.z; + int k_tiles = K / 64; + int k_begin = 0; + unsigned int _phase_tma_empty = 1; + if (elect_sync()) { + #pragma unroll 4 + for (int k_tile = k_begin; k_tile < k_begin + k_tiles; k_tile++) { + mbarrier_wait(tma_empty_addr + (w_stage) * 8, _phase_tma_empty); + asm volatile( + "cp.async.bulk.tensor.4d.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint" + " [%0], [%1, {%2, %3, %4, %5}], [%6], %7;" + :: "r"(smem_w_addr + w_stage * 16384), "l"((&W)), "r"(0), "r"(w_idx), "r"(k_tile), "r"(b_idx), + "r"(tma_full_addr + (w_stage) * 8), "l"(0x12F0000000000000ULL) : "memory"); + mbarrier_arrive_expect_tx(tma_full_addr + (w_stage) * 8, 16384); + w_stage += 1; + if (w_stage == 8) { w_stage = 0; _phase_tma_empty ^= 1; } + } + } + } + } + // ---- Role: dma_a ---- + if (warp == 1) { + { // dma_a_main + unsigned int a_stage = 0; + int t_idx = blockIdx.y * 64; + int b_idx_a = blockIdx.z; + int k_tiles_a = K / 64; + int k_begin_a = 0; + unsigned int _phase_tma_empty_1 = 1; + if (elect_sync()) { + asm volatile("griddepcontrol.wait;" ::: "memory"); + #pragma unroll 4 + for (int k_tile_1 = k_begin_a; k_tile_1 < k_begin_a + k_tiles_a; k_tile_1++) { + mbarrier_wait(tma_empty_addr + (a_stage) * 8, _phase_tma_empty_1); + tma_4d_gmem2smem(smem_a_addr + a_stage * 8192, (&A), 0, t_idx, k_tile_1, b_idx_a, tma_full_addr + (a_stage) * 8); + mbarrier_arrive_expect_tx(tma_full_addr + (a_stage) * 8, 8192); + a_stage += 1; + if (a_stage == 8) { a_stage = 0; _phase_tma_empty_1 ^= 1; } + } + } + } + } + // ---- Role: mma ---- + if (warp == 2) { + { // mma_main + unsigned int mma_stage = 0; + int k_tiles_m = K / 64; + unsigned int _phase_tma_full = 0; + #pragma unroll 2 + for (int k_tile_2 = 0; k_tile_2 < k_tiles_m; k_tile_2++) { + mbarrier_wait(tma_full_addr + (mma_stage) * 8, _phase_tma_full); + asm volatile("tcgen05.fence::after_thread_sync;"); + int init_flag = ((k_tile_2 == 0) ? 1 : 0); +#if __CUDA_ARCH__ == 1070 + int _mma_a_lo_0 = make_warp_uniform((((smem_w_addr) >> 4) & 0x7FFF) + (mma_stage) * 1024); + int _mma_b_lo_0 = make_warp_uniform((((smem_a_addr) >> 4) & 0x7FFF) + (mma_stage) * 512); +#else + int _mma_a_lo_0 = make_warp_uniform((((smem_w_addr) >> 4) & 0x3FFF) + (mma_stage) * 1024); + int _mma_b_lo_0 = make_warp_uniform((((smem_a_addr) >> 4) & 0x3FFF) + (mma_stage) * 512); +#endif + { + uint64_t _mma_ss_a_desc_0 = (static_cast(0x40004040U) << 32) | static_cast(_mma_a_lo_0); + uint64_t _mma_ss_b_desc_0 = (static_cast(0x40004040U) << 32) | static_cast(_mma_b_lo_0); + if (elect_sync()) { + tcgen05_mma_f16(tmem_accum, _mma_ss_a_desc_0, _mma_ss_b_desc_0, 135267472, ((init_flag) ? 0 : 1)); + } + incr_smem_desc_lo(_mma_ss_a_desc_0, 2U); + incr_smem_desc_lo(_mma_ss_b_desc_0, 2U); + if (elect_sync()) { + tcgen05_mma_f16(tmem_accum, _mma_ss_a_desc_0, _mma_ss_b_desc_0, 135267472, 1); + } + incr_smem_desc_lo(_mma_ss_a_desc_0, 2U); + incr_smem_desc_lo(_mma_ss_b_desc_0, 2U); + if (elect_sync()) { + tcgen05_mma_f16(tmem_accum, _mma_ss_a_desc_0, _mma_ss_b_desc_0, 135267472, 1); + } + incr_smem_desc_lo(_mma_ss_a_desc_0, 2U); + incr_smem_desc_lo(_mma_ss_b_desc_0, 2U); + if (elect_sync()) { + tcgen05_mma_f16(tmem_accum, _mma_ss_a_desc_0, _mma_ss_b_desc_0, 135267472, 1); + } + } + elect_commit(tma_empty_addr + (mma_stage) * 8); + mma_stage += 1; + if (mma_stage == 8) { mma_stage = 0; _phase_tma_full ^= 1; } + } + elect_commit(mma_epi_full_addr); + } + } + // ---- Role: epilogue ---- + if (warp >= 4 && warp <= 7) { + { // epilogue_main + const int epi_warp = warp % 4; + const int epi_tid = epi_warp * 32 + lane; + int row_local = epi_tid; + int col_half = 0; + int n_row = blockIdx.x * 128 + row_local; + int m_first = blockIdx.y * 64 + col_half * 64; + int b_idx_e = blockIdx.z; + mbarrier_wait(mma_epi_full_addr, 0); + asm volatile("tcgen05.fence::after_thread_sync;"); + unsigned int tmem_addr = taddr + (unsigned int)(epi_warp * 32 << 16); + float _tmem_load_0[64]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x32.b32" + " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];" + : "=f"(_tmem_load_0[0]), "=f"(_tmem_load_0[1]), "=f"(_tmem_load_0[2]), "=f"(_tmem_load_0[3]), "=f"(_tmem_load_0[4]), "=f"(_tmem_load_0[5]), "=f"(_tmem_load_0[6]), "=f"(_tmem_load_0[7]), "=f"(_tmem_load_0[8]), "=f"(_tmem_load_0[9]), "=f"(_tmem_load_0[10]), "=f"(_tmem_load_0[11]), "=f"(_tmem_load_0[12]), "=f"(_tmem_load_0[13]), "=f"(_tmem_load_0[14]), "=f"(_tmem_load_0[15]), "=f"(_tmem_load_0[16]), "=f"(_tmem_load_0[17]), "=f"(_tmem_load_0[18]), "=f"(_tmem_load_0[19]), "=f"(_tmem_load_0[20]), "=f"(_tmem_load_0[21]), "=f"(_tmem_load_0[22]), "=f"(_tmem_load_0[23]), "=f"(_tmem_load_0[24]), "=f"(_tmem_load_0[25]), "=f"(_tmem_load_0[26]), "=f"(_tmem_load_0[27]), "=f"(_tmem_load_0[28]), "=f"(_tmem_load_0[29]), "=f"(_tmem_load_0[30]), "=f"(_tmem_load_0[31]) + : "r"(tmem_addr)); + asm volatile("tcgen05.wait::ld.sync.aligned;"); + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x32.b32" + " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];" + : "=f"(_tmem_load_0[32]), "=f"(_tmem_load_0[33]), "=f"(_tmem_load_0[34]), "=f"(_tmem_load_0[35]), "=f"(_tmem_load_0[36]), "=f"(_tmem_load_0[37]), "=f"(_tmem_load_0[38]), "=f"(_tmem_load_0[39]), "=f"(_tmem_load_0[40]), "=f"(_tmem_load_0[41]), "=f"(_tmem_load_0[42]), "=f"(_tmem_load_0[43]), "=f"(_tmem_load_0[44]), "=f"(_tmem_load_0[45]), "=f"(_tmem_load_0[46]), "=f"(_tmem_load_0[47]), "=f"(_tmem_load_0[48]), "=f"(_tmem_load_0[49]), "=f"(_tmem_load_0[50]), "=f"(_tmem_load_0[51]), "=f"(_tmem_load_0[52]), "=f"(_tmem_load_0[53]), "=f"(_tmem_load_0[54]), "=f"(_tmem_load_0[55]), "=f"(_tmem_load_0[56]), "=f"(_tmem_load_0[57]), "=f"(_tmem_load_0[58]), "=f"(_tmem_load_0[59]), "=f"(_tmem_load_0[60]), "=f"(_tmem_load_0[61]), "=f"(_tmem_load_0[62]), "=f"(_tmem_load_0[63]) + : "r"(tmem_addr + 32)); + asm volatile("tcgen05.wait::ld.sync.aligned;"); + { + __nv_bfloat16 _bval_0 = __float2bfloat16_rn(_tmem_load_0[0]); + uint16_t _bits_0 = *(uint16_t*)&_bval_0; + uint32_t _addr_0 = static_cast(staging_addr + (unsigned int)((col_half * 64 * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_0), "h"(_bits_0)); + } + { + __nv_bfloat16 _bval_1 = __float2bfloat16_rn(_tmem_load_0[1]); + uint16_t _bits_1 = *(uint16_t*)&_bval_1; + uint32_t _addr_1 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 1) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_1), "h"(_bits_1)); + } + { + __nv_bfloat16 _bval_2 = __float2bfloat16_rn(_tmem_load_0[2]); + uint16_t _bits_2 = *(uint16_t*)&_bval_2; + uint32_t _addr_2 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 2) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_2), "h"(_bits_2)); + } + { + __nv_bfloat16 _bval_3 = __float2bfloat16_rn(_tmem_load_0[3]); + uint16_t _bits_3 = *(uint16_t*)&_bval_3; + uint32_t _addr_3 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 3) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_3), "h"(_bits_3)); + } + { + __nv_bfloat16 _bval_4 = __float2bfloat16_rn(_tmem_load_0[4]); + uint16_t _bits_4 = *(uint16_t*)&_bval_4; + uint32_t _addr_4 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 4) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_4), "h"(_bits_4)); + } + { + __nv_bfloat16 _bval_5 = __float2bfloat16_rn(_tmem_load_0[5]); + uint16_t _bits_5 = *(uint16_t*)&_bval_5; + uint32_t _addr_5 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 5) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_5), "h"(_bits_5)); + } + { + __nv_bfloat16 _bval_6 = __float2bfloat16_rn(_tmem_load_0[6]); + uint16_t _bits_6 = *(uint16_t*)&_bval_6; + uint32_t _addr_6 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 6) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_6), "h"(_bits_6)); + } + { + __nv_bfloat16 _bval_7 = __float2bfloat16_rn(_tmem_load_0[7]); + uint16_t _bits_7 = *(uint16_t*)&_bval_7; + uint32_t _addr_7 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 7) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_7), "h"(_bits_7)); + } + { + __nv_bfloat16 _bval_8 = __float2bfloat16_rn(_tmem_load_0[8]); + uint16_t _bits_8 = *(uint16_t*)&_bval_8; + uint32_t _addr_8 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 8) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_8), "h"(_bits_8)); + } + { + __nv_bfloat16 _bval_9 = __float2bfloat16_rn(_tmem_load_0[9]); + uint16_t _bits_9 = *(uint16_t*)&_bval_9; + uint32_t _addr_9 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 9) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_9), "h"(_bits_9)); + } + { + __nv_bfloat16 _bval_10 = __float2bfloat16_rn(_tmem_load_0[10]); + uint16_t _bits_10 = *(uint16_t*)&_bval_10; + uint32_t _addr_10 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 10) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_10), "h"(_bits_10)); + } + { + __nv_bfloat16 _bval_11 = __float2bfloat16_rn(_tmem_load_0[11]); + uint16_t _bits_11 = *(uint16_t*)&_bval_11; + uint32_t _addr_11 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 11) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_11), "h"(_bits_11)); + } + { + __nv_bfloat16 _bval_12 = __float2bfloat16_rn(_tmem_load_0[12]); + uint16_t _bits_12 = *(uint16_t*)&_bval_12; + uint32_t _addr_12 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 12) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_12), "h"(_bits_12)); + } + { + __nv_bfloat16 _bval_13 = __float2bfloat16_rn(_tmem_load_0[13]); + uint16_t _bits_13 = *(uint16_t*)&_bval_13; + uint32_t _addr_13 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 13) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_13), "h"(_bits_13)); + } + { + __nv_bfloat16 _bval_14 = __float2bfloat16_rn(_tmem_load_0[14]); + uint16_t _bits_14 = *(uint16_t*)&_bval_14; + uint32_t _addr_14 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 14) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_14), "h"(_bits_14)); + } + { + __nv_bfloat16 _bval_15 = __float2bfloat16_rn(_tmem_load_0[15]); + uint16_t _bits_15 = *(uint16_t*)&_bval_15; + uint32_t _addr_15 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 15) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_15), "h"(_bits_15)); + } + { + __nv_bfloat16 _bval_16 = __float2bfloat16_rn(_tmem_load_0[16]); + uint16_t _bits_16 = *(uint16_t*)&_bval_16; + uint32_t _addr_16 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 16) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_16), "h"(_bits_16)); + } + { + __nv_bfloat16 _bval_17 = __float2bfloat16_rn(_tmem_load_0[17]); + uint16_t _bits_17 = *(uint16_t*)&_bval_17; + uint32_t _addr_17 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 17) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_17), "h"(_bits_17)); + } + { + __nv_bfloat16 _bval_18 = __float2bfloat16_rn(_tmem_load_0[18]); + uint16_t _bits_18 = *(uint16_t*)&_bval_18; + uint32_t _addr_18 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 18) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_18), "h"(_bits_18)); + } + { + __nv_bfloat16 _bval_19 = __float2bfloat16_rn(_tmem_load_0[19]); + uint16_t _bits_19 = *(uint16_t*)&_bval_19; + uint32_t _addr_19 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 19) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_19), "h"(_bits_19)); + } + { + __nv_bfloat16 _bval_20 = __float2bfloat16_rn(_tmem_load_0[20]); + uint16_t _bits_20 = *(uint16_t*)&_bval_20; + uint32_t _addr_20 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 20) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_20), "h"(_bits_20)); + } + { + __nv_bfloat16 _bval_21 = __float2bfloat16_rn(_tmem_load_0[21]); + uint16_t _bits_21 = *(uint16_t*)&_bval_21; + uint32_t _addr_21 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 21) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_21), "h"(_bits_21)); + } + { + __nv_bfloat16 _bval_22 = __float2bfloat16_rn(_tmem_load_0[22]); + uint16_t _bits_22 = *(uint16_t*)&_bval_22; + uint32_t _addr_22 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 22) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_22), "h"(_bits_22)); + } + { + __nv_bfloat16 _bval_23 = __float2bfloat16_rn(_tmem_load_0[23]); + uint16_t _bits_23 = *(uint16_t*)&_bval_23; + uint32_t _addr_23 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 23) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_23), "h"(_bits_23)); + } + { + __nv_bfloat16 _bval_24 = __float2bfloat16_rn(_tmem_load_0[24]); + uint16_t _bits_24 = *(uint16_t*)&_bval_24; + uint32_t _addr_24 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 24) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_24), "h"(_bits_24)); + } + { + __nv_bfloat16 _bval_25 = __float2bfloat16_rn(_tmem_load_0[25]); + uint16_t _bits_25 = *(uint16_t*)&_bval_25; + uint32_t _addr_25 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 25) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_25), "h"(_bits_25)); + } + { + __nv_bfloat16 _bval_26 = __float2bfloat16_rn(_tmem_load_0[26]); + uint16_t _bits_26 = *(uint16_t*)&_bval_26; + uint32_t _addr_26 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 26) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_26), "h"(_bits_26)); + } + { + __nv_bfloat16 _bval_27 = __float2bfloat16_rn(_tmem_load_0[27]); + uint16_t _bits_27 = *(uint16_t*)&_bval_27; + uint32_t _addr_27 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 27) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_27), "h"(_bits_27)); + } + { + __nv_bfloat16 _bval_28 = __float2bfloat16_rn(_tmem_load_0[28]); + uint16_t _bits_28 = *(uint16_t*)&_bval_28; + uint32_t _addr_28 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 28) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_28), "h"(_bits_28)); + } + { + __nv_bfloat16 _bval_29 = __float2bfloat16_rn(_tmem_load_0[29]); + uint16_t _bits_29 = *(uint16_t*)&_bval_29; + uint32_t _addr_29 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 29) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_29), "h"(_bits_29)); + } + { + __nv_bfloat16 _bval_30 = __float2bfloat16_rn(_tmem_load_0[30]); + uint16_t _bits_30 = *(uint16_t*)&_bval_30; + uint32_t _addr_30 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 30) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_30), "h"(_bits_30)); + } + { + __nv_bfloat16 _bval_31 = __float2bfloat16_rn(_tmem_load_0[31]); + uint16_t _bits_31 = *(uint16_t*)&_bval_31; + uint32_t _addr_31 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 31) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_31), "h"(_bits_31)); + } + { + __nv_bfloat16 _bval_32 = __float2bfloat16_rn(_tmem_load_0[32]); + uint16_t _bits_32 = *(uint16_t*)&_bval_32; + uint32_t _addr_32 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 32) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_32), "h"(_bits_32)); + } + { + __nv_bfloat16 _bval_33 = __float2bfloat16_rn(_tmem_load_0[33]); + uint16_t _bits_33 = *(uint16_t*)&_bval_33; + uint32_t _addr_33 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 33) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_33), "h"(_bits_33)); + } + { + __nv_bfloat16 _bval_34 = __float2bfloat16_rn(_tmem_load_0[34]); + uint16_t _bits_34 = *(uint16_t*)&_bval_34; + uint32_t _addr_34 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 34) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_34), "h"(_bits_34)); + } + { + __nv_bfloat16 _bval_35 = __float2bfloat16_rn(_tmem_load_0[35]); + uint16_t _bits_35 = *(uint16_t*)&_bval_35; + uint32_t _addr_35 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 35) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_35), "h"(_bits_35)); + } + { + __nv_bfloat16 _bval_36 = __float2bfloat16_rn(_tmem_load_0[36]); + uint16_t _bits_36 = *(uint16_t*)&_bval_36; + uint32_t _addr_36 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 36) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_36), "h"(_bits_36)); + } + { + __nv_bfloat16 _bval_37 = __float2bfloat16_rn(_tmem_load_0[37]); + uint16_t _bits_37 = *(uint16_t*)&_bval_37; + uint32_t _addr_37 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 37) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_37), "h"(_bits_37)); + } + { + __nv_bfloat16 _bval_38 = __float2bfloat16_rn(_tmem_load_0[38]); + uint16_t _bits_38 = *(uint16_t*)&_bval_38; + uint32_t _addr_38 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 38) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_38), "h"(_bits_38)); + } + { + __nv_bfloat16 _bval_39 = __float2bfloat16_rn(_tmem_load_0[39]); + uint16_t _bits_39 = *(uint16_t*)&_bval_39; + uint32_t _addr_39 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 39) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_39), "h"(_bits_39)); + } + { + __nv_bfloat16 _bval_40 = __float2bfloat16_rn(_tmem_load_0[40]); + uint16_t _bits_40 = *(uint16_t*)&_bval_40; + uint32_t _addr_40 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 40) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_40), "h"(_bits_40)); + } + { + __nv_bfloat16 _bval_41 = __float2bfloat16_rn(_tmem_load_0[41]); + uint16_t _bits_41 = *(uint16_t*)&_bval_41; + uint32_t _addr_41 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 41) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_41), "h"(_bits_41)); + } + { + __nv_bfloat16 _bval_42 = __float2bfloat16_rn(_tmem_load_0[42]); + uint16_t _bits_42 = *(uint16_t*)&_bval_42; + uint32_t _addr_42 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 42) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_42), "h"(_bits_42)); + } + { + __nv_bfloat16 _bval_43 = __float2bfloat16_rn(_tmem_load_0[43]); + uint16_t _bits_43 = *(uint16_t*)&_bval_43; + uint32_t _addr_43 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 43) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_43), "h"(_bits_43)); + } + { + __nv_bfloat16 _bval_44 = __float2bfloat16_rn(_tmem_load_0[44]); + uint16_t _bits_44 = *(uint16_t*)&_bval_44; + uint32_t _addr_44 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 44) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_44), "h"(_bits_44)); + } + { + __nv_bfloat16 _bval_45 = __float2bfloat16_rn(_tmem_load_0[45]); + uint16_t _bits_45 = *(uint16_t*)&_bval_45; + uint32_t _addr_45 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 45) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_45), "h"(_bits_45)); + } + { + __nv_bfloat16 _bval_46 = __float2bfloat16_rn(_tmem_load_0[46]); + uint16_t _bits_46 = *(uint16_t*)&_bval_46; + uint32_t _addr_46 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 46) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_46), "h"(_bits_46)); + } + { + __nv_bfloat16 _bval_47 = __float2bfloat16_rn(_tmem_load_0[47]); + uint16_t _bits_47 = *(uint16_t*)&_bval_47; + uint32_t _addr_47 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 47) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_47), "h"(_bits_47)); + } + { + __nv_bfloat16 _bval_48 = __float2bfloat16_rn(_tmem_load_0[48]); + uint16_t _bits_48 = *(uint16_t*)&_bval_48; + uint32_t _addr_48 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 48) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_48), "h"(_bits_48)); + } + { + __nv_bfloat16 _bval_49 = __float2bfloat16_rn(_tmem_load_0[49]); + uint16_t _bits_49 = *(uint16_t*)&_bval_49; + uint32_t _addr_49 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 49) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_49), "h"(_bits_49)); + } + { + __nv_bfloat16 _bval_50 = __float2bfloat16_rn(_tmem_load_0[50]); + uint16_t _bits_50 = *(uint16_t*)&_bval_50; + uint32_t _addr_50 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 50) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_50), "h"(_bits_50)); + } + { + __nv_bfloat16 _bval_51 = __float2bfloat16_rn(_tmem_load_0[51]); + uint16_t _bits_51 = *(uint16_t*)&_bval_51; + uint32_t _addr_51 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 51) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_51), "h"(_bits_51)); + } + { + __nv_bfloat16 _bval_52 = __float2bfloat16_rn(_tmem_load_0[52]); + uint16_t _bits_52 = *(uint16_t*)&_bval_52; + uint32_t _addr_52 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 52) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_52), "h"(_bits_52)); + } + { + __nv_bfloat16 _bval_53 = __float2bfloat16_rn(_tmem_load_0[53]); + uint16_t _bits_53 = *(uint16_t*)&_bval_53; + uint32_t _addr_53 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 53) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_53), "h"(_bits_53)); + } + { + __nv_bfloat16 _bval_54 = __float2bfloat16_rn(_tmem_load_0[54]); + uint16_t _bits_54 = *(uint16_t*)&_bval_54; + uint32_t _addr_54 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 54) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_54), "h"(_bits_54)); + } + { + __nv_bfloat16 _bval_55 = __float2bfloat16_rn(_tmem_load_0[55]); + uint16_t _bits_55 = *(uint16_t*)&_bval_55; + uint32_t _addr_55 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 55) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_55), "h"(_bits_55)); + } + { + __nv_bfloat16 _bval_56 = __float2bfloat16_rn(_tmem_load_0[56]); + uint16_t _bits_56 = *(uint16_t*)&_bval_56; + uint32_t _addr_56 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 56) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_56), "h"(_bits_56)); + } + { + __nv_bfloat16 _bval_57 = __float2bfloat16_rn(_tmem_load_0[57]); + uint16_t _bits_57 = *(uint16_t*)&_bval_57; + uint32_t _addr_57 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 57) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_57), "h"(_bits_57)); + } + { + __nv_bfloat16 _bval_58 = __float2bfloat16_rn(_tmem_load_0[58]); + uint16_t _bits_58 = *(uint16_t*)&_bval_58; + uint32_t _addr_58 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 58) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_58), "h"(_bits_58)); + } + { + __nv_bfloat16 _bval_59 = __float2bfloat16_rn(_tmem_load_0[59]); + uint16_t _bits_59 = *(uint16_t*)&_bval_59; + uint32_t _addr_59 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 59) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_59), "h"(_bits_59)); + } + { + __nv_bfloat16 _bval_60 = __float2bfloat16_rn(_tmem_load_0[60]); + uint16_t _bits_60 = *(uint16_t*)&_bval_60; + uint32_t _addr_60 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 60) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_60), "h"(_bits_60)); + } + { + __nv_bfloat16 _bval_61 = __float2bfloat16_rn(_tmem_load_0[61]); + uint16_t _bits_61 = *(uint16_t*)&_bval_61; + uint32_t _addr_61 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 61) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_61), "h"(_bits_61)); + } + { + __nv_bfloat16 _bval_62 = __float2bfloat16_rn(_tmem_load_0[62]); + uint16_t _bits_62 = *(uint16_t*)&_bval_62; + uint32_t _addr_62 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 62) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_62), "h"(_bits_62)); + } + { + __nv_bfloat16 _bval_63 = __float2bfloat16_rn(_tmem_load_0[63]); + uint16_t _bits_63 = *(uint16_t*)&_bval_63; + uint32_t _addr_63 = static_cast(staging_addr + (unsigned int)(((col_half * 64 + 63) * 128 + row_local) * 2)); + asm volatile("st.shared.b16 [%0], %1;" :: "r"(_addr_63), "h"(_bits_63)); + } + asm volatile("tcgen05.fence::before_thread_sync;"); + asm volatile("bar.sync 2, 128;" ::: "memory"); + int c = epi_tid; + int t_row = c / 16; + int m_row = blockIdx.y * 64 + t_row; + int n_col = blockIdx.x * 128 + c % 16 * 8; + if (m_row < M && n_col < N) { + unsigned int words[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row * (long long)ldo + (long long)n_col) * 2)))[0] = reinterpret_cast(words)[0]; + } + int c_0 = epi_tid + 128; + int t_row_0 = c_0 / 16; + int m_row_0 = blockIdx.y * 64 + t_row_0; + int n_col_0 = blockIdx.x * 128 + c_0 % 16 * 8; + if (m_row_0 < M && n_col_0 < N) { + unsigned int words_1[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words_1[0])), "=r"(*reinterpret_cast(&words_1[(0) + 1])), "=r"(*reinterpret_cast(&words_1[(0) + 2])), "=r"(*reinterpret_cast(&words_1[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c_0 * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row_0 * (long long)ldo + (long long)n_col_0) * 2)))[0] = reinterpret_cast(words_1)[0]; + } + int c_1 = epi_tid + 256; + int t_row_1 = c_1 / 16; + int m_row_1 = blockIdx.y * 64 + t_row_1; + int n_col_1 = blockIdx.x * 128 + c_1 % 16 * 8; + if (m_row_1 < M && n_col_1 < N) { + unsigned int words_2[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words_2[0])), "=r"(*reinterpret_cast(&words_2[(0) + 1])), "=r"(*reinterpret_cast(&words_2[(0) + 2])), "=r"(*reinterpret_cast(&words_2[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c_1 * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row_1 * (long long)ldo + (long long)n_col_1) * 2)))[0] = reinterpret_cast(words_2)[0]; + } + int c_2 = epi_tid + 384; + int t_row_2 = c_2 / 16; + int m_row_2 = blockIdx.y * 64 + t_row_2; + int n_col_2 = blockIdx.x * 128 + c_2 % 16 * 8; + if (m_row_2 < M && n_col_2 < N) { + unsigned int words_3[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words_3[0])), "=r"(*reinterpret_cast(&words_3[(0) + 1])), "=r"(*reinterpret_cast(&words_3[(0) + 2])), "=r"(*reinterpret_cast(&words_3[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c_2 * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row_2 * (long long)ldo + (long long)n_col_2) * 2)))[0] = reinterpret_cast(words_3)[0]; + } + int c_3 = epi_tid + 512; + int t_row_3 = c_3 / 16; + int m_row_3 = blockIdx.y * 64 + t_row_3; + int n_col_3 = blockIdx.x * 128 + c_3 % 16 * 8; + if (m_row_3 < M && n_col_3 < N) { + unsigned int words_4[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words_4[0])), "=r"(*reinterpret_cast(&words_4[(0) + 1])), "=r"(*reinterpret_cast(&words_4[(0) + 2])), "=r"(*reinterpret_cast(&words_4[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c_3 * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row_3 * (long long)ldo + (long long)n_col_3) * 2)))[0] = reinterpret_cast(words_4)[0]; + } + int c_4 = epi_tid + 640; + int t_row_4 = c_4 / 16; + int m_row_4 = blockIdx.y * 64 + t_row_4; + int n_col_4 = blockIdx.x * 128 + c_4 % 16 * 8; + if (m_row_4 < M && n_col_4 < N) { + unsigned int words_5[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words_5[0])), "=r"(*reinterpret_cast(&words_5[(0) + 1])), "=r"(*reinterpret_cast(&words_5[(0) + 2])), "=r"(*reinterpret_cast(&words_5[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c_4 * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row_4 * (long long)ldo + (long long)n_col_4) * 2)))[0] = reinterpret_cast(words_5)[0]; + } + int c_5 = epi_tid + 768; + int t_row_5 = c_5 / 16; + int m_row_5 = blockIdx.y * 64 + t_row_5; + int n_col_5 = blockIdx.x * 128 + c_5 % 16 * 8; + if (m_row_5 < M && n_col_5 < N) { + unsigned int words_6[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words_6[0])), "=r"(*reinterpret_cast(&words_6[(0) + 1])), "=r"(*reinterpret_cast(&words_6[(0) + 2])), "=r"(*reinterpret_cast(&words_6[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c_5 * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row_5 * (long long)ldo + (long long)n_col_5) * 2)))[0] = reinterpret_cast(words_6)[0]; + } + int c_6 = epi_tid + 896; + int t_row_6 = c_6 / 16; + int m_row_6 = blockIdx.y * 64 + t_row_6; + int n_col_6 = blockIdx.x * 128 + c_6 % 16 * 8; + if (m_row_6 < M && n_col_6 < N) { + unsigned int words_7[4]; + asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" + : "=r"(*reinterpret_cast(&words_7[0])), "=r"(*reinterpret_cast(&words_7[(0) + 1])), "=r"(*reinterpret_cast(&words_7[(0) + 2])), "=r"(*reinterpret_cast(&words_7[(0) + 3])) + : "r"(staging_addr + (unsigned int)(c_6 * 16))); + reinterpret_cast(reinterpret_cast(out + (((long long)b_idx_e * (long long)out_l + (long long)m_row_6 * (long long)ldo + (long long)n_col_6) * 2)))[0] = reinterpret_cast(words_7)[0]; + } + } + } + + // Cleanup + asm volatile("tcgen05.fence::before_thread_sync;" ::: "memory"); + __syncthreads(); // barrier before TMEM dealloc + + if (warp == 2) { + asm volatile("tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" :: "r"(tmem_addr_storage[0]), "r"(64)); + } +} + +} // extern "C" diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu index 8ba61c3c5f6..014afabd9ba 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e_kernel.cu @@ -124,9 +124,9 @@ kernel_cake_dense_projection_det_gemm_f2e6d4e79cf21105fa1e(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 208); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_kernel.cu index 4294dee7785..5b2e9739a4c 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c_kernel.cu @@ -129,9 +129,9 @@ kernel_cake_dense_projection_det_gemm_f31e1dcb1df00973dc0c(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 144); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_fb4bcdf13915dfb96c59_kernel.cu b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_fb4bcdf13915dfb96c59_kernel.cu index 67b4fae8b1a..a728d09dc9a 100644 --- a/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_fb4bcdf13915dfb96c59_kernel.cu +++ b/flashinfer/experimental/dense_projection_det_gemm/csrc/cake_dense_projection_det_gemm/cake_dense_projection_det_gemm_fb4bcdf13915dfb96c59_kernel.cu @@ -122,9 +122,9 @@ kernel_cake_dense_projection_det_gemm_fb4bcdf13915dfb96c59(const __grid_constant } __syncwarp(); + asm volatile("barrier.cluster.arrive.relaxed.aligned;" ::: "memory"); asm volatile("barrier.cluster.wait.acquire.aligned;" ::: "memory"); - // TMEM alloc (512 columns, 512 used) volatile int* tmem_addr_storage = (volatile int*)(smem_raw + 208); if (warp == 1) { diff --git a/flashinfer/experimental/dense_projection_gemm/cake_backend.py b/flashinfer/experimental/dense_projection_gemm/cake_backend.py index 93bdedc66f6..458a964dc7f 100644 --- a/flashinfer/experimental/dense_projection_gemm/cake_backend.py +++ b/flashinfer/experimental/dense_projection_gemm/cake_backend.py @@ -2588,16 +2588,19 @@ def dense_projection_gemm( *, transposed_out: bool = False, deterministic: bool = True, + enable_pdl: bool = False, ) -> torch.Tensor: """``out[l, m, n] = sum_k A[l, m, k] B[l, k, n]`` for strided views (the cached convenience path; see the deterministic package for the view contract). ``deterministic=False`` runs a weight-gradient layout (both operands MN-major) on this package's split-K program where the planner takes the row; every other layout and - row runs the deterministic program.""" + row runs the deterministic program. ``enable_pdl`` reaches the deterministic program, whose swapped + small-token forward launches as a programmatic dependent launch (see its ``dense_projection_det_gemm``); this + package's programs launch without the attribute.""" if _resolve_deterministic(deterministic, "dense_projection_gemm") or ( _route(_k1_layout(A, B)) == "det" ): return _det_entry("dense_projection_gemm")( - A, B, out, transposed_out=transposed_out + A, B, out, transposed_out=transposed_out, enable_pdl=enable_pdl ) return _atomic_k1( "launch", @@ -2607,7 +2610,7 @@ def dense_projection_gemm( transposed_out=transposed_out, own={}, det_call=lambda: _det_entry("dense_projection_gemm")( - A, B, out, transposed_out=transposed_out + A, B, out, transposed_out=transposed_out, enable_pdl=enable_pdl ), ) diff --git a/tests/experimental/test_cake_dense_projection_det_gemm.py b/tests/experimental/test_cake_dense_projection_det_gemm.py index 0b007a4da83..76acdc16959 100644 --- a/tests/experimental/test_cake_dense_projection_det_gemm.py +++ b/tests/experimental/test_cake_dense_projection_det_gemm.py @@ -797,7 +797,7 @@ def _l2_bytes() -> int: # --------------------------------------------------------------------------- -ARG_KINDS = {"buffer", "tma_buffer", "workspace", "parameter", "grid"} +ARG_KINDS = {"buffer", "tma_buffer", "workspace", "parameter", "launch", "grid"} def test_registry_records_are_well_formed(): @@ -905,6 +905,19 @@ def test_registry_records_are_well_formed(): # the fp32 source, the [3, ...] BF16 stack and the element count assert ["buffer", "src"] in plan_items and ["buffer", "parts"] in plan_items assert ["parameter", "n"] in plan_items + elif record["template"].startswith("swap_ab_"): + # the swapped forward: the views' strides in its tensor maps and output pitches, PDL per call, 256 + # threads, no cluster, every architecture + assert ["parameter", "ldo"] in plan_items and [ + "parameter", + "out_l", + ] in plan_items + assert plan_items[-4] == ["launch", "enable_pdl"] + assert list(record["launch"]["block"]) == [256, 1, 1], name + assert record["template"] in dict( + (t, w) for w, t in cake_backend.SWAP_TEMPLATES + ) + assert record["arches"] == list(cake_jit.ARCHES), name else: assert ["parameter", "k_iters_split"] in plan_items launch = record["launch"] @@ -914,6 +927,7 @@ def test_registry_records_are_well_formed(): [1, 1, 1] if record["template"].endswith("_c1") or record["template"] == cake_backend.SPLIT_TEMPLATE + or record["template"].startswith("swap_ab_") else [CTA_GROUP, 1, 1] ) assert list(launch["cluster"]) == want_cluster, (name, launch) @@ -6243,9 +6257,9 @@ def test_template_knobs_invert_instance_symbol(): for template in table: tokens = cake_backend._template_tokens(template) if tokens is None: - # the router programs and the fused split program carry no knob tokens + # the router programs, the fused split program and the swapped forward carry no knob tokens assert ( - template.startswith("router_fp32_det_gemm_") + template.startswith(("router_fp32_det_gemm_", "swap_ab_")) or template == cake_backend.SPLIT_TEMPLATE ), ( arch, @@ -9149,3 +9163,187 @@ def test_repeated_router_launches_are_bitwise_identical(op, T): prepared.launch() torch.cuda.synchronize() assert torch.equal(v["out"], first) + + +# --------------------------------------------------------------------------- +# The swapped small-token forward (``swap_ab_`` programs) +# --------------------------------------------------------------------------- + +SWAP_TEMPLATES = dict((t, w) for w, t in cake_backend.SWAP_TEMPLATES) + + +def test_swap_ab_template_fills_one_wave_with_the_narrow_tile(): + """The narrow weight tile while its CTAs fit one wave of the device's SMs, the wide one up to two waves, the + planner's program beyond; ``None`` on an architecture that registers neither.""" + pick = cake_backend.swap_ab_template + for arch in cake_jit.ARCHES: + # the grouped wo_a rows: 8 groups x 1024 weight rows, 64-token tiles + assert pick(8, 64, 1024, 152, arch) == "swap_ab_w64_t64_s12" # 128 CTAs + assert pick(8, 65, 1024, 152, arch) == "swap_ab_w128_t64_s8" # 256 > 152 + assert pick(8, 128, 1024, 256, arch) == "swap_ab_w64_t64_s12" # 256 <= 256 + assert pick(8, 256, 1024, 152, arch) == "swap_ab_w128_t64_s8" # 256 <= 2 waves + # past two waves of the wide tile the planner's program runs + assert pick(8, 64, 4096, 152, arch) == "swap_ab_w128_t64_s8" # 256 CTAs + assert pick(8, 128, 4096, 152, arch) is None # 512 CTAs + assert pick(8, 128, 4096, 256, arch) == "swap_ab_w128_t64_s8" # 512 <= 2 x 256 + assert pick(8, 64, 1024, 152, "sm_90a") is None + + +def _swap_views(T, G=8, D=4096, R=1024, seed=SEED, token_major=True): + """The grouped projection views ``A [G, T, D]``, ``B [G, D, R]`` (K-major weights) and ``out [G, T, R]``: + token-major ``[T, G, *]`` storage permuted (SGLang's ``o`` and ``result``), or contiguous ``[G, T, *]``.""" + g = torch.Generator(device="cuda").manual_seed(seed) + w = (torch.randn(G, R, D, device="cuda", generator=g) * D**-0.5).to(torch.bfloat16) + if token_major: + o = torch.randn(T, G, D, device="cuda", generator=g).to(torch.bfloat16) + res = torch.full((T, G, R), float("nan"), device="cuda", dtype=torch.bfloat16) + return o.transpose(0, 1), w.transpose(1, 2), res.transpose(0, 1) + o = torch.randn(G, T, D, device="cuda", generator=g).to(torch.bfloat16) + res = torch.full((G, T, R), float("nan"), device="cuda", dtype=torch.bfloat16) + return o, w.transpose(1, 2), res + + +def _require_swap(): + if not _device_supported(): + pytest.skip( + "requires a device with a registered generated instance " + f"(compute capability {_registered_capabilities()})" + ) + for template in SWAP_TEMPLATES: + _require_program(template) + + +def _swap_launch(A, B, out): + return cake_backend._swap_ab_cached(A, B, out, False) + + +def _swap_expected(A, B): + """The template :func:`swap_ab_template` picks for these views on this device.""" + L, M, _ = A.shape + facts = cake_backend.device_facts(A.device) + return cake_backend.swap_ab_template(L, M, B.shape[2], facts.sm_count, facts.arch) + + +@pytest.mark.parametrize("token_major", [True, False]) +@pytest.mark.parametrize("T", [1, 7, 16, 63, 64, 65, 100, 128, 192, 255, 256]) +def test_swapped_forward_matches_fp64_on_the_grouped_views(T, token_major): + """Every batched forward row of at most 256 tokens runs a swapped program, in place on SGLang's token-major + views or on contiguous ones, within the BF16 budget of the FP64 reference.""" + _require_swap() + A, B, out = _swap_views(T, token_major=token_major) + launch, template = _swap_launch(A, B, out), _swap_expected(A, B) + assert ( + template is not None + ) # the grouped wo_a rows fit two waves of a Blackwell part + assert cake_jit.MODULES[launch.module_name]["template"] == template + cake_backend.dense_projection_det_gemm(A, B, out) + torch.cuda.synchronize() + _check(out, torch.einsum("gtd,gdr->gtr", A.double(), B.double())) + + +@pytest.mark.parametrize( + "L, T, K, N", [(2, 33, 128, 512), (3, 100, 4160, 1032), (16, 8, 576, 256)] +) +def test_swapped_forward_other_shapes(L, T, K, N): + """Other batches, K (a multiple of 64) and N (a multiple of 8, not of the weight tile): tails in both tiles.""" + _require_swap() + A, B, out = _swap_views(T, G=L, D=K, R=N, seed=SEED + 1) + launch = _swap_launch(A, B, out) + assert launch is not None + assert cake_jit.MODULES[launch.module_name]["template"] == _swap_expected(A, B) + cake_backend.dense_projection_det_gemm(A, B, out) + torch.cuda.synchronize() + _check(out, torch.einsum("gtd,gdr->gtr", A.double(), B.double())) + + +def test_swapped_forward_declines_the_rows_it_does_not_take(): + """Rows the swapped programs do not serve stay on the planner: more than 256 tokens, more than two waves of + tiles, one batch, an FP32 output, a transposed store, an MN-major weight view, a K that is not a multiple of + 64.""" + _require_swap() + A, B, out = _swap_views(257) + assert _swap_launch(A, B, out) is None + wide = 128 * (2 * cake_backend.device_sm_count(A.device) // 8 + 1) + A, B, out = _swap_views(64, G=8, D=1024, R=wide) + assert ( + _swap_expected(A, B) is None and _swap_launch(A, B, out) is None + ) # past two waves + A, B, out = _swap_views(64, G=1) + assert _swap_launch(A[0], B[0], out[0]) is None and _swap_launch(A, B, out) is None + A, B, out = _swap_views(64) + assert _swap_launch(A, B, out.float()) is None + assert _swap_launch(A, B.contiguous(), out) is None # B unit stride on n: MN-major + A, B, out = _swap_views(64, D=4096 + 8) + assert _swap_launch(A, B, out) is None + assert cake_backend._swap_ab_launch(*_swap_views(64), True) is None + + +def test_swapped_forward_is_bitwise_repeatable(): + """One FP32 sum per output element, rounded once: reruns give the same bits.""" + _require_swap() + for T in (64, 128): + A, B, out = _swap_views(T, seed=SEED + 2) + cake_backend.dense_projection_det_gemm(A, B, out) + first = out.clone() + for _ in range(REPEATS): + out.fill_(float("nan")) + cake_backend.dense_projection_det_gemm(A, B, out, enable_pdl=True) + torch.cuda.synchronize() + assert torch.equal(out, first) + + +@pytest.mark.parametrize("T", [16, 128]) +def test_swapped_forward_pdl_waits_for_an_early_producer(T): + """A chain of programmatic dependent launches whose producer triggers its dependents at entry: the first + projection (K = 16384, long) writes the activation the second reads. The second starts while the first runs + (its weights load ahead of the wait) and must still read every token the first wrote.""" + _require_swap() + G, D1, R1, R2 = 8, 16384, 1024, 2048 + g = torch.Generator(device="cuda").manual_seed(SEED + 3) + x = torch.randn(T, G, D1, device="cuda", generator=g).to(torch.bfloat16) + w1 = (torch.randn(G, R1, D1, device="cuda", generator=g) * D1**-0.5).to( + torch.bfloat16 + ) + w2 = (torch.randn(G, R2, R1, device="cuda", generator=g) * R1**-0.5).to( + torch.bfloat16 + ) + for _ in range(3): + mid = torch.full((T, G, R1), float("nan"), device="cuda", dtype=torch.bfloat16) + y = torch.full((T, G, R2), float("nan"), device="cuda", dtype=torch.bfloat16) + A1, B1, O1 = x.transpose(0, 1), w1.transpose(1, 2), mid.transpose(0, 1) + A2, B2, O2 = mid.transpose(0, 1), w2.transpose(1, 2), y.transpose(0, 1) + assert _swap_launch(A1, B1, O1) is not None + assert _swap_launch(A2, B2, O2) is not None + cake_backend.dense_projection_det_gemm(A1, B1, O1, enable_pdl=True) + cake_backend.dense_projection_det_gemm(A2, B2, O2, enable_pdl=True) + torch.cuda.synchronize() + assert bool(torch.isfinite(mid).all()) and bool(torch.isfinite(y).all()) + _check(O2, torch.einsum("gtd,gdr->gtr", A2.double(), B2.double())) + + +def test_swapped_forward_replays_in_a_cuda_graph_with_pdl(): + """The swapped launch allocates nothing: a PDL chain captured into a CUDA graph replays on new values written + into the bound views.""" + _require_swap() + A, B, out = _swap_views(96, seed=SEED + 4) + _, B2, out2 = _swap_views(96, R=4096, D=1024, seed=SEED + 5) + A2 = out # the second projection reads the first's output + + def chain(): + cake_backend.dense_projection_det_gemm(A, B, out, enable_pdl=True) + cake_backend.dense_projection_det_gemm(A2, B2, out2, enable_pdl=True) + + chain() + torch.cuda.synchronize() + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + chain() + A.copy_(torch.randn(A.shape, device="cuda").to(torch.bfloat16)) + out.fill_(float("nan")) + out2.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + _check(out, torch.einsum("gtd,gdr->gtr", A.double(), B.double())) + _check(out2, torch.einsum("gtd,gdr->gtr", A2.double(), B2.double())) diff --git a/tests/experimental/test_cake_dense_projection_det_gemm_names.py b/tests/experimental/test_cake_dense_projection_det_gemm_names.py index f48a03c67ed..97c13ef4929 100644 --- a/tests/experimental/test_cake_dense_projection_det_gemm_names.py +++ b/tests/experimental/test_cake_dense_projection_det_gemm_names.py @@ -166,7 +166,12 @@ def test_the_registry_spells_only_canonical_names(): "cake::dense_projection_det_gemm::" ) assert record["template"].startswith( - ("dense_proj_det_gemm_", "router_fp32_det_gemm_", "split_fp32_det_bf16x3") + ( + "dense_proj_det_gemm_", + "router_fp32_det_gemm_", + "split_fp32_det_bf16x3", + "swap_ab_", + ) ) assert ( "build" not in record diff --git a/tests/experimental/test_cake_dense_projection_gemm.py b/tests/experimental/test_cake_dense_projection_gemm.py index 63269b8c4b9..b5ffe48d9ea 100644 --- a/tests/experimental/test_cake_dense_projection_gemm.py +++ b/tests/experimental/test_cake_dense_projection_gemm.py @@ -292,7 +292,11 @@ def test_use_deterministic_backend_binds_a_vendored_copy(monkeypatch): # entry point -> (positional arguments, keyword arguments, the call the deterministic object receives) # fmt: off DEFAULT_CALLS = { - "dense_projection_gemm": (("A", "B", "out"), {"transposed_out": True}, (("A", "B", "out"), {"transposed_out": True})), + "dense_projection_gemm": ( + ("A", "B", "out"), + {"transposed_out": True, "enable_pdl": True}, + (("A", "B", "out"), {"transposed_out": True, "enable_pdl": True}), + ), "plan_dense_projection_gemm": ( ("A", "B", "out"), {"sm_count": 148, "l2_bytes": 1 << 20, "transposed_out": True, "block_n": 128}, @@ -395,10 +399,14 @@ def test_k1_routes(monkeypatch, layout): assert args[0] == "launch" and args[1] is A and args[2] is B and args[3] is out assert kwargs["transposed_out"] is False and kwargs["own"] == {} assert kwargs["det_call"]() is det.result # a row the planner does not take - assert det.calls == [((A, B, out), {"transposed_out": False})] + assert det.calls == [ + ((A, B, out), {"transposed_out": False, "enable_pdl": False}) + ] else: assert result is det.result and not atomic.calls - assert det.calls == [((A, B, out), {"transposed_out": False})] + assert det.calls == [ + ((A, B, out), {"transposed_out": False, "enable_pdl": False}) + ] def test_k1_operation_routes(monkeypatch):