From db076112ed7fde1a25180c5a44b194e45dd02eaf Mon Sep 17 00:00:00 2001 From: Yingyi Huang Date: Tue, 27 Oct 2026 02:14:20 -0700 Subject: [PATCH] perf(cake_gated_act_mxfp8): register-fitted band-path and packed-sigmoid programs in eight M * K ranges (SM100 / SM103) Eight per-architecture M * K ranges (sm_100a 5, sm_103a 3) now select a program whose register budget is fitted to a CTA count per SM. Below M * K = 2^20, backward rowwise and backward both take a 32 x 32 tile band-path program on sm_100a and sm_103a. On sm_100a, forward colwise, backward colwise and backward both split at 2^27; below the split they take a band-path program from 2^26 (forward) or 2^24 (backward). On sm_103a, backward rowwise L1::no_allocate from 2^26 moves from the 12-CTA build, which spills, to the packed-sigmoid build at 10 CTAs. Outputs are byte-identical; the public API is unchanged. The package goes from 45 to 50 programs (6 new, 1 removed); sm_107a programs are unchanged. Co-Authored-By: Claude --- .../cake_gated_act_mxfp8_backend/build.json | 287 +- .../cake_gated_act_mxfp8_backend/cake_jit.py | 340 +- ..._act_mxfp8_0b9dfb9aa4b837a6b7a4_binding.cu | 93 + ...d_act_mxfp8_0b9dfb9aa4b837a6b7a4_kernel.cu | 2215 ++++++++++++ ..._act_mxfp8_383546f101950463dcf8_binding.cu | 98 + ...d_act_mxfp8_383546f101950463dcf8_kernel.cu | 1057 ++++++ ...act_mxfp8_3e0ac746df31b17317e0_binding.cu} | 14 +- ..._act_mxfp8_3e0ac746df31b17317e0_kernel.cu} | 854 +++-- ..._act_mxfp8_44395eef5de97bba1695_binding.cu | 98 + ...d_act_mxfp8_44395eef5de97bba1695_kernel.cu | 2989 +++++++++++++++++ ..._act_mxfp8_54573bbcd9b4aadbd39f_binding.cu | 98 + ...d_act_mxfp8_54573bbcd9b4aadbd39f_kernel.cu | 891 +++++ ..._act_mxfp8_9c68fc985d41586ecbe8_binding.cu | 98 + ...d_act_mxfp8_9c68fc985d41586ecbe8_kernel.cu | 798 +++++ 14 files changed, 9329 insertions(+), 601 deletions(-) create mode 100644 flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_binding.cu create mode 100644 flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_kernel.cu create mode 100644 flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_383546f101950463dcf8_binding.cu create mode 100644 flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_383546f101950463dcf8_kernel.cu rename flashinfer/cake_gated_act_mxfp8_backend/csrc/{cake_gated_act_mxfp8_077a568b810590574917_binding.cu => cake_gated_act_mxfp8_3e0ac746df31b17317e0_binding.cu} (91%) rename flashinfer/cake_gated_act_mxfp8_backend/csrc/{cake_gated_act_mxfp8_077a568b810590574917_kernel.cu => cake_gated_act_mxfp8_3e0ac746df31b17317e0_kernel.cu} (65%) create mode 100644 flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_binding.cu create mode 100644 flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_kernel.cu create mode 100644 flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_binding.cu create mode 100644 flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_kernel.cu create mode 100644 flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_binding.cu create mode 100644 flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_kernel.cu diff --git a/flashinfer/cake_gated_act_mxfp8_backend/build.json b/flashinfer/cake_gated_act_mxfp8_backend/build.json index edc71afd015..430be3f87ba 100644 --- a/flashinfer/cake_gated_act_mxfp8_backend/build.json +++ b/flashinfer/cake_gated_act_mxfp8_backend/build.json @@ -130,49 +130,49 @@ ] }, { - "arch": "sm_103a", - "closure_sha256": "55a1974ce2ca50bc9f49fbaf40fec355ada1788818d4b6c37b5791a30f035d2f", + "arch": "sm_100a", + "closure_sha256": "61246e27b73a04d4d6def58e0261c49fd69219ddcf01a6bff5886220838ecebc", "entry": "run", - "gencode": "-gencode=arch=compute_103a,code=sm_103a", - "min_cuda": "12.9", - "name": "cake_gated_act_mxfp8_077a568b810590574917_sm_103a", + "gencode": "-gencode=arch=compute_100a,code=sm_100a", + "min_cuda": "12.8", + "name": "cake_gated_act_mxfp8_07f12e4d85523cdf2557_sm_100a", "nvcc_flags": [ "--prec-div=true", "--prec-sqrt=true", "--ftz=false" ], "nvcc_flags_if": [], - "path": "aot/cake_gated_act_mxfp8_077a568b810590574917_sm_103a.so", + "path": "aot/cake_gated_act_mxfp8_07f12e4d85523cdf2557_sm_100a.so", "sha256": { - "csrc/cake_gated_act_mxfp8_077a568b810590574917_binding.cu": "631064e06912ea1a05b3382cc7be41a68fb14e5275fa70172f1a6394d0ead0eb", - "csrc/cake_gated_act_mxfp8_077a568b810590574917_kernel.cu": "84ac48b28ba1bd828e7ff9797872e7fbb93938cad6c91afc564ad056df86265b" + "csrc/cake_gated_act_mxfp8_07f12e4d85523cdf2557_binding.cu": "64d90c79530380a946c4b201b47513c36ccd8ea5c400eedd5a25428346ef993b", + "csrc/cake_gated_act_mxfp8_07f12e4d85523cdf2557_kernel.cu": "00a33d70a078cf3ddb75dbb3a054b07e67c7aa82eb988f8123e8bb8320b27bed" }, "sources": [ - "csrc/cake_gated_act_mxfp8_077a568b810590574917_kernel.cu", - "csrc/cake_gated_act_mxfp8_077a568b810590574917_binding.cu" + "csrc/cake_gated_act_mxfp8_07f12e4d85523cdf2557_kernel.cu", + "csrc/cake_gated_act_mxfp8_07f12e4d85523cdf2557_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "61246e27b73a04d4d6def58e0261c49fd69219ddcf01a6bff5886220838ecebc", + "closure_sha256": "319423a87d1477a2efd36b41c2983a9dc1d2607d05a46e8b60c76d0d27d7999f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gated_act_mxfp8_07f12e4d85523cdf2557_sm_100a", + "name": "cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_sm_100a", "nvcc_flags": [ "--prec-div=true", "--prec-sqrt=true", "--ftz=false" ], "nvcc_flags_if": [], - "path": "aot/cake_gated_act_mxfp8_07f12e4d85523cdf2557_sm_100a.so", + "path": "aot/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_sm_100a.so", "sha256": { - "csrc/cake_gated_act_mxfp8_07f12e4d85523cdf2557_binding.cu": "64d90c79530380a946c4b201b47513c36ccd8ea5c400eedd5a25428346ef993b", - "csrc/cake_gated_act_mxfp8_07f12e4d85523cdf2557_kernel.cu": "00a33d70a078cf3ddb75dbb3a054b07e67c7aa82eb988f8123e8bb8320b27bed" + "csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_binding.cu": "59e046b79c459c9d9cd8610fb5feea431334625d1051e297344441d46da32a03", + "csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_kernel.cu": "aaa7fba663ab1b70e90b90ecf5baa748ebb2a5e629bae21a7ba70abd339541bd" }, "sources": [ - "csrc/cake_gated_act_mxfp8_07f12e4d85523cdf2557_kernel.cu", - "csrc/cake_gated_act_mxfp8_07f12e4d85523cdf2557_binding.cu" + "csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_kernel.cu", + "csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_binding.cu" ] }, { @@ -336,6 +336,52 @@ "csrc/cake_gated_act_mxfp8_363476aea39117786ffc_binding.cu" ] }, + { + "arch": "sm_100a", + "closure_sha256": "ff71091db59a80281afd89a74f9bbdefc203a799e3f806671546bcb8db8d7c06", + "entry": "run", + "gencode": "-gencode=arch=compute_100a,code=sm_100a", + "min_cuda": "12.8", + "name": "cake_gated_act_mxfp8_383546f101950463dcf8_sm_100a", + "nvcc_flags": [ + "--prec-div=true", + "--prec-sqrt=true", + "--ftz=false" + ], + "nvcc_flags_if": [], + "path": "aot/cake_gated_act_mxfp8_383546f101950463dcf8_sm_100a.so", + "sha256": { + "csrc/cake_gated_act_mxfp8_383546f101950463dcf8_binding.cu": "594a684243aa2701382f6bd9ec774431c1a4f261887e3068fa429789bcb93754", + "csrc/cake_gated_act_mxfp8_383546f101950463dcf8_kernel.cu": "21ca01549097774d9963a056db8bd9e951d160dc49aa69a411fcc70e32c83acc" + }, + "sources": [ + "csrc/cake_gated_act_mxfp8_383546f101950463dcf8_kernel.cu", + "csrc/cake_gated_act_mxfp8_383546f101950463dcf8_binding.cu" + ] + }, + { + "arch": "sm_103a", + "closure_sha256": "a5dd7f16e824946e7c56631bf0a3f5f0959c950085a35328a1d6fd6ca26f6958", + "entry": "run", + "gencode": "-gencode=arch=compute_103a,code=sm_103a", + "min_cuda": "12.9", + "name": "cake_gated_act_mxfp8_383546f101950463dcf8_sm_103a", + "nvcc_flags": [ + "--prec-div=true", + "--prec-sqrt=true", + "--ftz=false" + ], + "nvcc_flags_if": [], + "path": "aot/cake_gated_act_mxfp8_383546f101950463dcf8_sm_103a.so", + "sha256": { + "csrc/cake_gated_act_mxfp8_383546f101950463dcf8_binding.cu": "594a684243aa2701382f6bd9ec774431c1a4f261887e3068fa429789bcb93754", + "csrc/cake_gated_act_mxfp8_383546f101950463dcf8_kernel.cu": "21ca01549097774d9963a056db8bd9e951d160dc49aa69a411fcc70e32c83acc" + }, + "sources": [ + "csrc/cake_gated_act_mxfp8_383546f101950463dcf8_kernel.cu", + "csrc/cake_gated_act_mxfp8_383546f101950463dcf8_binding.cu" + ] + }, { "arch": "sm_107a", "closure_sha256": "e2b37b35130860face7d3f21fb525605fec67f0276ce7aaec6473eadbc0d25db", @@ -359,6 +405,29 @@ "csrc/cake_gated_act_mxfp8_3b4157aba492acf411db_binding.cu" ] }, + { + "arch": "sm_103a", + "closure_sha256": "6ec26c785650a555e3d83b92663e34083e039bb38bccdb3b697f20b3a9d4a0f1", + "entry": "run", + "gencode": "-gencode=arch=compute_103a,code=sm_103a", + "min_cuda": "12.9", + "name": "cake_gated_act_mxfp8_3e0ac746df31b17317e0_sm_103a", + "nvcc_flags": [ + "--prec-div=true", + "--prec-sqrt=true", + "--ftz=false" + ], + "nvcc_flags_if": [], + "path": "aot/cake_gated_act_mxfp8_3e0ac746df31b17317e0_sm_103a.so", + "sha256": { + "csrc/cake_gated_act_mxfp8_3e0ac746df31b17317e0_binding.cu": "b61d94c564242f045baa2a42877c96f817619e535b5e12d5c68a373bb3b8358c", + "csrc/cake_gated_act_mxfp8_3e0ac746df31b17317e0_kernel.cu": "d2274be3de2f6935e3063e5f48ad15ccaac6987df935e983f1ca50a6f66ec8ff" + }, + "sources": [ + "csrc/cake_gated_act_mxfp8_3e0ac746df31b17317e0_kernel.cu", + "csrc/cake_gated_act_mxfp8_3e0ac746df31b17317e0_binding.cu" + ] + }, { "arch": "sm_107a", "closure_sha256": "027bffc16db73c143f80bc2dbda27e2f42b3793b11990b32759ea6cb8ec4efac", @@ -382,6 +451,29 @@ "csrc/cake_gated_act_mxfp8_43b7d4d5cc26486e8036_binding.cu" ] }, + { + "arch": "sm_100a", + "closure_sha256": "1f0457199e8f5b90f4dcde3bcf4bd8d3f64211a98db3769ac59ba260172d06ab", + "entry": "run", + "gencode": "-gencode=arch=compute_100a,code=sm_100a", + "min_cuda": "12.8", + "name": "cake_gated_act_mxfp8_44395eef5de97bba1695_sm_100a", + "nvcc_flags": [ + "--prec-div=true", + "--prec-sqrt=true", + "--ftz=false" + ], + "nvcc_flags_if": [], + "path": "aot/cake_gated_act_mxfp8_44395eef5de97bba1695_sm_100a.so", + "sha256": { + "csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_binding.cu": "a84c471e41afd66b759f6ea5798c8c89f8e5146b30f1ad68bb00fdc8247253ea", + "csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_kernel.cu": "2414cd199968a70d5c832081c080850f333d551b36276a60b82d1c764d558c5c" + }, + "sources": [ + "csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_kernel.cu", + "csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_binding.cu" + ] + }, { "arch": "sm_107a", "closure_sha256": "a585c8405414c38793a78a1063ef47948130bce4468514c25178b205300c54cc", @@ -497,6 +589,29 @@ "csrc/cake_gated_act_mxfp8_52421da70218abf13caf_binding.cu" ] }, + { + "arch": "sm_103a", + "closure_sha256": "4488c9e1f9baef1905157de639c3e9f446d8df1d459964287c464cc74685387d", + "entry": "run", + "gencode": "-gencode=arch=compute_103a,code=sm_103a", + "min_cuda": "12.9", + "name": "cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_sm_103a", + "nvcc_flags": [ + "--prec-div=true", + "--prec-sqrt=true", + "--ftz=false" + ], + "nvcc_flags_if": [], + "path": "aot/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_sm_103a.so", + "sha256": { + "csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_binding.cu": "1b848c621b210c8adec067f37fa534e680d68f86c855a40521be399c11dcac61", + "csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_kernel.cu": "7e79a5710ffe16f594ecc54881997be7f6f04b725d125ca696cd600dd10c2a29" + }, + "sources": [ + "csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_kernel.cu", + "csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_binding.cu" + ] + }, { "arch": "sm_100a", "closure_sha256": "da88ab1594152a00948e0ea90f112b50c6241523ff4935dc0ca130378f321f4f", @@ -681,6 +796,29 @@ "csrc/cake_gated_act_mxfp8_6d1956b2f454b08e6d34_binding.cu" ] }, + { + "arch": "sm_100a", + "closure_sha256": "59bf523125a046b183cbb4c1628d9ec912d88416e3a9654b5fbf033a68725b60", + "entry": "run", + "gencode": "-gencode=arch=compute_100a,code=sm_100a", + "min_cuda": "12.8", + "name": "cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd_sm_100a", + "nvcc_flags": [ + "--prec-div=true", + "--prec-sqrt=true", + "--ftz=false" + ], + "nvcc_flags_if": [], + "path": "aot/cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd_sm_100a.so", + "sha256": { + "csrc/cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd_binding.cu": "96eb5f01b4be297772bb7e942abddbec6f038a8d8983d1d1457cd7304dc592aa", + "csrc/cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd_kernel.cu": "abd2d38fa9f9040ce0b04a969aa8eb7b2e8a87623f58621dfbb7dc112c7c1d16" + }, + "sources": [ + "csrc/cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd_kernel.cu", + "csrc/cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd_binding.cu" + ] + }, { "arch": "sm_103a", "closure_sha256": "db865419412826d7ca0fe3d62b081d869bf6732a43b04826f16f710aca34eec1", @@ -934,52 +1072,6 @@ "csrc/cake_gated_act_mxfp8_8511a80499e75655d79e_binding.cu" ] }, - { - "arch": "sm_100a", - "closure_sha256": "0764c64bd9ceae4970ef36cf3c6d7381d117133b4062e6eac6b6b2fff0b76ac8", - "entry": "run", - "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", - "name": "cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_sm_100a", - "nvcc_flags": [ - "--prec-div=true", - "--prec-sqrt=true", - "--ftz=false" - ], - "nvcc_flags_if": [], - "path": "aot/cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_sm_100a.so", - "sha256": { - "csrc/cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_binding.cu": "e03989d29a58ec88ac8f373aca760e0ab4d358e7dc77ff85f1cb6d7e3ed4de9e", - "csrc/cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_kernel.cu": "efa5313e0da323c51ff539e2d6dcd8f5a18609376a53070763d78d3ab13515dc" - }, - "sources": [ - "csrc/cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_kernel.cu", - "csrc/cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_binding.cu" - ] - }, - { - "arch": "sm_103a", - "closure_sha256": "015db7e2628839b30b947818ccaa04e3a7486137ada60af847a9ee2a83b94ebd", - "entry": "run", - "gencode": "-gencode=arch=compute_103a,code=sm_103a", - "min_cuda": "12.9", - "name": "cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_sm_103a", - "nvcc_flags": [ - "--prec-div=true", - "--prec-sqrt=true", - "--ftz=false" - ], - "nvcc_flags_if": [], - "path": "aot/cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_sm_103a.so", - "sha256": { - "csrc/cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_binding.cu": "e03989d29a58ec88ac8f373aca760e0ab4d358e7dc77ff85f1cb6d7e3ed4de9e", - "csrc/cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_kernel.cu": "efa5313e0da323c51ff539e2d6dcd8f5a18609376a53070763d78d3ab13515dc" - }, - "sources": [ - "csrc/cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_kernel.cu", - "csrc/cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_binding.cu" - ] - }, { "arch": "sm_107a", "closure_sha256": "5c49e1ce7d0ec65cc171a462062439233ea40e1a7983d1317ba483322e95df66", @@ -1003,52 +1095,6 @@ "csrc/cake_gated_act_mxfp8_8c43730c1e3d76ce89f1_binding.cu" ] }, - { - "arch": "sm_100a", - "closure_sha256": "d565588653ad500ee56dd8a86d2eec9327cc89ddab40bdb834dbe7bab3dbdfc7", - "entry": "run", - "gencode": "-gencode=arch=compute_100a,code=sm_100a", - "min_cuda": "12.8", - "name": "cake_gated_act_mxfp8_8da8bc14e105ce409f74_sm_100a", - "nvcc_flags": [ - "--prec-div=true", - "--prec-sqrt=true", - "--ftz=false" - ], - "nvcc_flags_if": [], - "path": "aot/cake_gated_act_mxfp8_8da8bc14e105ce409f74_sm_100a.so", - "sha256": { - "csrc/cake_gated_act_mxfp8_8da8bc14e105ce409f74_binding.cu": "b04c32d80fb901bc8b4df084014a06bebe7d9dcf133c896a4aedc7a6a1606ad8", - "csrc/cake_gated_act_mxfp8_8da8bc14e105ce409f74_kernel.cu": "01c4c2714594582664e15ebbdb6879c6c2de4c2ca5031425b406e7d0143abace" - }, - "sources": [ - "csrc/cake_gated_act_mxfp8_8da8bc14e105ce409f74_kernel.cu", - "csrc/cake_gated_act_mxfp8_8da8bc14e105ce409f74_binding.cu" - ] - }, - { - "arch": "sm_103a", - "closure_sha256": "751fae2754aaed39a6b45bcae0a0920d1ff89d2b1ecd9576f94d55448456a01d", - "entry": "run", - "gencode": "-gencode=arch=compute_103a,code=sm_103a", - "min_cuda": "12.9", - "name": "cake_gated_act_mxfp8_8da8bc14e105ce409f74_sm_103a", - "nvcc_flags": [ - "--prec-div=true", - "--prec-sqrt=true", - "--ftz=false" - ], - "nvcc_flags_if": [], - "path": "aot/cake_gated_act_mxfp8_8da8bc14e105ce409f74_sm_103a.so", - "sha256": { - "csrc/cake_gated_act_mxfp8_8da8bc14e105ce409f74_binding.cu": "b04c32d80fb901bc8b4df084014a06bebe7d9dcf133c896a4aedc7a6a1606ad8", - "csrc/cake_gated_act_mxfp8_8da8bc14e105ce409f74_kernel.cu": "01c4c2714594582664e15ebbdb6879c6c2de4c2ca5031425b406e7d0143abace" - }, - "sources": [ - "csrc/cake_gated_act_mxfp8_8da8bc14e105ce409f74_kernel.cu", - "csrc/cake_gated_act_mxfp8_8da8bc14e105ce409f74_binding.cu" - ] - }, { "arch": "sm_107a", "closure_sha256": "886ced9f6c174a91ea1716174e4b76277fafe41edadc62814b409b30b0cff216", @@ -1141,6 +1187,29 @@ "csrc/cake_gated_act_mxfp8_96497ddc9daf22dc35a5_binding.cu" ] }, + { + "arch": "sm_100a", + "closure_sha256": "b232bb9fde3c9c61e3e8f27bb407df83d50cf44d5e1c7729d2e05ea8f58015a9", + "entry": "run", + "gencode": "-gencode=arch=compute_100a,code=sm_100a", + "min_cuda": "12.8", + "name": "cake_gated_act_mxfp8_9c68fc985d41586ecbe8_sm_100a", + "nvcc_flags": [ + "--prec-div=true", + "--prec-sqrt=true", + "--ftz=false" + ], + "nvcc_flags_if": [], + "path": "aot/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_sm_100a.so", + "sha256": { + "csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_binding.cu": "de2b5189d689e7a3123b85455ce13c971d6fb43e78ca2ea0592da11d17504408", + "csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_kernel.cu": "d1eb1255a7a52fb78d119f0ecd222a2efe565da8074d67a47b8d93bfd838668d" + }, + "sources": [ + "csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_kernel.cu", + "csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_binding.cu" + ] + }, { "arch": "sm_103a", "closure_sha256": "26fe31a8e76697e0500c631d2eea00856dc5926acc0de0b0aca357c068670c6d", diff --git a/flashinfer/cake_gated_act_mxfp8_backend/cake_jit.py b/flashinfer/cake_gated_act_mxfp8_backend/cake_jit.py index 70cb2eb8178..45f157335dc 100644 --- a/flashinfer/cake_gated_act_mxfp8_backend/cake_jit.py +++ b/flashinfer/cake_gated_act_mxfp8_backend/cake_jit.py @@ -634,8 +634,61 @@ "kind": "elements_at_least", }, }, + # The forward, colwise, with two rows per thread, no_allocate loads, 5 CTAs + # per SM and the inline band path (on sm_100a only): it runs when M * K >= + # 2^26 and no record of a higher threshold admits the call. + "cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4": { + "arches": [ + "sm_100a", + ], + "arg_plan": [ + ["buffer", "gated_input"], + ["buffer", "row_output"], + ["buffer", "col_output"], + ["buffer", "row_scales"], + ["buffer", "col_scales"], + ["parameter", "M"], + ["parameter", "K"], + ["grid", "grid_x"], + ["grid", "grid_y"], + ["grid", "grid_z"], + ], + "ffi_entry": "run", + "grid_rule": { + "kind": "tiles", + "tile_cols": 128, + "tile_rows": 64, + }, + "kernel": "cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4", + "kernel_symbol": "kernel_cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4", + "launch": { + "block": (256, 1, 1), + "cluster": (1, 1, 1), + "cooperative": False, + "dynamic_smem_bytes": 16384, + "use_pdl": False, + }, + "library": "cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4", + "logical": { + "K": "K", + "M": "M", + "col_output": "col_output", + "col_scales": "col_scales", + "gated_input": "gated_input", + "row_output": "row_output", + "row_scales": "row_scales", + }, + "role": "fwd", + "route": "col", + "select_rule": { + "elements": { + "sm_100a": 67108864, + }, + "kind": "elements_at_least", + }, + }, # The forward, colwise, with two rows per thread and no_allocate loads (on - # sm_100a only): it runs when M * K >= 2^26 and no record of a higher + # sm_100a only): it runs when M * K >= 2^27 and no record of a higher # threshold admits the call. "cake_gated_act_mxfp8_d4edc57ff03b5fc922ab": { "arches": [ @@ -682,7 +735,7 @@ "route": "col", "select_rule": { "elements": { - "sm_100a": 67108864, + "sm_100a": 134217728, }, "kind": "elements_at_least", }, @@ -1427,12 +1480,108 @@ "kind": "elements_at_least", }, }, - # The backward, rowwise, with a 32 x 32 tile: the route's fallback (no - # select_rule). - "cake_gated_act_mxfp8_8da8bc14e105ce409f74": { + # The backward, rowwise, with a 32 x 32 tile, 12 CTAs per SM and the called + # band path: the route's fallback on sm_100a (no select_rule). + "cake_gated_act_mxfp8_9c68fc985d41586ecbe8": { "arches": [ "sm_100a", + ], + "arg_plan": [ + ["buffer", "gated_input"], + ["buffer", "grad"], + ["buffer", "row_output"], + ["buffer", "col_output"], + ["buffer", "row_scales"], + ["buffer", "col_scales"], + ["parameter", "M"], + ["parameter", "K"], + ["grid", "grid_x"], + ["grid", "grid_y"], + ["grid", "grid_z"], + ], + "ffi_entry": "run", + "grid_rule": { + "kind": "tiles", + "tile_cols": 32, + "tile_rows": 32, + }, + "kernel": "cake_gated_act_mxfp8_9c68fc985d41586ecbe8", + "kernel_symbol": "kernel_cake_gated_act_mxfp8_9c68fc985d41586ecbe8", + "launch": { + "block": (128, 1, 1), + "cluster": (1, 1, 1), + "cooperative": False, + "dynamic_smem_bytes": 0, + "use_pdl": False, + }, + "library": "cake_gated_act_mxfp8_9c68fc985d41586ecbe8", + "logical": { + "K": "K", + "M": "M", + "col_output": "col_output", + "col_scales": "col_scales", + "gated_input": "gated_input", + "grad": "grad_output", + "row_output": "row_output", + "row_scales": "row_scales", + }, + "role": "bwd", + "route": "row", + "select_rule": None, + }, + # The backward, rowwise, with a 32 x 32 tile, 12 CTAs per SM and the inline + # band path: the route's fallback on sm_103a (no select_rule). + "cake_gated_act_mxfp8_54573bbcd9b4aadbd39f": { + "arches": [ "sm_103a", + ], + "arg_plan": [ + ["buffer", "gated_input"], + ["buffer", "grad"], + ["buffer", "row_output"], + ["buffer", "col_output"], + ["buffer", "row_scales"], + ["buffer", "col_scales"], + ["parameter", "M"], + ["parameter", "K"], + ["grid", "grid_x"], + ["grid", "grid_y"], + ["grid", "grid_z"], + ], + "ffi_entry": "run", + "grid_rule": { + "kind": "tiles", + "tile_cols": 32, + "tile_rows": 32, + }, + "kernel": "cake_gated_act_mxfp8_54573bbcd9b4aadbd39f", + "kernel_symbol": "kernel_cake_gated_act_mxfp8_54573bbcd9b4aadbd39f", + "launch": { + "block": (128, 1, 1), + "cluster": (1, 1, 1), + "cooperative": False, + "dynamic_smem_bytes": 0, + "use_pdl": False, + }, + "library": "cake_gated_act_mxfp8_54573bbcd9b4aadbd39f", + "logical": { + "K": "K", + "M": "M", + "col_output": "col_output", + "col_scales": "col_scales", + "gated_input": "gated_input", + "grad": "grad_output", + "row_output": "row_output", + "row_scales": "row_scales", + }, + "role": "bwd", + "route": "row", + "select_rule": None, + }, + # The backward, rowwise, with a 32 x 32 tile: the route's fallback on + # sm_107a (no select_rule). + "cake_gated_act_mxfp8_8da8bc14e105ce409f74": { + "arches": [ "sm_107a", ], "arg_plan": [ @@ -1700,10 +1849,10 @@ "kind": "elements_at_least", }, }, - # The backward, rowwise, with no_allocate loads and 12 CTAs per SM (on - # sm_103a only): it runs when M * K >= 2^26 and no record of a higher - # threshold admits the call. - "cake_gated_act_mxfp8_077a568b810590574917": { + # The backward, rowwise, with no_allocate loads, 10 CTAs per SM and the + # packed sigmoid (on sm_103a only): it runs when M * K >= 2^26 and no + # record of a higher threshold admits the call. + "cake_gated_act_mxfp8_3e0ac746df31b17317e0": { "arches": [ "sm_103a", ], @@ -1726,8 +1875,8 @@ "tile_cols": 64, "tile_rows": 32, }, - "kernel": "cake_gated_act_mxfp8_077a568b810590574917", - "kernel_symbol": "kernel_cake_gated_act_mxfp8_077a568b810590574917", + "kernel": "cake_gated_act_mxfp8_3e0ac746df31b17317e0", + "kernel_symbol": "kernel_cake_gated_act_mxfp8_3e0ac746df31b17317e0", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -1735,7 +1884,7 @@ "dynamic_smem_bytes": 0, "use_pdl": False, }, - "library": "cake_gated_act_mxfp8_077a568b810590574917", + "library": "cake_gated_act_mxfp8_3e0ac746df31b17317e0", "logical": { "K": "K", "M": "M", @@ -1920,12 +2069,14 @@ "kind": "elements_at_least", }, }, - # The backward, colwise, with two rows per thread, no_allocate loads and 6 - # CTAs per SM (on sm_100a only): it runs when M * K >= 2^24 and no record - # of a higher threshold admits the call. - "cake_gated_act_mxfp8_8511a80499e75655d79e": { + # The backward, colwise, with two rows per thread, no_allocate loads, 6 + # CTAs per SM, the packed sigmoid and the called band path: it runs when M + # * K >= 2^24 and no record of a higher threshold admits the call. + "cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd": { "arches": [ "sm_100a", + "sm_103a", + "sm_107a", ], "arg_plan": [ ["buffer", "gated_input"], @@ -1946,8 +2097,8 @@ "tile_cols": 64, "tile_rows": 64, }, - "kernel": "cake_gated_act_mxfp8_8511a80499e75655d79e", - "kernel_symbol": "kernel_cake_gated_act_mxfp8_8511a80499e75655d79e", + "kernel": "cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd", + "kernel_symbol": "kernel_cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -1955,7 +2106,7 @@ "dynamic_smem_bytes": 16384, "use_pdl": False, }, - "library": "cake_gated_act_mxfp8_8511a80499e75655d79e", + "library": "cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd", "logical": { "K": "K", "M": "M", @@ -1971,18 +2122,18 @@ "select_rule": { "elements": { "sm_100a": 16777216, + "sm_103a": 16777216, + "sm_107a": 16777216, }, "kind": "elements_at_least", }, }, - # The backward, colwise, with two rows per thread, no_allocate loads, 6 - # CTAs per SM, the packed sigmoid and the called band path (on sm_103a and - # sm_107a only): it runs when M * K >= 2^24 and no record of a higher - # threshold admits the call. - "cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd": { + # The backward, colwise, with two rows per thread, no_allocate loads and 6 + # CTAs per SM (on sm_100a only): it runs when M * K >= 2^27 and no record + # of a higher threshold admits the call. + "cake_gated_act_mxfp8_8511a80499e75655d79e": { "arches": [ - "sm_103a", - "sm_107a", + "sm_100a", ], "arg_plan": [ ["buffer", "gated_input"], @@ -2003,8 +2154,8 @@ "tile_cols": 64, "tile_rows": 64, }, - "kernel": "cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd", - "kernel_symbol": "kernel_cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd", + "kernel": "cake_gated_act_mxfp8_8511a80499e75655d79e", + "kernel_symbol": "kernel_cake_gated_act_mxfp8_8511a80499e75655d79e", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -2012,7 +2163,7 @@ "dynamic_smem_bytes": 16384, "use_pdl": False, }, - "library": "cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd", + "library": "cake_gated_act_mxfp8_8511a80499e75655d79e", "logical": { "K": "K", "M": "M", @@ -2027,8 +2178,7 @@ "route": "col", "select_rule": { "elements": { - "sm_103a": 16777216, - "sm_107a": 16777216, + "sm_100a": 134217728, }, "kind": "elements_at_least", }, @@ -2088,12 +2238,61 @@ "kind": "elements_at_least", }, }, - # The backward, rowwise and colwise, with a 32 x 32 tile: the route's - # fallback (no select_rule). - "cake_gated_act_mxfp8_8c43730c1e3d76ce89f1": { + # The backward, rowwise and colwise, with a 32 x 32 tile, 12 CTAs per SM + # and the ffs band path: the route's fallback on sm_100a and sm_103a (no + # select_rule). + "cake_gated_act_mxfp8_383546f101950463dcf8": { "arches": [ "sm_100a", "sm_103a", + ], + "arg_plan": [ + ["buffer", "gated_input"], + ["buffer", "grad"], + ["buffer", "row_output"], + ["buffer", "col_output"], + ["buffer", "row_scales"], + ["buffer", "col_scales"], + ["parameter", "M"], + ["parameter", "K"], + ["grid", "grid_x"], + ["grid", "grid_y"], + ["grid", "grid_z"], + ], + "ffi_entry": "run", + "grid_rule": { + "kind": "tiles", + "tile_cols": 32, + "tile_rows": 32, + }, + "kernel": "cake_gated_act_mxfp8_383546f101950463dcf8", + "kernel_symbol": "kernel_cake_gated_act_mxfp8_383546f101950463dcf8", + "launch": { + "block": (128, 1, 1), + "cluster": (1, 1, 1), + "cooperative": False, + "dynamic_smem_bytes": 4096, + "use_pdl": False, + }, + "library": "cake_gated_act_mxfp8_383546f101950463dcf8", + "logical": { + "K": "K", + "M": "M", + "col_output": "col_output", + "col_scales": "col_scales", + "gated_input": "gated_input", + "grad": "grad_output", + "row_output": "row_output", + "row_scales": "row_scales", + }, + "role": "bwd", + "route": "both", + "select_rule": None, + }, + # The backward, rowwise and colwise, with a 32 x 32 tile: the route's + # fallback on sm_107a (no select_rule). + "cake_gated_act_mxfp8_8c43730c1e3d76ce89f1": { + "arches": [ "sm_107a", ], "arg_plan": [ @@ -2252,8 +2451,64 @@ "kind": "elements_at_least", }, }, + # The backward, rowwise and colwise, with two rows per thread, no_allocate + # loads, 7 CTAs per SM, the packed sigmoid and the called band path (on + # sm_100a only): it runs when M * K >= 2^24 and no record of a higher + # threshold admits the call. + "cake_gated_act_mxfp8_44395eef5de97bba1695": { + "arches": [ + "sm_100a", + ], + "arg_plan": [ + ["buffer", "gated_input"], + ["buffer", "grad"], + ["buffer", "row_output"], + ["buffer", "col_output"], + ["buffer", "row_scales"], + ["buffer", "col_scales"], + ["parameter", "M"], + ["parameter", "K"], + ["grid", "grid_x"], + ["grid", "grid_y"], + ["grid", "grid_z"], + ], + "ffi_entry": "run", + "grid_rule": { + "kind": "tiles", + "tile_cols": 64, + "tile_rows": 64, + }, + "kernel": "cake_gated_act_mxfp8_44395eef5de97bba1695", + "kernel_symbol": "kernel_cake_gated_act_mxfp8_44395eef5de97bba1695", + "launch": { + "block": (128, 1, 1), + "cluster": (1, 1, 1), + "cooperative": False, + "dynamic_smem_bytes": 16384, + "use_pdl": False, + }, + "library": "cake_gated_act_mxfp8_44395eef5de97bba1695", + "logical": { + "K": "K", + "M": "M", + "col_output": "col_output", + "col_scales": "col_scales", + "gated_input": "gated_input", + "grad": "grad_output", + "row_output": "row_output", + "row_scales": "row_scales", + }, + "role": "bwd", + "route": "both", + "select_rule": { + "elements": { + "sm_100a": 16777216, + }, + "kind": "elements_at_least", + }, + }, # The backward, rowwise and colwise, with two rows per thread and - # no_allocate loads (on sm_100a only): it runs when M * K >= 2^24 and no + # no_allocate loads (on sm_100a only): it runs when M * K >= 2^27 and no # record of a higher threshold admits the call. "cake_gated_act_mxfp8_e96f02e396eaf24e2caa": { "arches": [ @@ -2302,7 +2557,7 @@ "route": "both", "select_rule": { "elements": { - "sm_100a": 16777216, + "sm_100a": 134217728, }, "kind": "elements_at_least", }, @@ -2486,6 +2741,7 @@ "cake_gated_act_mxfp8_7e191474abd37ba82e73", "cake_gated_act_mxfp8_ff6e1cf61c61f4b595fa", "cake_gated_act_mxfp8_07f12e4d85523cdf2557", + "cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4", "cake_gated_act_mxfp8_d4edc57ff03b5fc922ab", ], "fwd_both": [ @@ -2494,7 +2750,7 @@ "cake_gated_act_mxfp8_c2957071a449135dded9", ], "bwd_row": [ - "cake_gated_act_mxfp8_8da8bc14e105ce409f74", + "cake_gated_act_mxfp8_9c68fc985d41586ecbe8", "cake_gated_act_mxfp8_5bf054fcafaa9784cea3", "cake_gated_act_mxfp8_0e06658456a152d0b214", "cake_gated_act_mxfp8_ea5e588e23b6923737f2", @@ -2502,12 +2758,14 @@ "bwd_col": [ "cake_gated_act_mxfp8_2513ecc12f03a6081c2e", "cake_gated_act_mxfp8_848a93885f596fa4cbf9", + "cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd", "cake_gated_act_mxfp8_8511a80499e75655d79e", ], "bwd_both": [ - "cake_gated_act_mxfp8_8c43730c1e3d76ce89f1", + "cake_gated_act_mxfp8_383546f101950463dcf8", "cake_gated_act_mxfp8_6d1956b2f454b08e6d34", "cake_gated_act_mxfp8_b25fe627c7656aebe218", + "cake_gated_act_mxfp8_44395eef5de97bba1695", "cake_gated_act_mxfp8_e96f02e396eaf24e2caa", ], }, @@ -2532,9 +2790,9 @@ "cake_gated_act_mxfp8_a133fc612a9b9a21e84b", ], "bwd_row": [ - "cake_gated_act_mxfp8_8da8bc14e105ce409f74", + "cake_gated_act_mxfp8_54573bbcd9b4aadbd39f", "cake_gated_act_mxfp8_957f6d5f3e03426c0964", - "cake_gated_act_mxfp8_077a568b810590574917", + "cake_gated_act_mxfp8_3e0ac746df31b17317e0", ], "bwd_col": [ "cake_gated_act_mxfp8_2513ecc12f03a6081c2e", @@ -2542,7 +2800,7 @@ "cake_gated_act_mxfp8_6ee084f4ded4d19cd8cd", ], "bwd_both": [ - "cake_gated_act_mxfp8_8c43730c1e3d76ce89f1", + "cake_gated_act_mxfp8_383546f101950463dcf8", "cake_gated_act_mxfp8_6d1956b2f454b08e6d34", "cake_gated_act_mxfp8_f4e33162da7e5d60c9b4", "cake_gated_act_mxfp8_eb924283e8b660bdeea8", diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_binding.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_binding.cu new file mode 100644 index 00000000000..8e9631b1622 --- /dev/null +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_binding.cu @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2026 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4, for the device source that defines it. Do not edit. +#include +#include + +#include "cake_ffi_checks.h" + +#include + +extern "C" __global__ void kernel_cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4(__nv_bfloat16* __restrict__ gated_input, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K); + +namespace cake_binding_kernel_cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4 { + +using tvm::ffi::TensorView; + +constexpr const char* KERNEL_NAME = "kernel_cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4"; + +void CheckCuda(cudaError_t status, const char* what) { + TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) + << KERNEL_NAME << ": " << what << " failed: " << cudaGetErrorString(status); +} + +void* DataPtr(const TensorView& t) { return static_cast(t.data_ptr()) + t.byte_offset(); } + +void Run(TensorView a_gated_input, TensorView a_row_output, TensorView a_col_output, TensorView a_row_scales, TensorView a_col_scales, int64_t a_M, int64_t a_K, int64_t grid_x, int64_t grid_y, int64_t grid_z) { + DLDevice dev = a_gated_input.device(); + tvm::ffi::CUDADeviceGuard device_guard(dev.device_id); + check_cuda_tensor(a_gated_input, "gated_input"); + check_dtype(a_gated_input, DLDataType{kDLBfloat, 16, 1}, "gated_input"); + check_cuda_tensor(a_row_output, "row_output"); + check_dtype(a_row_output, DLDataType{kDLUInt, 8, 1}, "row_output"); + check_cuda_tensor(a_col_output, "col_output"); + check_dtype(a_col_output, DLDataType{kDLUInt, 8, 1}, "col_output"); + check_cuda_tensor(a_row_scales, "row_scales"); + check_dtype(a_row_scales, DLDataType{kDLUInt, 8, 1}, "row_scales"); + check_cuda_tensor(a_col_scales, "col_scales"); + check_dtype(a_col_scales, DLDataType{kDLUInt, 8, 1}, "col_scales"); + TVM_FFI_CHECK(a_M >= -2147483648LL && a_M <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'M' = " << a_M + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_K >= -2147483648LL && a_K <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'K' = " << a_K + << " is outside the i32 range [-2147483648, 2147483647]"; + check_contiguous(a_gated_input, "gated_input"); + check_contiguous(a_row_output, "row_output"); + check_same_device(a_row_output, a_gated_input, "row_output", "gated_input"); + check_contiguous(a_col_output, "col_output"); + check_same_device(a_col_output, a_gated_input, "col_output", "gated_input"); + check_contiguous(a_row_scales, "row_scales"); + check_same_device(a_row_scales, a_gated_input, "row_scales", "gated_input"); + check_contiguous(a_col_scales, "col_scales"); + check_same_device(a_col_scales, a_gated_input, "col_scales", "gated_input"); + TVM_FFI_CHECK(grid_x > 0 && grid_y > 0 && grid_z > 0, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") must be positive"; + TVM_FFI_CHECK(grid_x <= 2147483647 && grid_y <= 65535 && grid_z <= 65535, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") exceeds CUDA's limits"; + cudaStream_t stream = static_cast(TVMFFIEnvGetStream(dev.device_type, dev.device_id)); + void* v_gated_input = DataPtr(a_gated_input); + void* v_row_output = DataPtr(a_row_output); + void* v_col_output = DataPtr(a_col_output); + void* v_row_scales = DataPtr(a_row_scales); + void* v_col_scales = DataPtr(a_col_scales); + int32_t v_M = static_cast(a_M); + int32_t v_K = static_cast(a_K); + void* kernel_args[] = {&v_gated_input, &v_row_output, &v_col_output, &v_row_scales, &v_col_scales, &v_M, &v_K}; + cudaLaunchConfig_t config{}; + config.gridDim = dim3(static_cast(grid_x), static_cast(grid_y), static_cast(grid_z)); + config.blockDim = dim3(256, 1, 1); + config.dynamicSmemBytes = 16384; + config.stream = stream; + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4), kernel_args), + "cudaLaunchKernelExC"); +} + +} // namespace cake_binding_kernel_cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4 + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_kernel.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_kernel.cu new file mode 100644 index 00000000000..4bdd64a5484 --- /dev/null +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4_kernel.cu @@ -0,0 +1,2215 @@ +/* + * Copyright (c) 2026 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +#if defined(__CUDACC_RTC__) +typedef unsigned long long uint64_t; +#else +typedef unsigned long uint64_t; +#endif +static_assert(sizeof(uint64_t) == 8, "Cake requires an LP64 CUDA host ABI"); +typedef signed int int32_t; +typedef short int int16_t; +struct __align__(64) CakeTensorMap64 { uint64_t opaque[16]; }; +static_assert(sizeof(CakeTensorMap64) == 128, "64-aligned tensor-map ABI size"); +static_assert(alignof(CakeTensorMap64) == 64, "64-aligned tensor-map ABI alignment"); + +#if defined(__CUDACC_RTC__) +typedef struct __align__(128) { uint64_t opaque[16]; } CUtensorMap; +#else +#include +#endif + +static_assert(sizeof(CUtensorMap) == 128, "CUtensorMap CUDA ABI must be 128 bytes"); +#include +#include + +__device__ __forceinline__ int make_warp_uniform(int x) { + int result; + asm volatile("shfl.sync.idx.b32 %0, %1, 0, 0x1F, 0xFFFFFFFF;" + : "=r"(result) : "r"(x)); + return result; +} + +#define CAKE_INF CUDART_INF_F +#define NUM_MAIN_STAGES 1 +#define SMEM_TILE_OFF 0 +#define SMEM_TILE_STAGE_BYTES 16384 +#define SMEM_TILE_STRIDE 16384 +#define SMEM_TOTAL 16384 +#define THREADS 256 +#define LAUNCH_MIN_BLOCKS 5 + +#include + +__device__ __forceinline__ float approx_exp2(float x) { + float y; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + +__device__ __forceinline__ float max_noftz(float a, float b) { + float c; + asm("max.f32 %0, %1, %2;" : "=f"(c) : "f"(a), "f"(b)); + return c; +} + + +__device__ __forceinline__ float2 mul_f32x2_noftz(float2 a, float2 b) { + float2 r; + asm("mul.rn.f32x2 %0, %1, %2;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(unsigned long long*)&a), "l"(*(unsigned long long*)&b)); + return r; +} + + +// ex2_emulation_f32x2 defined in softmax_frag_exp2_cast helper (or standalone) + + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + +__device__ __forceinline__ __nv_bfloat162 __as_bf16x2(unsigned int v) { + __nv_bfloat162_raw raw; + raw.x = static_cast(v); + raw.y = static_cast(v >> 16); + return __nv_bfloat162(raw); +} + +__device__ __forceinline__ float2 cake_bf16x2_to_f32x2(__nv_bfloat162 v) { + const unsigned int w = *reinterpret_cast(&v); + return make_float2(__uint_as_float(w << 16), __uint_as_float(w & 0xffff0000u)); +} + +__device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { + float r; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} + +extern "C" { + +__global__ __launch_bounds__(256, LAUNCH_MIN_BLOCKS) void +kernel_cake_gated_act_mxfp8_0b9dfb9aa4b837a6b7a4(__nv_bfloat16* __restrict__ gated_input, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K) +{ + const int tid = threadIdx.x; + const int warp = make_warp_uniform(tid / 32); + const int lane = tid % 32; + + extern __shared__ __align__(1024) char smem_raw[]; + int smem; +#if __CUDA_ARCH__ == 1000 + asm volatile("{ .reg .u64 smem_ptr; cvta.to.shared.u64 smem_ptr, %1; cvt.u32.u64 %0, smem_ptr; }" : "=r"(smem) : "l"(smem_raw)); + smem = make_warp_uniform(smem); +#else + smem = (int)(unsigned long long)__cvta_generic_to_shared(smem_raw); +#endif + + const int bid = blockIdx.x; + const int num_bids = gridDim.x; + + const int cta_rank = 0; + + // Kernel setup ops + uint8_t* tile = reinterpret_cast(smem_raw + 0); + const int tile_addr = smem + 0; + + // === Task calls (declaration order) === + int half = tid & 1; + int blk = tid >> 1 & 3; + int row = tid >> 3; + int bx = blockIdx.x; + int by = blockIdx.y; + int grow = by * 64 + row; + int col = bx * 128 + blk * 32 + half * 16; + int gate_index = grow * (2 * K) + col; + unsigned int _vec_load_0[8]; + { + uint32_t _iv_0_0; + uint32_t _iv_0_1; + uint32_t _iv_0_2; + uint32_t _iv_0_3; + uint32_t _iv_0_4; + uint32_t _iv_0_5; + uint32_t _iv_0_6; + uint32_t _iv_0_7; + asm volatile("ld.global.L1::no_allocate.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(_iv_0_0), "=r"(_iv_0_1), "=r"(_iv_0_2), "=r"(_iv_0_3), "=r"(_iv_0_4), "=r"(_iv_0_5), "=r"(_iv_0_6), "=r"(_iv_0_7) : "l"((const void*)(gated_input + gate_index + (0))) : "memory"); + _vec_load_0[0 + 0] = (uint32_t)_iv_0_0; + _vec_load_0[0 + 1] = (uint32_t)_iv_0_1; + _vec_load_0[0 + 2] = (uint32_t)_iv_0_2; + _vec_load_0[0 + 3] = (uint32_t)_iv_0_3; + _vec_load_0[0 + 4] = (uint32_t)_iv_0_4; + _vec_load_0[0 + 5] = (uint32_t)_iv_0_5; + _vec_load_0[0 + 6] = (uint32_t)_iv_0_6; + _vec_load_0[0 + 7] = (uint32_t)_iv_0_7; + } + unsigned int _vec_load_1[8]; + { + uint32_t _iv_1_0; + uint32_t _iv_1_1; + uint32_t _iv_1_2; + uint32_t _iv_1_3; + uint32_t _iv_1_4; + uint32_t _iv_1_5; + uint32_t _iv_1_6; + uint32_t _iv_1_7; + asm volatile("ld.global.L1::no_allocate.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(_iv_1_0), "=r"(_iv_1_1), "=r"(_iv_1_2), "=r"(_iv_1_3), "=r"(_iv_1_4), "=r"(_iv_1_5), "=r"(_iv_1_6), "=r"(_iv_1_7) : "l"((const void*)(gated_input + (gate_index + K) + (0))) : "memory"); + _vec_load_1[0 + 0] = (uint32_t)_iv_1_0; + _vec_load_1[0 + 1] = (uint32_t)_iv_1_1; + _vec_load_1[0 + 2] = (uint32_t)_iv_1_2; + _vec_load_1[0 + 3] = (uint32_t)_iv_1_3; + _vec_load_1[0 + 4] = (uint32_t)_iv_1_4; + _vec_load_1[0 + 5] = (uint32_t)_iv_1_5; + _vec_load_1[0 + 6] = (uint32_t)_iv_1_6; + _vec_load_1[0 + 7] = (uint32_t)_iv_1_7; + } + int grow_hi = grow + 32; + int gate_index_hi = grow_hi * (2 * K) + col; + unsigned int _vec_load_2[8]; + { + uint32_t _iv_2_0; + uint32_t _iv_2_1; + uint32_t _iv_2_2; + uint32_t _iv_2_3; + uint32_t _iv_2_4; + uint32_t _iv_2_5; + uint32_t _iv_2_6; + uint32_t _iv_2_7; + asm volatile("ld.global.L1::no_allocate.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(_iv_2_0), "=r"(_iv_2_1), "=r"(_iv_2_2), "=r"(_iv_2_3), "=r"(_iv_2_4), "=r"(_iv_2_5), "=r"(_iv_2_6), "=r"(_iv_2_7) : "l"((const void*)(gated_input + gate_index_hi + (0))) : "memory"); + _vec_load_2[0 + 0] = (uint32_t)_iv_2_0; + _vec_load_2[0 + 1] = (uint32_t)_iv_2_1; + _vec_load_2[0 + 2] = (uint32_t)_iv_2_2; + _vec_load_2[0 + 3] = (uint32_t)_iv_2_3; + _vec_load_2[0 + 4] = (uint32_t)_iv_2_4; + _vec_load_2[0 + 5] = (uint32_t)_iv_2_5; + _vec_load_2[0 + 6] = (uint32_t)_iv_2_6; + _vec_load_2[0 + 7] = (uint32_t)_iv_2_7; + } + unsigned int _vec_load_3[8]; + { + uint32_t _iv_3_0; + uint32_t _iv_3_1; + uint32_t _iv_3_2; + uint32_t _iv_3_3; + uint32_t _iv_3_4; + uint32_t _iv_3_5; + uint32_t _iv_3_6; + uint32_t _iv_3_7; + asm volatile("ld.global.L1::no_allocate.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(_iv_3_0), "=r"(_iv_3_1), "=r"(_iv_3_2), "=r"(_iv_3_3), "=r"(_iv_3_4), "=r"(_iv_3_5), "=r"(_iv_3_6), "=r"(_iv_3_7) : "l"((const void*)(gated_input + (gate_index_hi + K) + (0))) : "memory"); + _vec_load_3[0 + 0] = (uint32_t)_iv_3_0; + _vec_load_3[0 + 1] = (uint32_t)_iv_3_1; + _vec_load_3[0 + 2] = (uint32_t)_iv_3_2; + _vec_load_3[0 + 3] = (uint32_t)_iv_3_3; + _vec_load_3[0 + 4] = (uint32_t)_iv_3_4; + _vec_load_3[0 + 5] = (uint32_t)_iv_3_5; + _vec_load_3[0 + 6] = (uint32_t)_iv_3_6; + _vec_load_3[0 + 7] = (uint32_t)_iv_3_7; + } + unsigned int packed[8]; + unsigned int packed_hi[8]; + __nv_bfloat162 lowest = __as_bf16x2(_vec_load_0[0]); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[1])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[2])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[3])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[4])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[5])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[6])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[7])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_2[0])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_2[1])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_2[2])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_2[3])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_2[4])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_2[5])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_2[6])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_2[7])); + float2 _cvt_f32_0 = cake_bf16x2_to_f32x2(lowest); + float _min_5; + asm("min.NaN.f32 %0, %1, %2;" : "=f"(_min_5) : "f"(_cvt_f32_0.x), "f"(_cvt_f32_0.y)); + int _warp_vote_4 = __all_sync(0xFFFFFFFF, _min_5 > -87.0f); + if (_warp_vote_4 != 0) { + float2 _cvt_f32_1 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[0])); + float2 _cvt_f32_2 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[0])); + float e0_0 = approx_exp2((-_cvt_f32_1.x) * 1.442695f); + float e1_0 = approx_exp2((-_cvt_f32_1.y) * 1.442695f); + float d0_0 = 1.0f + e0_0; + float d1_0 = 1.0f + e1_0; + float r0_0 = approx_rcp(d0_0); + float r1_0 = approx_rcp(d1_0); + float _fma_7; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_7) : "f"(-d0_0), "f"(r0_0), "f"(1.0f)); + float _fma_6; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_6) : "f"(r0_0), "f"(_fma_7), "f"(r0_0)); + float _fma_9; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_9) : "f"(-d1_0), "f"(r1_0), "f"(1.0f)); + float _fma_8; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_8) : "f"(r1_0), "f"(_fma_9), "f"(r1_0)); + float2 _f32x2_mul_a_10 = mul_f32x2_noftz(_cvt_f32_1, make_float2(_fma_6, _fma_8)); + float2 _mul_0; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_10), "l"(*(const unsigned long long*)&_cvt_f32_2)); + packed[0] = __as_u32(__float22bfloat162_rn(make_float2(_mul_0.x, _mul_0.y))); + float2 _cvt_f32_3 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[1])); + float2 _cvt_f32_4 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[1])); + float e0_1 = approx_exp2((-_cvt_f32_3.x) * 1.442695f); + float e1_1 = approx_exp2((-_cvt_f32_3.y) * 1.442695f); + float d0_1 = 1.0f + e0_1; + float d1_1 = 1.0f + e1_1; + float r0_1 = approx_rcp(d0_1); + float r1_1 = approx_rcp(d1_1); + float _fma_12; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_12) : "f"(-d0_1), "f"(r0_1), "f"(1.0f)); + float _fma_11; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_11) : "f"(r0_1), "f"(_fma_12), "f"(r0_1)); + float _fma_14; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_14) : "f"(-d1_1), "f"(r1_1), "f"(1.0f)); + float _fma_13; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_13) : "f"(r1_1), "f"(_fma_14), "f"(r1_1)); + float2 _f32x2_mul_a_15 = mul_f32x2_noftz(_cvt_f32_3, make_float2(_fma_11, _fma_13)); + float2 _mul_1; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_1) : "l"(*(const unsigned long long*)&_f32x2_mul_a_15), "l"(*(const unsigned long long*)&_cvt_f32_4)); + packed[1] = __as_u32(__float22bfloat162_rn(make_float2(_mul_1.x, _mul_1.y))); + float2 _cvt_f32_5 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[2])); + float2 _cvt_f32_6 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[2])); + float e0_2 = approx_exp2((-_cvt_f32_5.x) * 1.442695f); + float e1_2 = approx_exp2((-_cvt_f32_5.y) * 1.442695f); + float d0_2 = 1.0f + e0_2; + float d1_2 = 1.0f + e1_2; + float r0_2 = approx_rcp(d0_2); + float r1_2 = approx_rcp(d1_2); + float _fma_17; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_17) : "f"(-d0_2), "f"(r0_2), "f"(1.0f)); + float _fma_16; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_16) : "f"(r0_2), "f"(_fma_17), "f"(r0_2)); + float _fma_19; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_19) : "f"(-d1_2), "f"(r1_2), "f"(1.0f)); + float _fma_18; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_18) : "f"(r1_2), "f"(_fma_19), "f"(r1_2)); + float2 _f32x2_mul_a_20 = mul_f32x2_noftz(_cvt_f32_5, make_float2(_fma_16, _fma_18)); + float2 _mul_2; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_20), "l"(*(const unsigned long long*)&_cvt_f32_6)); + packed[2] = __as_u32(__float22bfloat162_rn(make_float2(_mul_2.x, _mul_2.y))); + float2 _cvt_f32_7 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[3])); + float2 _cvt_f32_8 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[3])); + float e0_3 = approx_exp2((-_cvt_f32_7.x) * 1.442695f); + float e1_3 = approx_exp2((-_cvt_f32_7.y) * 1.442695f); + float d0_3 = 1.0f + e0_3; + float d1_3 = 1.0f + e1_3; + float r0_3 = approx_rcp(d0_3); + float r1_3 = approx_rcp(d1_3); + float _fma_22; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_22) : "f"(-d0_3), "f"(r0_3), "f"(1.0f)); + float _fma_21; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_21) : "f"(r0_3), "f"(_fma_22), "f"(r0_3)); + float _fma_24; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_24) : "f"(-d1_3), "f"(r1_3), "f"(1.0f)); + float _fma_23; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_23) : "f"(r1_3), "f"(_fma_24), "f"(r1_3)); + float2 _f32x2_mul_a_25 = mul_f32x2_noftz(_cvt_f32_7, make_float2(_fma_21, _fma_23)); + float2 _mul_3; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_3) : "l"(*(const unsigned long long*)&_f32x2_mul_a_25), "l"(*(const unsigned long long*)&_cvt_f32_8)); + packed[3] = __as_u32(__float22bfloat162_rn(make_float2(_mul_3.x, _mul_3.y))); + float2 _cvt_f32_9 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[4])); + float2 _cvt_f32_10 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[4])); + float e0_4 = approx_exp2((-_cvt_f32_9.x) * 1.442695f); + float e1_4 = approx_exp2((-_cvt_f32_9.y) * 1.442695f); + float d0_4 = 1.0f + e0_4; + float d1_4 = 1.0f + e1_4; + float r0_4 = approx_rcp(d0_4); + float r1_4 = approx_rcp(d1_4); + float _fma_27; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_27) : "f"(-d0_4), "f"(r0_4), "f"(1.0f)); + float _fma_26; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_26) : "f"(r0_4), "f"(_fma_27), "f"(r0_4)); + float _fma_29; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_29) : "f"(-d1_4), "f"(r1_4), "f"(1.0f)); + float _fma_28; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_28) : "f"(r1_4), "f"(_fma_29), "f"(r1_4)); + float2 _f32x2_mul_a_30 = mul_f32x2_noftz(_cvt_f32_9, make_float2(_fma_26, _fma_28)); + float2 _mul_4; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_30), "l"(*(const unsigned long long*)&_cvt_f32_10)); + packed[4] = __as_u32(__float22bfloat162_rn(make_float2(_mul_4.x, _mul_4.y))); + float2 _cvt_f32_11 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[5])); + float2 _cvt_f32_12 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[5])); + float e0_5 = approx_exp2((-_cvt_f32_11.x) * 1.442695f); + float e1_5 = approx_exp2((-_cvt_f32_11.y) * 1.442695f); + float d0_5 = 1.0f + e0_5; + float d1_5 = 1.0f + e1_5; + float r0_5 = approx_rcp(d0_5); + float r1_5 = approx_rcp(d1_5); + float _fma_32; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_32) : "f"(-d0_5), "f"(r0_5), "f"(1.0f)); + float _fma_31; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_31) : "f"(r0_5), "f"(_fma_32), "f"(r0_5)); + float _fma_34; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_34) : "f"(-d1_5), "f"(r1_5), "f"(1.0f)); + float _fma_33; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_33) : "f"(r1_5), "f"(_fma_34), "f"(r1_5)); + float2 _f32x2_mul_a_35 = mul_f32x2_noftz(_cvt_f32_11, make_float2(_fma_31, _fma_33)); + float2 _mul_5; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_5) : "l"(*(const unsigned long long*)&_f32x2_mul_a_35), "l"(*(const unsigned long long*)&_cvt_f32_12)); + packed[5] = __as_u32(__float22bfloat162_rn(make_float2(_mul_5.x, _mul_5.y))); + float2 _cvt_f32_13 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[6])); + float2 _cvt_f32_14 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[6])); + float e0_6 = approx_exp2((-_cvt_f32_13.x) * 1.442695f); + float e1_6 = approx_exp2((-_cvt_f32_13.y) * 1.442695f); + float d0_6 = 1.0f + e0_6; + float d1_6 = 1.0f + e1_6; + float r0_6 = approx_rcp(d0_6); + float r1_6 = approx_rcp(d1_6); + float _fma_37; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_37) : "f"(-d0_6), "f"(r0_6), "f"(1.0f)); + float _fma_36; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_36) : "f"(r0_6), "f"(_fma_37), "f"(r0_6)); + float _fma_39; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_39) : "f"(-d1_6), "f"(r1_6), "f"(1.0f)); + float _fma_38; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_38) : "f"(r1_6), "f"(_fma_39), "f"(r1_6)); + float2 _f32x2_mul_a_40 = mul_f32x2_noftz(_cvt_f32_13, make_float2(_fma_36, _fma_38)); + float2 _mul_6; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_40), "l"(*(const unsigned long long*)&_cvt_f32_14)); + packed[6] = __as_u32(__float22bfloat162_rn(make_float2(_mul_6.x, _mul_6.y))); + float2 _cvt_f32_15 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[7])); + float2 _cvt_f32_16 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[7])); + float e0_7 = approx_exp2((-_cvt_f32_15.x) * 1.442695f); + float e1_7 = approx_exp2((-_cvt_f32_15.y) * 1.442695f); + float d0_7 = 1.0f + e0_7; + float d1_7 = 1.0f + e1_7; + float r0_7 = approx_rcp(d0_7); + float r1_7 = approx_rcp(d1_7); + float _fma_42; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_42) : "f"(-d0_7), "f"(r0_7), "f"(1.0f)); + float _fma_41; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_41) : "f"(r0_7), "f"(_fma_42), "f"(r0_7)); + float _fma_44; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_44) : "f"(-d1_7), "f"(r1_7), "f"(1.0f)); + float _fma_43; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_43) : "f"(r1_7), "f"(_fma_44), "f"(r1_7)); + float2 _f32x2_mul_a_45 = mul_f32x2_noftz(_cvt_f32_15, make_float2(_fma_41, _fma_43)); + float2 _mul_7; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_7) : "l"(*(const unsigned long long*)&_f32x2_mul_a_45), "l"(*(const unsigned long long*)&_cvt_f32_16)); + packed[7] = __as_u32(__float22bfloat162_rn(make_float2(_mul_7.x, _mul_7.y))); + float2 _cvt_f32_17 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[0])); + float2 _cvt_f32_18 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[0])); + float e0_8 = approx_exp2((-_cvt_f32_17.x) * 1.442695f); + float e1_8 = approx_exp2((-_cvt_f32_17.y) * 1.442695f); + float d0_8 = 1.0f + e0_8; + float d1_8 = 1.0f + e1_8; + float r0_8 = approx_rcp(d0_8); + float r1_8 = approx_rcp(d1_8); + float _fma_47; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_47) : "f"(-d0_8), "f"(r0_8), "f"(1.0f)); + float _fma_46; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_46) : "f"(r0_8), "f"(_fma_47), "f"(r0_8)); + float _fma_49; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_49) : "f"(-d1_8), "f"(r1_8), "f"(1.0f)); + float _fma_48; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_48) : "f"(r1_8), "f"(_fma_49), "f"(r1_8)); + float2 _f32x2_mul_a_50 = mul_f32x2_noftz(_cvt_f32_17, make_float2(_fma_46, _fma_48)); + float2 _mul_8; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_50), "l"(*(const unsigned long long*)&_cvt_f32_18)); + packed_hi[0] = __as_u32(__float22bfloat162_rn(make_float2(_mul_8.x, _mul_8.y))); + float2 _cvt_f32_19 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[1])); + float2 _cvt_f32_20 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[1])); + float e0_9 = approx_exp2((-_cvt_f32_19.x) * 1.442695f); + float e1_9 = approx_exp2((-_cvt_f32_19.y) * 1.442695f); + float d0_9 = 1.0f + e0_9; + float d1_9 = 1.0f + e1_9; + float r0_9 = approx_rcp(d0_9); + float r1_9 = approx_rcp(d1_9); + float _fma_52; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_52) : "f"(-d0_9), "f"(r0_9), "f"(1.0f)); + float _fma_51; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_51) : "f"(r0_9), "f"(_fma_52), "f"(r0_9)); + float _fma_54; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_54) : "f"(-d1_9), "f"(r1_9), "f"(1.0f)); + float _fma_53; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_53) : "f"(r1_9), "f"(_fma_54), "f"(r1_9)); + float2 _f32x2_mul_a_55 = mul_f32x2_noftz(_cvt_f32_19, make_float2(_fma_51, _fma_53)); + float2 _mul_9; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_9) : "l"(*(const unsigned long long*)&_f32x2_mul_a_55), "l"(*(const unsigned long long*)&_cvt_f32_20)); + packed_hi[1] = __as_u32(__float22bfloat162_rn(make_float2(_mul_9.x, _mul_9.y))); + float2 _cvt_f32_21 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[2])); + float2 _cvt_f32_22 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[2])); + float e0_10 = approx_exp2((-_cvt_f32_21.x) * 1.442695f); + float e1_10 = approx_exp2((-_cvt_f32_21.y) * 1.442695f); + float d0_10 = 1.0f + e0_10; + float d1_10 = 1.0f + e1_10; + float r0_10 = approx_rcp(d0_10); + float r1_10 = approx_rcp(d1_10); + float _fma_57; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_57) : "f"(-d0_10), "f"(r0_10), "f"(1.0f)); + float _fma_56; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_56) : "f"(r0_10), "f"(_fma_57), "f"(r0_10)); + float _fma_59; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_59) : "f"(-d1_10), "f"(r1_10), "f"(1.0f)); + float _fma_58; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_58) : "f"(r1_10), "f"(_fma_59), "f"(r1_10)); + float2 _f32x2_mul_a_60 = mul_f32x2_noftz(_cvt_f32_21, make_float2(_fma_56, _fma_58)); + float2 _mul_10; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_60), "l"(*(const unsigned long long*)&_cvt_f32_22)); + packed_hi[2] = __as_u32(__float22bfloat162_rn(make_float2(_mul_10.x, _mul_10.y))); + float2 _cvt_f32_23 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[3])); + float2 _cvt_f32_24 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[3])); + float e0_11 = approx_exp2((-_cvt_f32_23.x) * 1.442695f); + float e1_11 = approx_exp2((-_cvt_f32_23.y) * 1.442695f); + float d0_11 = 1.0f + e0_11; + float d1_11 = 1.0f + e1_11; + float r0_11 = approx_rcp(d0_11); + float r1_11 = approx_rcp(d1_11); + float _fma_62; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_62) : "f"(-d0_11), "f"(r0_11), "f"(1.0f)); + float _fma_61; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_61) : "f"(r0_11), "f"(_fma_62), "f"(r0_11)); + float _fma_64; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_64) : "f"(-d1_11), "f"(r1_11), "f"(1.0f)); + float _fma_63; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_63) : "f"(r1_11), "f"(_fma_64), "f"(r1_11)); + float2 _f32x2_mul_a_65 = mul_f32x2_noftz(_cvt_f32_23, make_float2(_fma_61, _fma_63)); + float2 _mul_11; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_11) : "l"(*(const unsigned long long*)&_f32x2_mul_a_65), "l"(*(const unsigned long long*)&_cvt_f32_24)); + packed_hi[3] = __as_u32(__float22bfloat162_rn(make_float2(_mul_11.x, _mul_11.y))); + float2 _cvt_f32_25 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[4])); + float2 _cvt_f32_26 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[4])); + float e0_12 = approx_exp2((-_cvt_f32_25.x) * 1.442695f); + float e1_12 = approx_exp2((-_cvt_f32_25.y) * 1.442695f); + float d0_12 = 1.0f + e0_12; + float d1_12 = 1.0f + e1_12; + float r0_12 = approx_rcp(d0_12); + float r1_12 = approx_rcp(d1_12); + float _fma_67; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_67) : "f"(-d0_12), "f"(r0_12), "f"(1.0f)); + float _fma_66; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_66) : "f"(r0_12), "f"(_fma_67), "f"(r0_12)); + float _fma_69; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_69) : "f"(-d1_12), "f"(r1_12), "f"(1.0f)); + float _fma_68; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_68) : "f"(r1_12), "f"(_fma_69), "f"(r1_12)); + float2 _f32x2_mul_a_70 = mul_f32x2_noftz(_cvt_f32_25, make_float2(_fma_66, _fma_68)); + float2 _mul_12; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_70), "l"(*(const unsigned long long*)&_cvt_f32_26)); + packed_hi[4] = __as_u32(__float22bfloat162_rn(make_float2(_mul_12.x, _mul_12.y))); + float2 _cvt_f32_27 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[5])); + float2 _cvt_f32_28 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[5])); + float e0_13 = approx_exp2((-_cvt_f32_27.x) * 1.442695f); + float e1_13 = approx_exp2((-_cvt_f32_27.y) * 1.442695f); + float d0_13 = 1.0f + e0_13; + float d1_13 = 1.0f + e1_13; + float r0_13 = approx_rcp(d0_13); + float r1_13 = approx_rcp(d1_13); + float _fma_72; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_72) : "f"(-d0_13), "f"(r0_13), "f"(1.0f)); + float _fma_71; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_71) : "f"(r0_13), "f"(_fma_72), "f"(r0_13)); + float _fma_74; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_74) : "f"(-d1_13), "f"(r1_13), "f"(1.0f)); + float _fma_73; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_73) : "f"(r1_13), "f"(_fma_74), "f"(r1_13)); + float2 _f32x2_mul_a_75 = mul_f32x2_noftz(_cvt_f32_27, make_float2(_fma_71, _fma_73)); + float2 _mul_13; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_13) : "l"(*(const unsigned long long*)&_f32x2_mul_a_75), "l"(*(const unsigned long long*)&_cvt_f32_28)); + packed_hi[5] = __as_u32(__float22bfloat162_rn(make_float2(_mul_13.x, _mul_13.y))); + float2 _cvt_f32_29 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[6])); + float2 _cvt_f32_30 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[6])); + float e0_14 = approx_exp2((-_cvt_f32_29.x) * 1.442695f); + float e1_14 = approx_exp2((-_cvt_f32_29.y) * 1.442695f); + float d0_14 = 1.0f + e0_14; + float d1_14 = 1.0f + e1_14; + float r0_14 = approx_rcp(d0_14); + float r1_14 = approx_rcp(d1_14); + float _fma_77; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_77) : "f"(-d0_14), "f"(r0_14), "f"(1.0f)); + float _fma_76; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_76) : "f"(r0_14), "f"(_fma_77), "f"(r0_14)); + float _fma_79; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_79) : "f"(-d1_14), "f"(r1_14), "f"(1.0f)); + float _fma_78; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_78) : "f"(r1_14), "f"(_fma_79), "f"(r1_14)); + float2 _f32x2_mul_a_80 = mul_f32x2_noftz(_cvt_f32_29, make_float2(_fma_76, _fma_78)); + float2 _mul_14; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_80), "l"(*(const unsigned long long*)&_cvt_f32_30)); + packed_hi[6] = __as_u32(__float22bfloat162_rn(make_float2(_mul_14.x, _mul_14.y))); + float2 _cvt_f32_31 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[7])); + float2 _cvt_f32_32 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[7])); + float e0_15 = approx_exp2((-_cvt_f32_31.x) * 1.442695f); + float e1_15 = approx_exp2((-_cvt_f32_31.y) * 1.442695f); + float d0_15 = 1.0f + e0_15; + float d1_15 = 1.0f + e1_15; + float r0_15 = approx_rcp(d0_15); + float r1_15 = approx_rcp(d1_15); + float _fma_82; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_82) : "f"(-d0_15), "f"(r0_15), "f"(1.0f)); + float _fma_81; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_81) : "f"(r0_15), "f"(_fma_82), "f"(r0_15)); + float _fma_84; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_84) : "f"(-d1_15), "f"(r1_15), "f"(1.0f)); + float _fma_83; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_83) : "f"(r1_15), "f"(_fma_84), "f"(r1_15)); + float2 _f32x2_mul_a_85 = mul_f32x2_noftz(_cvt_f32_31, make_float2(_fma_81, _fma_83)); + float2 _mul_15; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_15) : "l"(*(const unsigned long long*)&_f32x2_mul_a_85), "l"(*(const unsigned long long*)&_cvt_f32_32)); + packed_hi[7] = __as_u32(__float22bfloat162_rn(make_float2(_mul_15.x, _mul_15.y))); + } else { + int band = 0; + float2 _cvt_f32_33 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[0])); + float2 _cvt_f32_34 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[0])); + float e0_16 = approx_exp2((-_cvt_f32_33.x) * 1.442695f); + float e1_16 = approx_exp2((-_cvt_f32_33.y) * 1.442695f); + float d0_16 = 1.0f + e0_16; + float d1_16 = 1.0f + e1_16; + float r0_16 = approx_rcp(d0_16); + float r1_16 = approx_rcp(d1_16); + float _fma_86; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_86) : "f"(-d0_16), "f"(r0_16), "f"(1.0f)); + float _fma_0; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_0) : "f"(r0_16), "f"(_fma_86), "f"(r0_16)); + float _fma_87; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_87) : "f"(-d1_16), "f"(r1_16), "f"(1.0f)); + float _fma_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_1) : "f"(r1_16), "f"(_fma_87), "f"(r1_16)); + float t0_16 = _fma_0; + float t1_16 = _fma_1; + int _warp_vote_88 = __any_sync(0xFFFFFFFF, fminf(t0_16, t1_16) == 0.0f); + if (_warp_vote_88 != 0) { + float d4_16a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_16a) : "f"(d0_16), "f"(0.25f)); + float r4_16a = approx_rcp(d4_16a); + float _fma_90; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_90) : "f"(-d4_16a), "f"(r4_16a), "f"(1.0f)); + float _fma_89; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_89) : "f"(r4_16a), "f"(_fma_90), "f"(r4_16a)); + float big_16a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_16a) : "f"(mul_f32_rn_noftz(_fma_89, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_91; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_91) : "f"(big_16a), "f"(8.388608e+06f)); + float whole_16a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_16a) : "f"(_add_91), "f"(8.388608e+06f)); + float above_16a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_16a) : "f"(-d4_16a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_16a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_93; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_93) : "f"(big_16a), "f"(whole_16a)); + float _abs_92; + asm("abs.f32 %0, %1;" : "=f"(_abs_92) : "f"(_sub_93)); + float _add_94; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_94) : "f"(big_16a), "f"(0.5f)); + float _sub_95; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_95) : "f"(big_16a), "f"(0.5f)); + float rounded_16a = ((_abs_92 == 0.5f) ? ((above_16a > 0.0f) ? _add_94 : _sub_95) : whole_16a); + t0_16 = ((d0_16 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_16a, 7.888609e-31f), 1.7763568e-15f) : t0_16); + float d4_16b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_16b) : "f"(d1_16), "f"(0.25f)); + float r4_16b = approx_rcp(d4_16b); + float _fma_97; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_97) : "f"(-d4_16b), "f"(r4_16b), "f"(1.0f)); + float _fma_96; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_96) : "f"(r4_16b), "f"(_fma_97), "f"(r4_16b)); + float big_16b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_16b) : "f"(mul_f32_rn_noftz(_fma_96, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_98; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_98) : "f"(big_16b), "f"(8.388608e+06f)); + float whole_16b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_16b) : "f"(_add_98), "f"(8.388608e+06f)); + float above_16b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_16b) : "f"(-d4_16b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_16b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_100; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_100) : "f"(big_16b), "f"(whole_16b)); + float _abs_99; + asm("abs.f32 %0, %1;" : "=f"(_abs_99) : "f"(_sub_100)); + float _add_101; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_101) : "f"(big_16b), "f"(0.5f)); + float _sub_102; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_102) : "f"(big_16b), "f"(0.5f)); + float rounded_16b = ((_abs_99 == 0.5f) ? ((above_16b > 0.0f) ? _add_101 : _sub_102) : whole_16b); + t1_16 = ((d1_16 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_16b, 7.888609e-31f), 1.7763568e-15f) : t1_16); + } + float2 _f32x2_mul_a_103 = mul_f32x2_noftz(_cvt_f32_33, make_float2(max_noftz(t0_16, 0.0f), max_noftz(t1_16, 0.0f))); + float2 _mul_16; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_16) : "l"(*(const unsigned long long*)&_f32x2_mul_a_103), "l"(*(const unsigned long long*)&_cvt_f32_34)); + packed[0] = __as_u32(__float22bfloat162_rn(make_float2(_mul_16.x, _mul_16.y))); + float2 _cvt_f32_35 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[1])); + float2 _cvt_f32_36 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[1])); + float e0_17 = approx_exp2((-_cvt_f32_35.x) * 1.442695f); + float e1_17 = approx_exp2((-_cvt_f32_35.y) * 1.442695f); + float d0_17 = 1.0f + e0_17; + float d1_17 = 1.0f + e1_17; + float r0_17 = approx_rcp(d0_17); + float r1_17 = approx_rcp(d1_17); + float _fma_104; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_104) : "f"(-d0_17), "f"(r0_17), "f"(1.0f)); + float _fma_2; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_2) : "f"(r0_17), "f"(_fma_104), "f"(r0_17)); + float _fma_105; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_105) : "f"(-d1_17), "f"(r1_17), "f"(1.0f)); + float _fma_3; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_3) : "f"(r1_17), "f"(_fma_105), "f"(r1_17)); + float t0_17 = _fma_2; + float t1_17 = _fma_3; + int _warp_vote_106 = __any_sync(0xFFFFFFFF, fminf(t0_17, t1_17) == 0.0f); + if (_warp_vote_106 != 0) { + float d4_17a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_17a) : "f"(d0_17), "f"(0.25f)); + float r4_17a = approx_rcp(d4_17a); + float _fma_108; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_108) : "f"(-d4_17a), "f"(r4_17a), "f"(1.0f)); + float _fma_107; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_107) : "f"(r4_17a), "f"(_fma_108), "f"(r4_17a)); + float big_17a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_17a) : "f"(mul_f32_rn_noftz(_fma_107, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_109; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_109) : "f"(big_17a), "f"(8.388608e+06f)); + float whole_17a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_17a) : "f"(_add_109), "f"(8.388608e+06f)); + float above_17a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_17a) : "f"(-d4_17a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_17a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_111; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_111) : "f"(big_17a), "f"(whole_17a)); + float _abs_110; + asm("abs.f32 %0, %1;" : "=f"(_abs_110) : "f"(_sub_111)); + float _add_112; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_112) : "f"(big_17a), "f"(0.5f)); + float _sub_113; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_113) : "f"(big_17a), "f"(0.5f)); + float rounded_17a = ((_abs_110 == 0.5f) ? ((above_17a > 0.0f) ? _add_112 : _sub_113) : whole_17a); + t0_17 = ((d0_17 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_17a, 7.888609e-31f), 1.7763568e-15f) : t0_17); + float d4_17b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_17b) : "f"(d1_17), "f"(0.25f)); + float r4_17b = approx_rcp(d4_17b); + float _fma_115; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_115) : "f"(-d4_17b), "f"(r4_17b), "f"(1.0f)); + float _fma_114; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_114) : "f"(r4_17b), "f"(_fma_115), "f"(r4_17b)); + float big_17b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_17b) : "f"(mul_f32_rn_noftz(_fma_114, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_116; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_116) : "f"(big_17b), "f"(8.388608e+06f)); + float whole_17b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_17b) : "f"(_add_116), "f"(8.388608e+06f)); + float above_17b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_17b) : "f"(-d4_17b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_17b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_118; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_118) : "f"(big_17b), "f"(whole_17b)); + float _abs_117; + asm("abs.f32 %0, %1;" : "=f"(_abs_117) : "f"(_sub_118)); + float _add_119; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_119) : "f"(big_17b), "f"(0.5f)); + float _sub_120; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_120) : "f"(big_17b), "f"(0.5f)); + float rounded_17b = ((_abs_117 == 0.5f) ? ((above_17b > 0.0f) ? _add_119 : _sub_120) : whole_17b); + t1_17 = ((d1_17 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_17b, 7.888609e-31f), 1.7763568e-15f) : t1_17); + } + float2 _f32x2_mul_a_121 = mul_f32x2_noftz(_cvt_f32_35, make_float2(max_noftz(t0_17, 0.0f), max_noftz(t1_17, 0.0f))); + float2 _mul_17; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_17) : "l"(*(const unsigned long long*)&_f32x2_mul_a_121), "l"(*(const unsigned long long*)&_cvt_f32_36)); + packed[1] = __as_u32(__float22bfloat162_rn(make_float2(_mul_17.x, _mul_17.y))); + float2 _cvt_f32_37 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[2])); + float2 _cvt_f32_38 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[2])); + float e0_18 = approx_exp2((-_cvt_f32_37.x) * 1.442695f); + float e1_18 = approx_exp2((-_cvt_f32_37.y) * 1.442695f); + float d0_18 = 1.0f + e0_18; + float d1_18 = 1.0f + e1_18; + float r0_18 = approx_rcp(d0_18); + float r1_18 = approx_rcp(d1_18); + float _fma_122; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_122) : "f"(-d0_18), "f"(r0_18), "f"(1.0f)); + float _fma_4; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_4) : "f"(r0_18), "f"(_fma_122), "f"(r0_18)); + float _fma_123; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_123) : "f"(-d1_18), "f"(r1_18), "f"(1.0f)); + float _fma_5; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_5) : "f"(r1_18), "f"(_fma_123), "f"(r1_18)); + float t0_18 = _fma_4; + float t1_18 = _fma_5; + int _warp_vote_124 = __any_sync(0xFFFFFFFF, fminf(t0_18, t1_18) == 0.0f); + if (_warp_vote_124 != 0) { + float d4_18a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_18a) : "f"(d0_18), "f"(0.25f)); + float r4_18a = approx_rcp(d4_18a); + float _fma_126; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_126) : "f"(-d4_18a), "f"(r4_18a), "f"(1.0f)); + float _fma_125; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_125) : "f"(r4_18a), "f"(_fma_126), "f"(r4_18a)); + float big_18a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_18a) : "f"(mul_f32_rn_noftz(_fma_125, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_127; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_127) : "f"(big_18a), "f"(8.388608e+06f)); + float whole_18a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_18a) : "f"(_add_127), "f"(8.388608e+06f)); + float above_18a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_18a) : "f"(-d4_18a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_18a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_129; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_129) : "f"(big_18a), "f"(whole_18a)); + float _abs_128; + asm("abs.f32 %0, %1;" : "=f"(_abs_128) : "f"(_sub_129)); + float _add_130; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_130) : "f"(big_18a), "f"(0.5f)); + float _sub_131; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_131) : "f"(big_18a), "f"(0.5f)); + float rounded_18a = ((_abs_128 == 0.5f) ? ((above_18a > 0.0f) ? _add_130 : _sub_131) : whole_18a); + t0_18 = ((d0_18 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_18a, 7.888609e-31f), 1.7763568e-15f) : t0_18); + float d4_18b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_18b) : "f"(d1_18), "f"(0.25f)); + float r4_18b = approx_rcp(d4_18b); + float _fma_133; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_133) : "f"(-d4_18b), "f"(r4_18b), "f"(1.0f)); + float _fma_132; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_132) : "f"(r4_18b), "f"(_fma_133), "f"(r4_18b)); + float big_18b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_18b) : "f"(mul_f32_rn_noftz(_fma_132, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_134; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_134) : "f"(big_18b), "f"(8.388608e+06f)); + float whole_18b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_18b) : "f"(_add_134), "f"(8.388608e+06f)); + float above_18b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_18b) : "f"(-d4_18b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_18b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_136; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_136) : "f"(big_18b), "f"(whole_18b)); + float _abs_135; + asm("abs.f32 %0, %1;" : "=f"(_abs_135) : "f"(_sub_136)); + float _add_137; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_137) : "f"(big_18b), "f"(0.5f)); + float _sub_138; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_138) : "f"(big_18b), "f"(0.5f)); + float rounded_18b = ((_abs_135 == 0.5f) ? ((above_18b > 0.0f) ? _add_137 : _sub_138) : whole_18b); + t1_18 = ((d1_18 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_18b, 7.888609e-31f), 1.7763568e-15f) : t1_18); + } + float2 _f32x2_mul_a_139 = mul_f32x2_noftz(_cvt_f32_37, make_float2(max_noftz(t0_18, 0.0f), max_noftz(t1_18, 0.0f))); + float2 _mul_18; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_18) : "l"(*(const unsigned long long*)&_f32x2_mul_a_139), "l"(*(const unsigned long long*)&_cvt_f32_38)); + packed[2] = __as_u32(__float22bfloat162_rn(make_float2(_mul_18.x, _mul_18.y))); + float2 _cvt_f32_39 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[3])); + float2 _cvt_f32_40 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[3])); + float e0_19 = approx_exp2((-_cvt_f32_39.x) * 1.442695f); + float e1_19 = approx_exp2((-_cvt_f32_39.y) * 1.442695f); + float d0_19 = 1.0f + e0_19; + float d1_19 = 1.0f + e1_19; + float r0_19 = approx_rcp(d0_19); + float r1_19 = approx_rcp(d1_19); + float _fma_140; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_140) : "f"(-d0_19), "f"(r0_19), "f"(1.0f)); + float _fma_6_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_6_1) : "f"(r0_19), "f"(_fma_140), "f"(r0_19)); + float _fma_141; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_141) : "f"(-d1_19), "f"(r1_19), "f"(1.0f)); + float _fma_7_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_7_1) : "f"(r1_19), "f"(_fma_141), "f"(r1_19)); + float t0_19 = _fma_6_1; + float t1_19 = _fma_7_1; + int _warp_vote_142 = __any_sync(0xFFFFFFFF, fminf(t0_19, t1_19) == 0.0f); + if (_warp_vote_142 != 0) { + float d4_19a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_19a) : "f"(d0_19), "f"(0.25f)); + float r4_19a = approx_rcp(d4_19a); + float _fma_144; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_144) : "f"(-d4_19a), "f"(r4_19a), "f"(1.0f)); + float _fma_143; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_143) : "f"(r4_19a), "f"(_fma_144), "f"(r4_19a)); + float big_19a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_19a) : "f"(mul_f32_rn_noftz(_fma_143, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_145; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_145) : "f"(big_19a), "f"(8.388608e+06f)); + float whole_19a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_19a) : "f"(_add_145), "f"(8.388608e+06f)); + float above_19a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_19a) : "f"(-d4_19a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_19a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_147; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_147) : "f"(big_19a), "f"(whole_19a)); + float _abs_146; + asm("abs.f32 %0, %1;" : "=f"(_abs_146) : "f"(_sub_147)); + float _add_148; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_148) : "f"(big_19a), "f"(0.5f)); + float _sub_149; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_149) : "f"(big_19a), "f"(0.5f)); + float rounded_19a = ((_abs_146 == 0.5f) ? ((above_19a > 0.0f) ? _add_148 : _sub_149) : whole_19a); + t0_19 = ((d0_19 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_19a, 7.888609e-31f), 1.7763568e-15f) : t0_19); + float d4_19b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_19b) : "f"(d1_19), "f"(0.25f)); + float r4_19b = approx_rcp(d4_19b); + float _fma_151; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_151) : "f"(-d4_19b), "f"(r4_19b), "f"(1.0f)); + float _fma_150; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_150) : "f"(r4_19b), "f"(_fma_151), "f"(r4_19b)); + float big_19b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_19b) : "f"(mul_f32_rn_noftz(_fma_150, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_152; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_152) : "f"(big_19b), "f"(8.388608e+06f)); + float whole_19b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_19b) : "f"(_add_152), "f"(8.388608e+06f)); + float above_19b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_19b) : "f"(-d4_19b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_19b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_154; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_154) : "f"(big_19b), "f"(whole_19b)); + float _abs_153; + asm("abs.f32 %0, %1;" : "=f"(_abs_153) : "f"(_sub_154)); + float _add_155; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_155) : "f"(big_19b), "f"(0.5f)); + float _sub_156; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_156) : "f"(big_19b), "f"(0.5f)); + float rounded_19b = ((_abs_153 == 0.5f) ? ((above_19b > 0.0f) ? _add_155 : _sub_156) : whole_19b); + t1_19 = ((d1_19 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_19b, 7.888609e-31f), 1.7763568e-15f) : t1_19); + } + float2 _f32x2_mul_a_157 = mul_f32x2_noftz(_cvt_f32_39, make_float2(max_noftz(t0_19, 0.0f), max_noftz(t1_19, 0.0f))); + float2 _mul_19; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_19) : "l"(*(const unsigned long long*)&_f32x2_mul_a_157), "l"(*(const unsigned long long*)&_cvt_f32_40)); + packed[3] = __as_u32(__float22bfloat162_rn(make_float2(_mul_19.x, _mul_19.y))); + float2 _cvt_f32_41 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[4])); + float2 _cvt_f32_42 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[4])); + float e0_20 = approx_exp2((-_cvt_f32_41.x) * 1.442695f); + float e1_20 = approx_exp2((-_cvt_f32_41.y) * 1.442695f); + float d0_20 = 1.0f + e0_20; + float d1_20 = 1.0f + e1_20; + float r0_20 = approx_rcp(d0_20); + float r1_20 = approx_rcp(d1_20); + float _fma_158; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_158) : "f"(-d0_20), "f"(r0_20), "f"(1.0f)); + float _fma_8_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_8_1) : "f"(r0_20), "f"(_fma_158), "f"(r0_20)); + float _fma_159; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_159) : "f"(-d1_20), "f"(r1_20), "f"(1.0f)); + float _fma_9_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_9_1) : "f"(r1_20), "f"(_fma_159), "f"(r1_20)); + float t0_20 = _fma_8_1; + float t1_20 = _fma_9_1; + int _warp_vote_160 = __any_sync(0xFFFFFFFF, fminf(t0_20, t1_20) == 0.0f); + if (_warp_vote_160 != 0) { + float d4_20a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_20a) : "f"(d0_20), "f"(0.25f)); + float r4_20a = approx_rcp(d4_20a); + float _fma_162; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_162) : "f"(-d4_20a), "f"(r4_20a), "f"(1.0f)); + float _fma_161; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_161) : "f"(r4_20a), "f"(_fma_162), "f"(r4_20a)); + float big_20a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_20a) : "f"(mul_f32_rn_noftz(_fma_161, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_163; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_163) : "f"(big_20a), "f"(8.388608e+06f)); + float whole_20a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_20a) : "f"(_add_163), "f"(8.388608e+06f)); + float above_20a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_20a) : "f"(-d4_20a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_20a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_165; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_165) : "f"(big_20a), "f"(whole_20a)); + float _abs_164; + asm("abs.f32 %0, %1;" : "=f"(_abs_164) : "f"(_sub_165)); + float _add_166; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_166) : "f"(big_20a), "f"(0.5f)); + float _sub_167; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_167) : "f"(big_20a), "f"(0.5f)); + float rounded_20a = ((_abs_164 == 0.5f) ? ((above_20a > 0.0f) ? _add_166 : _sub_167) : whole_20a); + t0_20 = ((d0_20 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_20a, 7.888609e-31f), 1.7763568e-15f) : t0_20); + float d4_20b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_20b) : "f"(d1_20), "f"(0.25f)); + float r4_20b = approx_rcp(d4_20b); + float _fma_169; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_169) : "f"(-d4_20b), "f"(r4_20b), "f"(1.0f)); + float _fma_168; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_168) : "f"(r4_20b), "f"(_fma_169), "f"(r4_20b)); + float big_20b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_20b) : "f"(mul_f32_rn_noftz(_fma_168, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_170; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_170) : "f"(big_20b), "f"(8.388608e+06f)); + float whole_20b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_20b) : "f"(_add_170), "f"(8.388608e+06f)); + float above_20b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_20b) : "f"(-d4_20b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_20b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_172; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_172) : "f"(big_20b), "f"(whole_20b)); + float _abs_171; + asm("abs.f32 %0, %1;" : "=f"(_abs_171) : "f"(_sub_172)); + float _add_173; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_173) : "f"(big_20b), "f"(0.5f)); + float _sub_174; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_174) : "f"(big_20b), "f"(0.5f)); + float rounded_20b = ((_abs_171 == 0.5f) ? ((above_20b > 0.0f) ? _add_173 : _sub_174) : whole_20b); + t1_20 = ((d1_20 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_20b, 7.888609e-31f), 1.7763568e-15f) : t1_20); + } + float2 _f32x2_mul_a_175 = mul_f32x2_noftz(_cvt_f32_41, make_float2(max_noftz(t0_20, 0.0f), max_noftz(t1_20, 0.0f))); + float2 _mul_20; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_175), "l"(*(const unsigned long long*)&_cvt_f32_42)); + packed[4] = __as_u32(__float22bfloat162_rn(make_float2(_mul_20.x, _mul_20.y))); + float2 _cvt_f32_43 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[5])); + float2 _cvt_f32_44 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[5])); + float e0_21 = approx_exp2((-_cvt_f32_43.x) * 1.442695f); + float e1_21 = approx_exp2((-_cvt_f32_43.y) * 1.442695f); + float d0_21 = 1.0f + e0_21; + float d1_21 = 1.0f + e1_21; + float r0_21 = approx_rcp(d0_21); + float r1_21 = approx_rcp(d1_21); + float _fma_176; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_176) : "f"(-d0_21), "f"(r0_21), "f"(1.0f)); + float _fma_10; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_10) : "f"(r0_21), "f"(_fma_176), "f"(r0_21)); + float _fma_177; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_177) : "f"(-d1_21), "f"(r1_21), "f"(1.0f)); + float _fma_11_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_11_1) : "f"(r1_21), "f"(_fma_177), "f"(r1_21)); + float t0_21 = _fma_10; + float t1_21 = _fma_11_1; + int _warp_vote_178 = __any_sync(0xFFFFFFFF, fminf(t0_21, t1_21) == 0.0f); + if (_warp_vote_178 != 0) { + float d4_21a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_21a) : "f"(d0_21), "f"(0.25f)); + float r4_21a = approx_rcp(d4_21a); + float _fma_180; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_180) : "f"(-d4_21a), "f"(r4_21a), "f"(1.0f)); + float _fma_179; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_179) : "f"(r4_21a), "f"(_fma_180), "f"(r4_21a)); + float big_21a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_21a) : "f"(mul_f32_rn_noftz(_fma_179, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_181; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_181) : "f"(big_21a), "f"(8.388608e+06f)); + float whole_21a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_21a) : "f"(_add_181), "f"(8.388608e+06f)); + float above_21a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_21a) : "f"(-d4_21a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_21a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_183; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_183) : "f"(big_21a), "f"(whole_21a)); + float _abs_182; + asm("abs.f32 %0, %1;" : "=f"(_abs_182) : "f"(_sub_183)); + float _add_184; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_184) : "f"(big_21a), "f"(0.5f)); + float _sub_185; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_185) : "f"(big_21a), "f"(0.5f)); + float rounded_21a = ((_abs_182 == 0.5f) ? ((above_21a > 0.0f) ? _add_184 : _sub_185) : whole_21a); + t0_21 = ((d0_21 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_21a, 7.888609e-31f), 1.7763568e-15f) : t0_21); + float d4_21b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_21b) : "f"(d1_21), "f"(0.25f)); + float r4_21b = approx_rcp(d4_21b); + float _fma_187; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_187) : "f"(-d4_21b), "f"(r4_21b), "f"(1.0f)); + float _fma_186; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_186) : "f"(r4_21b), "f"(_fma_187), "f"(r4_21b)); + float big_21b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_21b) : "f"(mul_f32_rn_noftz(_fma_186, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_188; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_188) : "f"(big_21b), "f"(8.388608e+06f)); + float whole_21b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_21b) : "f"(_add_188), "f"(8.388608e+06f)); + float above_21b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_21b) : "f"(-d4_21b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_21b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_190; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_190) : "f"(big_21b), "f"(whole_21b)); + float _abs_189; + asm("abs.f32 %0, %1;" : "=f"(_abs_189) : "f"(_sub_190)); + float _add_191; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_191) : "f"(big_21b), "f"(0.5f)); + float _sub_192; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_192) : "f"(big_21b), "f"(0.5f)); + float rounded_21b = ((_abs_189 == 0.5f) ? ((above_21b > 0.0f) ? _add_191 : _sub_192) : whole_21b); + t1_21 = ((d1_21 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_21b, 7.888609e-31f), 1.7763568e-15f) : t1_21); + } + float2 _f32x2_mul_a_193 = mul_f32x2_noftz(_cvt_f32_43, make_float2(max_noftz(t0_21, 0.0f), max_noftz(t1_21, 0.0f))); + float2 _mul_21; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_21) : "l"(*(const unsigned long long*)&_f32x2_mul_a_193), "l"(*(const unsigned long long*)&_cvt_f32_44)); + packed[5] = __as_u32(__float22bfloat162_rn(make_float2(_mul_21.x, _mul_21.y))); + float2 _cvt_f32_45 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[6])); + float2 _cvt_f32_46 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[6])); + float e0_22 = approx_exp2((-_cvt_f32_45.x) * 1.442695f); + float e1_22 = approx_exp2((-_cvt_f32_45.y) * 1.442695f); + float d0_22 = 1.0f + e0_22; + float d1_22 = 1.0f + e1_22; + float r0_22 = approx_rcp(d0_22); + float r1_22 = approx_rcp(d1_22); + float _fma_194; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_194) : "f"(-d0_22), "f"(r0_22), "f"(1.0f)); + float _fma_12_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_12_1) : "f"(r0_22), "f"(_fma_194), "f"(r0_22)); + float _fma_195; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_195) : "f"(-d1_22), "f"(r1_22), "f"(1.0f)); + float _fma_13_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_13_1) : "f"(r1_22), "f"(_fma_195), "f"(r1_22)); + float t0_22 = _fma_12_1; + float t1_22 = _fma_13_1; + int _warp_vote_196 = __any_sync(0xFFFFFFFF, fminf(t0_22, t1_22) == 0.0f); + if (_warp_vote_196 != 0) { + float d4_22a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_22a) : "f"(d0_22), "f"(0.25f)); + float r4_22a = approx_rcp(d4_22a); + float _fma_198; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_198) : "f"(-d4_22a), "f"(r4_22a), "f"(1.0f)); + float _fma_197; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_197) : "f"(r4_22a), "f"(_fma_198), "f"(r4_22a)); + float big_22a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_22a) : "f"(mul_f32_rn_noftz(_fma_197, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_199; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_199) : "f"(big_22a), "f"(8.388608e+06f)); + float whole_22a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_22a) : "f"(_add_199), "f"(8.388608e+06f)); + float above_22a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_22a) : "f"(-d4_22a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_22a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_201; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_201) : "f"(big_22a), "f"(whole_22a)); + float _abs_200; + asm("abs.f32 %0, %1;" : "=f"(_abs_200) : "f"(_sub_201)); + float _add_202; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_202) : "f"(big_22a), "f"(0.5f)); + float _sub_203; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_203) : "f"(big_22a), "f"(0.5f)); + float rounded_22a = ((_abs_200 == 0.5f) ? ((above_22a > 0.0f) ? _add_202 : _sub_203) : whole_22a); + t0_22 = ((d0_22 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_22a, 7.888609e-31f), 1.7763568e-15f) : t0_22); + float d4_22b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_22b) : "f"(d1_22), "f"(0.25f)); + float r4_22b = approx_rcp(d4_22b); + float _fma_205; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_205) : "f"(-d4_22b), "f"(r4_22b), "f"(1.0f)); + float _fma_204; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_204) : "f"(r4_22b), "f"(_fma_205), "f"(r4_22b)); + float big_22b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_22b) : "f"(mul_f32_rn_noftz(_fma_204, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_206; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_206) : "f"(big_22b), "f"(8.388608e+06f)); + float whole_22b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_22b) : "f"(_add_206), "f"(8.388608e+06f)); + float above_22b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_22b) : "f"(-d4_22b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_22b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_208; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_208) : "f"(big_22b), "f"(whole_22b)); + float _abs_207; + asm("abs.f32 %0, %1;" : "=f"(_abs_207) : "f"(_sub_208)); + float _add_209; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_209) : "f"(big_22b), "f"(0.5f)); + float _sub_210; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_210) : "f"(big_22b), "f"(0.5f)); + float rounded_22b = ((_abs_207 == 0.5f) ? ((above_22b > 0.0f) ? _add_209 : _sub_210) : whole_22b); + t1_22 = ((d1_22 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_22b, 7.888609e-31f), 1.7763568e-15f) : t1_22); + } + float2 _f32x2_mul_a_211 = mul_f32x2_noftz(_cvt_f32_45, make_float2(max_noftz(t0_22, 0.0f), max_noftz(t1_22, 0.0f))); + float2 _mul_22; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_211), "l"(*(const unsigned long long*)&_cvt_f32_46)); + packed[6] = __as_u32(__float22bfloat162_rn(make_float2(_mul_22.x, _mul_22.y))); + float2 _cvt_f32_47 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[7])); + float2 _cvt_f32_48 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[7])); + float e0_23 = approx_exp2((-_cvt_f32_47.x) * 1.442695f); + float e1_23 = approx_exp2((-_cvt_f32_47.y) * 1.442695f); + float d0_23 = 1.0f + e0_23; + float d1_23 = 1.0f + e1_23; + float r0_23 = approx_rcp(d0_23); + float r1_23 = approx_rcp(d1_23); + float _fma_212; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_212) : "f"(-d0_23), "f"(r0_23), "f"(1.0f)); + float _fma_14_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_14_1) : "f"(r0_23), "f"(_fma_212), "f"(r0_23)); + float _fma_213; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_213) : "f"(-d1_23), "f"(r1_23), "f"(1.0f)); + float _fma_15; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_15) : "f"(r1_23), "f"(_fma_213), "f"(r1_23)); + float t0_23 = _fma_14_1; + float t1_23 = _fma_15; + int _warp_vote_214 = __any_sync(0xFFFFFFFF, fminf(t0_23, t1_23) == 0.0f); + if (_warp_vote_214 != 0) { + float d4_23a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_23a) : "f"(d0_23), "f"(0.25f)); + float r4_23a = approx_rcp(d4_23a); + float _fma_216; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_216) : "f"(-d4_23a), "f"(r4_23a), "f"(1.0f)); + float _fma_215; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_215) : "f"(r4_23a), "f"(_fma_216), "f"(r4_23a)); + float big_23a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_23a) : "f"(mul_f32_rn_noftz(_fma_215, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_217; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_217) : "f"(big_23a), "f"(8.388608e+06f)); + float whole_23a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_23a) : "f"(_add_217), "f"(8.388608e+06f)); + float above_23a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_23a) : "f"(-d4_23a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_23a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_219; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_219) : "f"(big_23a), "f"(whole_23a)); + float _abs_218; + asm("abs.f32 %0, %1;" : "=f"(_abs_218) : "f"(_sub_219)); + float _add_220; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_220) : "f"(big_23a), "f"(0.5f)); + float _sub_221; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_221) : "f"(big_23a), "f"(0.5f)); + float rounded_23a = ((_abs_218 == 0.5f) ? ((above_23a > 0.0f) ? _add_220 : _sub_221) : whole_23a); + t0_23 = ((d0_23 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_23a, 7.888609e-31f), 1.7763568e-15f) : t0_23); + float d4_23b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_23b) : "f"(d1_23), "f"(0.25f)); + float r4_23b = approx_rcp(d4_23b); + float _fma_223; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_223) : "f"(-d4_23b), "f"(r4_23b), "f"(1.0f)); + float _fma_222; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_222) : "f"(r4_23b), "f"(_fma_223), "f"(r4_23b)); + float big_23b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_23b) : "f"(mul_f32_rn_noftz(_fma_222, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_224; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_224) : "f"(big_23b), "f"(8.388608e+06f)); + float whole_23b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_23b) : "f"(_add_224), "f"(8.388608e+06f)); + float above_23b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_23b) : "f"(-d4_23b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_23b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_226; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_226) : "f"(big_23b), "f"(whole_23b)); + float _abs_225; + asm("abs.f32 %0, %1;" : "=f"(_abs_225) : "f"(_sub_226)); + float _add_227; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_227) : "f"(big_23b), "f"(0.5f)); + float _sub_228; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_228) : "f"(big_23b), "f"(0.5f)); + float rounded_23b = ((_abs_225 == 0.5f) ? ((above_23b > 0.0f) ? _add_227 : _sub_228) : whole_23b); + t1_23 = ((d1_23 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_23b, 7.888609e-31f), 1.7763568e-15f) : t1_23); + } + float2 _f32x2_mul_a_229 = mul_f32x2_noftz(_cvt_f32_47, make_float2(max_noftz(t0_23, 0.0f), max_noftz(t1_23, 0.0f))); + float2 _mul_23; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_23) : "l"(*(const unsigned long long*)&_f32x2_mul_a_229), "l"(*(const unsigned long long*)&_cvt_f32_48)); + packed[7] = __as_u32(__float22bfloat162_rn(make_float2(_mul_23.x, _mul_23.y))); + int band_hi = 0; + float2 _cvt_f32_49 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[0])); + float2 _cvt_f32_50 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[0])); + float e0_24 = approx_exp2((-_cvt_f32_49.x) * 1.442695f); + float e1_24 = approx_exp2((-_cvt_f32_49.y) * 1.442695f); + float d0_24 = 1.0f + e0_24; + float d1_24 = 1.0f + e1_24; + float r0_24 = approx_rcp(d0_24); + float r1_24 = approx_rcp(d1_24); + float _fma_230; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_230) : "f"(-d0_24), "f"(r0_24), "f"(1.0f)); + float _fma_16_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_16_1) : "f"(r0_24), "f"(_fma_230), "f"(r0_24)); + float _fma_231; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_231) : "f"(-d1_24), "f"(r1_24), "f"(1.0f)); + float _fma_17_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_17_1) : "f"(r1_24), "f"(_fma_231), "f"(r1_24)); + float t0_24 = _fma_16_1; + float t1_24 = _fma_17_1; + int _warp_vote_232 = __any_sync(0xFFFFFFFF, fminf(t0_24, t1_24) == 0.0f); + if (_warp_vote_232 != 0) { + float d4_24a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_24a) : "f"(d0_24), "f"(0.25f)); + float r4_24a = approx_rcp(d4_24a); + float _fma_234; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_234) : "f"(-d4_24a), "f"(r4_24a), "f"(1.0f)); + float _fma_233; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_233) : "f"(r4_24a), "f"(_fma_234), "f"(r4_24a)); + float big_24a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_24a) : "f"(mul_f32_rn_noftz(_fma_233, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_235; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_235) : "f"(big_24a), "f"(8.388608e+06f)); + float whole_24a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_24a) : "f"(_add_235), "f"(8.388608e+06f)); + float above_24a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_24a) : "f"(-d4_24a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_24a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_237; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_237) : "f"(big_24a), "f"(whole_24a)); + float _abs_236; + asm("abs.f32 %0, %1;" : "=f"(_abs_236) : "f"(_sub_237)); + float _add_238; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_238) : "f"(big_24a), "f"(0.5f)); + float _sub_239; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_239) : "f"(big_24a), "f"(0.5f)); + float rounded_24a = ((_abs_236 == 0.5f) ? ((above_24a > 0.0f) ? _add_238 : _sub_239) : whole_24a); + t0_24 = ((d0_24 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_24a, 7.888609e-31f), 1.7763568e-15f) : t0_24); + float d4_24b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_24b) : "f"(d1_24), "f"(0.25f)); + float r4_24b = approx_rcp(d4_24b); + float _fma_241; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_241) : "f"(-d4_24b), "f"(r4_24b), "f"(1.0f)); + float _fma_240; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_240) : "f"(r4_24b), "f"(_fma_241), "f"(r4_24b)); + float big_24b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_24b) : "f"(mul_f32_rn_noftz(_fma_240, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_242; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_242) : "f"(big_24b), "f"(8.388608e+06f)); + float whole_24b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_24b) : "f"(_add_242), "f"(8.388608e+06f)); + float above_24b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_24b) : "f"(-d4_24b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_24b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_244; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_244) : "f"(big_24b), "f"(whole_24b)); + float _abs_243; + asm("abs.f32 %0, %1;" : "=f"(_abs_243) : "f"(_sub_244)); + float _add_245; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_245) : "f"(big_24b), "f"(0.5f)); + float _sub_246; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_246) : "f"(big_24b), "f"(0.5f)); + float rounded_24b = ((_abs_243 == 0.5f) ? ((above_24b > 0.0f) ? _add_245 : _sub_246) : whole_24b); + t1_24 = ((d1_24 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_24b, 7.888609e-31f), 1.7763568e-15f) : t1_24); + } + float2 _f32x2_mul_a_247 = mul_f32x2_noftz(_cvt_f32_49, make_float2(max_noftz(t0_24, 0.0f), max_noftz(t1_24, 0.0f))); + float2 _mul_24; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_24) : "l"(*(const unsigned long long*)&_f32x2_mul_a_247), "l"(*(const unsigned long long*)&_cvt_f32_50)); + packed_hi[0] = __as_u32(__float22bfloat162_rn(make_float2(_mul_24.x, _mul_24.y))); + float2 _cvt_f32_51 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[1])); + float2 _cvt_f32_52 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[1])); + float e0_25 = approx_exp2((-_cvt_f32_51.x) * 1.442695f); + float e1_25 = approx_exp2((-_cvt_f32_51.y) * 1.442695f); + float d0_25 = 1.0f + e0_25; + float d1_25 = 1.0f + e1_25; + float r0_25 = approx_rcp(d0_25); + float r1_25 = approx_rcp(d1_25); + float _fma_248; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_248) : "f"(-d0_25), "f"(r0_25), "f"(1.0f)); + float _fma_18_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_18_1) : "f"(r0_25), "f"(_fma_248), "f"(r0_25)); + float _fma_249; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_249) : "f"(-d1_25), "f"(r1_25), "f"(1.0f)); + float _fma_19_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_19_1) : "f"(r1_25), "f"(_fma_249), "f"(r1_25)); + float t0_25 = _fma_18_1; + float t1_25 = _fma_19_1; + int _warp_vote_250 = __any_sync(0xFFFFFFFF, fminf(t0_25, t1_25) == 0.0f); + if (_warp_vote_250 != 0) { + float d4_25a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_25a) : "f"(d0_25), "f"(0.25f)); + float r4_25a = approx_rcp(d4_25a); + float _fma_252; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_252) : "f"(-d4_25a), "f"(r4_25a), "f"(1.0f)); + float _fma_251; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_251) : "f"(r4_25a), "f"(_fma_252), "f"(r4_25a)); + float big_25a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_25a) : "f"(mul_f32_rn_noftz(_fma_251, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_253; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_253) : "f"(big_25a), "f"(8.388608e+06f)); + float whole_25a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_25a) : "f"(_add_253), "f"(8.388608e+06f)); + float above_25a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_25a) : "f"(-d4_25a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_25a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_255; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_255) : "f"(big_25a), "f"(whole_25a)); + float _abs_254; + asm("abs.f32 %0, %1;" : "=f"(_abs_254) : "f"(_sub_255)); + float _add_256; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_256) : "f"(big_25a), "f"(0.5f)); + float _sub_257; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_257) : "f"(big_25a), "f"(0.5f)); + float rounded_25a = ((_abs_254 == 0.5f) ? ((above_25a > 0.0f) ? _add_256 : _sub_257) : whole_25a); + t0_25 = ((d0_25 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_25a, 7.888609e-31f), 1.7763568e-15f) : t0_25); + float d4_25b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_25b) : "f"(d1_25), "f"(0.25f)); + float r4_25b = approx_rcp(d4_25b); + float _fma_259; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_259) : "f"(-d4_25b), "f"(r4_25b), "f"(1.0f)); + float _fma_258; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_258) : "f"(r4_25b), "f"(_fma_259), "f"(r4_25b)); + float big_25b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_25b) : "f"(mul_f32_rn_noftz(_fma_258, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_260; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_260) : "f"(big_25b), "f"(8.388608e+06f)); + float whole_25b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_25b) : "f"(_add_260), "f"(8.388608e+06f)); + float above_25b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_25b) : "f"(-d4_25b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_25b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_262; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_262) : "f"(big_25b), "f"(whole_25b)); + float _abs_261; + asm("abs.f32 %0, %1;" : "=f"(_abs_261) : "f"(_sub_262)); + float _add_263; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_263) : "f"(big_25b), "f"(0.5f)); + float _sub_264; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_264) : "f"(big_25b), "f"(0.5f)); + float rounded_25b = ((_abs_261 == 0.5f) ? ((above_25b > 0.0f) ? _add_263 : _sub_264) : whole_25b); + t1_25 = ((d1_25 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_25b, 7.888609e-31f), 1.7763568e-15f) : t1_25); + } + float2 _f32x2_mul_a_265 = mul_f32x2_noftz(_cvt_f32_51, make_float2(max_noftz(t0_25, 0.0f), max_noftz(t1_25, 0.0f))); + float2 _mul_25; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_25) : "l"(*(const unsigned long long*)&_f32x2_mul_a_265), "l"(*(const unsigned long long*)&_cvt_f32_52)); + packed_hi[1] = __as_u32(__float22bfloat162_rn(make_float2(_mul_25.x, _mul_25.y))); + float2 _cvt_f32_53 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[2])); + float2 _cvt_f32_54 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[2])); + float e0_26 = approx_exp2((-_cvt_f32_53.x) * 1.442695f); + float e1_26 = approx_exp2((-_cvt_f32_53.y) * 1.442695f); + float d0_26 = 1.0f + e0_26; + float d1_26 = 1.0f + e1_26; + float r0_26 = approx_rcp(d0_26); + float r1_26 = approx_rcp(d1_26); + float _fma_266; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_266) : "f"(-d0_26), "f"(r0_26), "f"(1.0f)); + float _fma_20; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_20) : "f"(r0_26), "f"(_fma_266), "f"(r0_26)); + float _fma_267; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_267) : "f"(-d1_26), "f"(r1_26), "f"(1.0f)); + float _fma_21_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_21_1) : "f"(r1_26), "f"(_fma_267), "f"(r1_26)); + float t0_26 = _fma_20; + float t1_26 = _fma_21_1; + int _warp_vote_268 = __any_sync(0xFFFFFFFF, fminf(t0_26, t1_26) == 0.0f); + if (_warp_vote_268 != 0) { + float d4_26a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_26a) : "f"(d0_26), "f"(0.25f)); + float r4_26a = approx_rcp(d4_26a); + float _fma_270; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_270) : "f"(-d4_26a), "f"(r4_26a), "f"(1.0f)); + float _fma_269; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_269) : "f"(r4_26a), "f"(_fma_270), "f"(r4_26a)); + float big_26a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_26a) : "f"(mul_f32_rn_noftz(_fma_269, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_271; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_271) : "f"(big_26a), "f"(8.388608e+06f)); + float whole_26a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_26a) : "f"(_add_271), "f"(8.388608e+06f)); + float above_26a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_26a) : "f"(-d4_26a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_26a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_273; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_273) : "f"(big_26a), "f"(whole_26a)); + float _abs_272; + asm("abs.f32 %0, %1;" : "=f"(_abs_272) : "f"(_sub_273)); + float _add_274; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_274) : "f"(big_26a), "f"(0.5f)); + float _sub_275; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_275) : "f"(big_26a), "f"(0.5f)); + float rounded_26a = ((_abs_272 == 0.5f) ? ((above_26a > 0.0f) ? _add_274 : _sub_275) : whole_26a); + t0_26 = ((d0_26 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_26a, 7.888609e-31f), 1.7763568e-15f) : t0_26); + float d4_26b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_26b) : "f"(d1_26), "f"(0.25f)); + float r4_26b = approx_rcp(d4_26b); + float _fma_277; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_277) : "f"(-d4_26b), "f"(r4_26b), "f"(1.0f)); + float _fma_276; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_276) : "f"(r4_26b), "f"(_fma_277), "f"(r4_26b)); + float big_26b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_26b) : "f"(mul_f32_rn_noftz(_fma_276, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_278; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_278) : "f"(big_26b), "f"(8.388608e+06f)); + float whole_26b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_26b) : "f"(_add_278), "f"(8.388608e+06f)); + float above_26b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_26b) : "f"(-d4_26b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_26b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_280; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_280) : "f"(big_26b), "f"(whole_26b)); + float _abs_279; + asm("abs.f32 %0, %1;" : "=f"(_abs_279) : "f"(_sub_280)); + float _add_281; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_281) : "f"(big_26b), "f"(0.5f)); + float _sub_282; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_282) : "f"(big_26b), "f"(0.5f)); + float rounded_26b = ((_abs_279 == 0.5f) ? ((above_26b > 0.0f) ? _add_281 : _sub_282) : whole_26b); + t1_26 = ((d1_26 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_26b, 7.888609e-31f), 1.7763568e-15f) : t1_26); + } + float2 _f32x2_mul_a_283 = mul_f32x2_noftz(_cvt_f32_53, make_float2(max_noftz(t0_26, 0.0f), max_noftz(t1_26, 0.0f))); + float2 _mul_26; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_26) : "l"(*(const unsigned long long*)&_f32x2_mul_a_283), "l"(*(const unsigned long long*)&_cvt_f32_54)); + packed_hi[2] = __as_u32(__float22bfloat162_rn(make_float2(_mul_26.x, _mul_26.y))); + float2 _cvt_f32_55 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[3])); + float2 _cvt_f32_56 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[3])); + float e0_27 = approx_exp2((-_cvt_f32_55.x) * 1.442695f); + float e1_27 = approx_exp2((-_cvt_f32_55.y) * 1.442695f); + float d0_27 = 1.0f + e0_27; + float d1_27 = 1.0f + e1_27; + float r0_27 = approx_rcp(d0_27); + float r1_27 = approx_rcp(d1_27); + float _fma_284; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_284) : "f"(-d0_27), "f"(r0_27), "f"(1.0f)); + float _fma_22_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_22_1) : "f"(r0_27), "f"(_fma_284), "f"(r0_27)); + float _fma_285; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_285) : "f"(-d1_27), "f"(r1_27), "f"(1.0f)); + float _fma_23_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_23_1) : "f"(r1_27), "f"(_fma_285), "f"(r1_27)); + float t0_27 = _fma_22_1; + float t1_27 = _fma_23_1; + int _warp_vote_286 = __any_sync(0xFFFFFFFF, fminf(t0_27, t1_27) == 0.0f); + if (_warp_vote_286 != 0) { + float d4_27a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_27a) : "f"(d0_27), "f"(0.25f)); + float r4_27a = approx_rcp(d4_27a); + float _fma_288; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_288) : "f"(-d4_27a), "f"(r4_27a), "f"(1.0f)); + float _fma_287; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_287) : "f"(r4_27a), "f"(_fma_288), "f"(r4_27a)); + float big_27a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_27a) : "f"(mul_f32_rn_noftz(_fma_287, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_289; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_289) : "f"(big_27a), "f"(8.388608e+06f)); + float whole_27a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_27a) : "f"(_add_289), "f"(8.388608e+06f)); + float above_27a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_27a) : "f"(-d4_27a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_27a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_291; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_291) : "f"(big_27a), "f"(whole_27a)); + float _abs_290; + asm("abs.f32 %0, %1;" : "=f"(_abs_290) : "f"(_sub_291)); + float _add_292; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_292) : "f"(big_27a), "f"(0.5f)); + float _sub_293; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_293) : "f"(big_27a), "f"(0.5f)); + float rounded_27a = ((_abs_290 == 0.5f) ? ((above_27a > 0.0f) ? _add_292 : _sub_293) : whole_27a); + t0_27 = ((d0_27 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_27a, 7.888609e-31f), 1.7763568e-15f) : t0_27); + float d4_27b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_27b) : "f"(d1_27), "f"(0.25f)); + float r4_27b = approx_rcp(d4_27b); + float _fma_295; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_295) : "f"(-d4_27b), "f"(r4_27b), "f"(1.0f)); + float _fma_294; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_294) : "f"(r4_27b), "f"(_fma_295), "f"(r4_27b)); + float big_27b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_27b) : "f"(mul_f32_rn_noftz(_fma_294, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_296; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_296) : "f"(big_27b), "f"(8.388608e+06f)); + float whole_27b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_27b) : "f"(_add_296), "f"(8.388608e+06f)); + float above_27b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_27b) : "f"(-d4_27b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_27b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_298; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_298) : "f"(big_27b), "f"(whole_27b)); + float _abs_297; + asm("abs.f32 %0, %1;" : "=f"(_abs_297) : "f"(_sub_298)); + float _add_299; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_299) : "f"(big_27b), "f"(0.5f)); + float _sub_300; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_300) : "f"(big_27b), "f"(0.5f)); + float rounded_27b = ((_abs_297 == 0.5f) ? ((above_27b > 0.0f) ? _add_299 : _sub_300) : whole_27b); + t1_27 = ((d1_27 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_27b, 7.888609e-31f), 1.7763568e-15f) : t1_27); + } + float2 _f32x2_mul_a_301 = mul_f32x2_noftz(_cvt_f32_55, make_float2(max_noftz(t0_27, 0.0f), max_noftz(t1_27, 0.0f))); + float2 _mul_27; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_27) : "l"(*(const unsigned long long*)&_f32x2_mul_a_301), "l"(*(const unsigned long long*)&_cvt_f32_56)); + packed_hi[3] = __as_u32(__float22bfloat162_rn(make_float2(_mul_27.x, _mul_27.y))); + float2 _cvt_f32_57 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[4])); + float2 _cvt_f32_58 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[4])); + float e0_28 = approx_exp2((-_cvt_f32_57.x) * 1.442695f); + float e1_28 = approx_exp2((-_cvt_f32_57.y) * 1.442695f); + float d0_28 = 1.0f + e0_28; + float d1_28 = 1.0f + e1_28; + float r0_28 = approx_rcp(d0_28); + float r1_28 = approx_rcp(d1_28); + float _fma_302; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_302) : "f"(-d0_28), "f"(r0_28), "f"(1.0f)); + float _fma_24_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_24_1) : "f"(r0_28), "f"(_fma_302), "f"(r0_28)); + float _fma_303; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_303) : "f"(-d1_28), "f"(r1_28), "f"(1.0f)); + float _fma_25; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_25) : "f"(r1_28), "f"(_fma_303), "f"(r1_28)); + float t0_28 = _fma_24_1; + float t1_28 = _fma_25; + int _warp_vote_304 = __any_sync(0xFFFFFFFF, fminf(t0_28, t1_28) == 0.0f); + if (_warp_vote_304 != 0) { + float d4_28a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_28a) : "f"(d0_28), "f"(0.25f)); + float r4_28a = approx_rcp(d4_28a); + float _fma_306; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_306) : "f"(-d4_28a), "f"(r4_28a), "f"(1.0f)); + float _fma_305; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_305) : "f"(r4_28a), "f"(_fma_306), "f"(r4_28a)); + float big_28a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_28a) : "f"(mul_f32_rn_noftz(_fma_305, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_307; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_307) : "f"(big_28a), "f"(8.388608e+06f)); + float whole_28a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_28a) : "f"(_add_307), "f"(8.388608e+06f)); + float above_28a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_28a) : "f"(-d4_28a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_28a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_309; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_309) : "f"(big_28a), "f"(whole_28a)); + float _abs_308; + asm("abs.f32 %0, %1;" : "=f"(_abs_308) : "f"(_sub_309)); + float _add_310; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_310) : "f"(big_28a), "f"(0.5f)); + float _sub_311; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_311) : "f"(big_28a), "f"(0.5f)); + float rounded_28a = ((_abs_308 == 0.5f) ? ((above_28a > 0.0f) ? _add_310 : _sub_311) : whole_28a); + t0_28 = ((d0_28 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_28a, 7.888609e-31f), 1.7763568e-15f) : t0_28); + float d4_28b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_28b) : "f"(d1_28), "f"(0.25f)); + float r4_28b = approx_rcp(d4_28b); + float _fma_313; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_313) : "f"(-d4_28b), "f"(r4_28b), "f"(1.0f)); + float _fma_312; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_312) : "f"(r4_28b), "f"(_fma_313), "f"(r4_28b)); + float big_28b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_28b) : "f"(mul_f32_rn_noftz(_fma_312, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_314; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_314) : "f"(big_28b), "f"(8.388608e+06f)); + float whole_28b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_28b) : "f"(_add_314), "f"(8.388608e+06f)); + float above_28b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_28b) : "f"(-d4_28b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_28b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_316; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_316) : "f"(big_28b), "f"(whole_28b)); + float _abs_315; + asm("abs.f32 %0, %1;" : "=f"(_abs_315) : "f"(_sub_316)); + float _add_317; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_317) : "f"(big_28b), "f"(0.5f)); + float _sub_318; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_318) : "f"(big_28b), "f"(0.5f)); + float rounded_28b = ((_abs_315 == 0.5f) ? ((above_28b > 0.0f) ? _add_317 : _sub_318) : whole_28b); + t1_28 = ((d1_28 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_28b, 7.888609e-31f), 1.7763568e-15f) : t1_28); + } + float2 _f32x2_mul_a_319 = mul_f32x2_noftz(_cvt_f32_57, make_float2(max_noftz(t0_28, 0.0f), max_noftz(t1_28, 0.0f))); + float2 _mul_28; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_28) : "l"(*(const unsigned long long*)&_f32x2_mul_a_319), "l"(*(const unsigned long long*)&_cvt_f32_58)); + packed_hi[4] = __as_u32(__float22bfloat162_rn(make_float2(_mul_28.x, _mul_28.y))); + float2 _cvt_f32_59 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[5])); + float2 _cvt_f32_60 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[5])); + float e0_29 = approx_exp2((-_cvt_f32_59.x) * 1.442695f); + float e1_29 = approx_exp2((-_cvt_f32_59.y) * 1.442695f); + float d0_29 = 1.0f + e0_29; + float d1_29 = 1.0f + e1_29; + float r0_29 = approx_rcp(d0_29); + float r1_29 = approx_rcp(d1_29); + float _fma_320; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_320) : "f"(-d0_29), "f"(r0_29), "f"(1.0f)); + float _fma_26_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_26_1) : "f"(r0_29), "f"(_fma_320), "f"(r0_29)); + float _fma_321; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_321) : "f"(-d1_29), "f"(r1_29), "f"(1.0f)); + float _fma_27_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_27_1) : "f"(r1_29), "f"(_fma_321), "f"(r1_29)); + float t0_29 = _fma_26_1; + float t1_29 = _fma_27_1; + int _warp_vote_322 = __any_sync(0xFFFFFFFF, fminf(t0_29, t1_29) == 0.0f); + if (_warp_vote_322 != 0) { + float d4_29a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_29a) : "f"(d0_29), "f"(0.25f)); + float r4_29a = approx_rcp(d4_29a); + float _fma_324; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_324) : "f"(-d4_29a), "f"(r4_29a), "f"(1.0f)); + float _fma_323; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_323) : "f"(r4_29a), "f"(_fma_324), "f"(r4_29a)); + float big_29a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_29a) : "f"(mul_f32_rn_noftz(_fma_323, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_325; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_325) : "f"(big_29a), "f"(8.388608e+06f)); + float whole_29a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_29a) : "f"(_add_325), "f"(8.388608e+06f)); + float above_29a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_29a) : "f"(-d4_29a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_29a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_327; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_327) : "f"(big_29a), "f"(whole_29a)); + float _abs_326; + asm("abs.f32 %0, %1;" : "=f"(_abs_326) : "f"(_sub_327)); + float _add_328; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_328) : "f"(big_29a), "f"(0.5f)); + float _sub_329; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_329) : "f"(big_29a), "f"(0.5f)); + float rounded_29a = ((_abs_326 == 0.5f) ? ((above_29a > 0.0f) ? _add_328 : _sub_329) : whole_29a); + t0_29 = ((d0_29 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_29a, 7.888609e-31f), 1.7763568e-15f) : t0_29); + float d4_29b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_29b) : "f"(d1_29), "f"(0.25f)); + float r4_29b = approx_rcp(d4_29b); + float _fma_331; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_331) : "f"(-d4_29b), "f"(r4_29b), "f"(1.0f)); + float _fma_330; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_330) : "f"(r4_29b), "f"(_fma_331), "f"(r4_29b)); + float big_29b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_29b) : "f"(mul_f32_rn_noftz(_fma_330, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_332; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_332) : "f"(big_29b), "f"(8.388608e+06f)); + float whole_29b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_29b) : "f"(_add_332), "f"(8.388608e+06f)); + float above_29b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_29b) : "f"(-d4_29b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_29b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_334; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_334) : "f"(big_29b), "f"(whole_29b)); + float _abs_333; + asm("abs.f32 %0, %1;" : "=f"(_abs_333) : "f"(_sub_334)); + float _add_335; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_335) : "f"(big_29b), "f"(0.5f)); + float _sub_336; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_336) : "f"(big_29b), "f"(0.5f)); + float rounded_29b = ((_abs_333 == 0.5f) ? ((above_29b > 0.0f) ? _add_335 : _sub_336) : whole_29b); + t1_29 = ((d1_29 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_29b, 7.888609e-31f), 1.7763568e-15f) : t1_29); + } + float2 _f32x2_mul_a_337 = mul_f32x2_noftz(_cvt_f32_59, make_float2(max_noftz(t0_29, 0.0f), max_noftz(t1_29, 0.0f))); + float2 _mul_29; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_29) : "l"(*(const unsigned long long*)&_f32x2_mul_a_337), "l"(*(const unsigned long long*)&_cvt_f32_60)); + packed_hi[5] = __as_u32(__float22bfloat162_rn(make_float2(_mul_29.x, _mul_29.y))); + float2 _cvt_f32_61 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[6])); + float2 _cvt_f32_62 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[6])); + float e0_30 = approx_exp2((-_cvt_f32_61.x) * 1.442695f); + float e1_30 = approx_exp2((-_cvt_f32_61.y) * 1.442695f); + float d0_30 = 1.0f + e0_30; + float d1_30 = 1.0f + e1_30; + float r0_30 = approx_rcp(d0_30); + float r1_30 = approx_rcp(d1_30); + float _fma_338; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_338) : "f"(-d0_30), "f"(r0_30), "f"(1.0f)); + float _fma_28_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_28_1) : "f"(r0_30), "f"(_fma_338), "f"(r0_30)); + float _fma_339; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_339) : "f"(-d1_30), "f"(r1_30), "f"(1.0f)); + float _fma_29_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_29_1) : "f"(r1_30), "f"(_fma_339), "f"(r1_30)); + float t0_30 = _fma_28_1; + float t1_30 = _fma_29_1; + int _warp_vote_340 = __any_sync(0xFFFFFFFF, fminf(t0_30, t1_30) == 0.0f); + if (_warp_vote_340 != 0) { + float d4_30a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_30a) : "f"(d0_30), "f"(0.25f)); + float r4_30a = approx_rcp(d4_30a); + float _fma_342; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_342) : "f"(-d4_30a), "f"(r4_30a), "f"(1.0f)); + float _fma_341; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_341) : "f"(r4_30a), "f"(_fma_342), "f"(r4_30a)); + float big_30a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_30a) : "f"(mul_f32_rn_noftz(_fma_341, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_343; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_343) : "f"(big_30a), "f"(8.388608e+06f)); + float whole_30a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_30a) : "f"(_add_343), "f"(8.388608e+06f)); + float above_30a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_30a) : "f"(-d4_30a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_30a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_345; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_345) : "f"(big_30a), "f"(whole_30a)); + float _abs_344; + asm("abs.f32 %0, %1;" : "=f"(_abs_344) : "f"(_sub_345)); + float _add_346; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_346) : "f"(big_30a), "f"(0.5f)); + float _sub_347; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_347) : "f"(big_30a), "f"(0.5f)); + float rounded_30a = ((_abs_344 == 0.5f) ? ((above_30a > 0.0f) ? _add_346 : _sub_347) : whole_30a); + t0_30 = ((d0_30 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_30a, 7.888609e-31f), 1.7763568e-15f) : t0_30); + float d4_30b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_30b) : "f"(d1_30), "f"(0.25f)); + float r4_30b = approx_rcp(d4_30b); + float _fma_349; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_349) : "f"(-d4_30b), "f"(r4_30b), "f"(1.0f)); + float _fma_348; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_348) : "f"(r4_30b), "f"(_fma_349), "f"(r4_30b)); + float big_30b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_30b) : "f"(mul_f32_rn_noftz(_fma_348, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_350; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_350) : "f"(big_30b), "f"(8.388608e+06f)); + float whole_30b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_30b) : "f"(_add_350), "f"(8.388608e+06f)); + float above_30b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_30b) : "f"(-d4_30b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_30b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_352; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_352) : "f"(big_30b), "f"(whole_30b)); + float _abs_351; + asm("abs.f32 %0, %1;" : "=f"(_abs_351) : "f"(_sub_352)); + float _add_353; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_353) : "f"(big_30b), "f"(0.5f)); + float _sub_354; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_354) : "f"(big_30b), "f"(0.5f)); + float rounded_30b = ((_abs_351 == 0.5f) ? ((above_30b > 0.0f) ? _add_353 : _sub_354) : whole_30b); + t1_30 = ((d1_30 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_30b, 7.888609e-31f), 1.7763568e-15f) : t1_30); + } + float2 _f32x2_mul_a_355 = mul_f32x2_noftz(_cvt_f32_61, make_float2(max_noftz(t0_30, 0.0f), max_noftz(t1_30, 0.0f))); + float2 _mul_30; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_30) : "l"(*(const unsigned long long*)&_f32x2_mul_a_355), "l"(*(const unsigned long long*)&_cvt_f32_62)); + packed_hi[6] = __as_u32(__float22bfloat162_rn(make_float2(_mul_30.x, _mul_30.y))); + float2 _cvt_f32_63 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[7])); + float2 _cvt_f32_64 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[7])); + float e0_31 = approx_exp2((-_cvt_f32_63.x) * 1.442695f); + float e1_31 = approx_exp2((-_cvt_f32_63.y) * 1.442695f); + float d0_31 = 1.0f + e0_31; + float d1_31 = 1.0f + e1_31; + float r0_31 = approx_rcp(d0_31); + float r1_31 = approx_rcp(d1_31); + float _fma_356; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_356) : "f"(-d0_31), "f"(r0_31), "f"(1.0f)); + float _fma_30; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_30) : "f"(r0_31), "f"(_fma_356), "f"(r0_31)); + float _fma_357; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_357) : "f"(-d1_31), "f"(r1_31), "f"(1.0f)); + float _fma_31_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_31_1) : "f"(r1_31), "f"(_fma_357), "f"(r1_31)); + float t0_31 = _fma_30; + float t1_31 = _fma_31_1; + int _warp_vote_358 = __any_sync(0xFFFFFFFF, fminf(t0_31, t1_31) == 0.0f); + if (_warp_vote_358 != 0) { + float d4_31a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_31a) : "f"(d0_31), "f"(0.25f)); + float r4_31a = approx_rcp(d4_31a); + float _fma_360; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_360) : "f"(-d4_31a), "f"(r4_31a), "f"(1.0f)); + float _fma_359; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_359) : "f"(r4_31a), "f"(_fma_360), "f"(r4_31a)); + float big_31a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_31a) : "f"(mul_f32_rn_noftz(_fma_359, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_361; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_361) : "f"(big_31a), "f"(8.388608e+06f)); + float whole_31a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_31a) : "f"(_add_361), "f"(8.388608e+06f)); + float above_31a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_31a) : "f"(-d4_31a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_31a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_363; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_363) : "f"(big_31a), "f"(whole_31a)); + float _abs_362; + asm("abs.f32 %0, %1;" : "=f"(_abs_362) : "f"(_sub_363)); + float _add_364; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_364) : "f"(big_31a), "f"(0.5f)); + float _sub_365; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_365) : "f"(big_31a), "f"(0.5f)); + float rounded_31a = ((_abs_362 == 0.5f) ? ((above_31a > 0.0f) ? _add_364 : _sub_365) : whole_31a); + t0_31 = ((d0_31 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_31a, 7.888609e-31f), 1.7763568e-15f) : t0_31); + float d4_31b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_31b) : "f"(d1_31), "f"(0.25f)); + float r4_31b = approx_rcp(d4_31b); + float _fma_367; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_367) : "f"(-d4_31b), "f"(r4_31b), "f"(1.0f)); + float _fma_366; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_366) : "f"(r4_31b), "f"(_fma_367), "f"(r4_31b)); + float big_31b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_31b) : "f"(mul_f32_rn_noftz(_fma_366, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_368; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_368) : "f"(big_31b), "f"(8.388608e+06f)); + float whole_31b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_31b) : "f"(_add_368), "f"(8.388608e+06f)); + float above_31b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_31b) : "f"(-d4_31b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_31b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_370; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_370) : "f"(big_31b), "f"(whole_31b)); + float _abs_369; + asm("abs.f32 %0, %1;" : "=f"(_abs_369) : "f"(_sub_370)); + float _add_371; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_371) : "f"(big_31b), "f"(0.5f)); + float _sub_372; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_372) : "f"(big_31b), "f"(0.5f)); + float rounded_31b = ((_abs_369 == 0.5f) ? ((above_31b > 0.0f) ? _add_371 : _sub_372) : whole_31b); + t1_31 = ((d1_31 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_31b, 7.888609e-31f), 1.7763568e-15f) : t1_31); + } + float2 _f32x2_mul_a_373 = mul_f32x2_noftz(_cvt_f32_63, make_float2(max_noftz(t0_31, 0.0f), max_noftz(t1_31, 0.0f))); + float2 _mul_31; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_31) : "l"(*(const unsigned long long*)&_f32x2_mul_a_373), "l"(*(const unsigned long long*)&_cvt_f32_64)); + packed_hi[7] = __as_u32(__float22bfloat162_rn(make_float2(_mul_31.x, _mul_31.y))); + } + unsigned int swizzle = (unsigned int)(row >> 3 & 6 | row & 1); + unsigned int row_base = tile_addr + (unsigned int)(row * 256); + unsigned int chunk = (unsigned int)((tid & 7) * 2); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base + ((chunk ^ swizzle) << 4)), "r"(packed[0]), "r"(packed[1]), "r"(packed[2]), "r"(packed[3]) : "memory"); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base + ((chunk + 1 ^ swizzle) << 4)), "r"(packed[4]), "r"(packed[5]), "r"(packed[6]), "r"(packed[7]) : "memory"); + unsigned int swizzle_hi = swizzle ^ 4; + unsigned int row_base_hi = row_base + 8192; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base_hi + ((chunk ^ swizzle_hi) << 4)), "r"(packed_hi[0]), "r"(packed_hi[1]), "r"(packed_hi[2]), "r"(packed_hi[3]) : "memory"); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base_hi + ((chunk + 1 ^ swizzle_hi) << 4)), "r"(packed_hi[4]), "r"(packed_hi[5]), "r"(packed_hi[6]), "r"(packed_hi[7]) : "memory"); + __syncthreads(); + int lane_1 = tid & 31; + int warp_1 = tid >> 5; + int k8 = lane_1 & 7; + int srow = (k8 >> 1) * 16 + (lane_1 >> 3) * 2 + (k8 & 1); + unsigned int sswizzle = (unsigned int)(srow >> 3 & 6 | srow & 1); + unsigned int cols[16]; + unsigned int strip_chunk0 = (unsigned int)(warp_1 * 2); + unsigned int strip_src0 = tile_addr + (unsigned int)(srow * 256) + ((strip_chunk0 ^ sswizzle) << 4); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols[0]), "=r"(cols[1]), "=r"(cols[2]), "=r"(cols[3]) + : "r"(strip_src0) + : "memory"); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols[4]), "=r"(cols[5]), "=r"(cols[6]), "=r"(cols[7]) + : "r"(strip_src0 + 2048) + : "memory"); + unsigned int strip_chunk1 = (unsigned int)(warp_1 * 2 + 1); + unsigned int strip_src1 = tile_addr + (unsigned int)(srow * 256) + ((strip_chunk1 ^ sswizzle) << 4); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols[8]), "=r"(cols[9]), "=r"(cols[10]), "=r"(cols[11]) + : "r"(strip_src1) + : "memory"); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols[12]), "=r"(cols[13]), "=r"(cols[14]), "=r"(cols[15]) + : "r"(strip_src1 + 2048) + : "memory"); + __nv_bfloat162 amax_a = __as_bf16x2(0); + __nv_bfloat162 amax_b = __as_bf16x2(0); + __nv_bfloat162 _hxa_375 = amax_a; + __nv_bfloat162 _hxb_375 = __as_bf16x2(cols[0]); + unsigned int _hxr_375; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_375) : "r"(*reinterpret_cast(&_hxa_375)), "r"(*reinterpret_cast(&_hxb_375))); + __nv_bfloat162 _max_374 = *reinterpret_cast(&_hxr_375); + amax_a = _max_374; + __nv_bfloat162 _hxa_377 = amax_b; + __nv_bfloat162 _hxb_377 = __as_bf16x2(cols[8]); + unsigned int _hxr_377; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_377) : "r"(*reinterpret_cast(&_hxa_377)), "r"(*reinterpret_cast(&_hxb_377))); + __nv_bfloat162 _max_376 = *reinterpret_cast(&_hxr_377); + amax_b = _max_376; + __nv_bfloat162 _hxa_379 = amax_a; + __nv_bfloat162 _hxb_379 = __as_bf16x2(cols[1]); + unsigned int _hxr_379; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_379) : "r"(*reinterpret_cast(&_hxa_379)), "r"(*reinterpret_cast(&_hxb_379))); + __nv_bfloat162 _max_378 = *reinterpret_cast(&_hxr_379); + amax_a = _max_378; + __nv_bfloat162 _hxa_381 = amax_b; + __nv_bfloat162 _hxb_381 = __as_bf16x2(cols[9]); + unsigned int _hxr_381; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_381) : "r"(*reinterpret_cast(&_hxa_381)), "r"(*reinterpret_cast(&_hxb_381))); + __nv_bfloat162 _max_380 = *reinterpret_cast(&_hxr_381); + amax_b = _max_380; + __nv_bfloat162 _hxa_383 = amax_a; + __nv_bfloat162 _hxb_383 = __as_bf16x2(cols[2]); + unsigned int _hxr_383; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_383) : "r"(*reinterpret_cast(&_hxa_383)), "r"(*reinterpret_cast(&_hxb_383))); + __nv_bfloat162 _max_382 = *reinterpret_cast(&_hxr_383); + amax_a = _max_382; + __nv_bfloat162 _hxa_385 = amax_b; + __nv_bfloat162 _hxb_385 = __as_bf16x2(cols[10]); + unsigned int _hxr_385; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_385) : "r"(*reinterpret_cast(&_hxa_385)), "r"(*reinterpret_cast(&_hxb_385))); + __nv_bfloat162 _max_384 = *reinterpret_cast(&_hxr_385); + amax_b = _max_384; + __nv_bfloat162 _hxa_387 = amax_a; + __nv_bfloat162 _hxb_387 = __as_bf16x2(cols[3]); + unsigned int _hxr_387; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_387) : "r"(*reinterpret_cast(&_hxa_387)), "r"(*reinterpret_cast(&_hxb_387))); + __nv_bfloat162 _max_386 = *reinterpret_cast(&_hxr_387); + amax_a = _max_386; + __nv_bfloat162 _hxa_389 = amax_b; + __nv_bfloat162 _hxb_389 = __as_bf16x2(cols[11]); + unsigned int _hxr_389; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_389) : "r"(*reinterpret_cast(&_hxa_389)), "r"(*reinterpret_cast(&_hxb_389))); + __nv_bfloat162 _max_388 = *reinterpret_cast(&_hxr_389); + amax_b = _max_388; + __nv_bfloat162 _hxa_391 = amax_a; + __nv_bfloat162 _hxb_391 = __as_bf16x2(cols[4]); + unsigned int _hxr_391; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_391) : "r"(*reinterpret_cast(&_hxa_391)), "r"(*reinterpret_cast(&_hxb_391))); + __nv_bfloat162 _max_390 = *reinterpret_cast(&_hxr_391); + amax_a = _max_390; + __nv_bfloat162 _hxa_393 = amax_b; + __nv_bfloat162 _hxb_393 = __as_bf16x2(cols[12]); + unsigned int _hxr_393; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_393) : "r"(*reinterpret_cast(&_hxa_393)), "r"(*reinterpret_cast(&_hxb_393))); + __nv_bfloat162 _max_392 = *reinterpret_cast(&_hxr_393); + amax_b = _max_392; + __nv_bfloat162 _hxa_395 = amax_a; + __nv_bfloat162 _hxb_395 = __as_bf16x2(cols[5]); + unsigned int _hxr_395; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_395) : "r"(*reinterpret_cast(&_hxa_395)), "r"(*reinterpret_cast(&_hxb_395))); + __nv_bfloat162 _max_394 = *reinterpret_cast(&_hxr_395); + amax_a = _max_394; + __nv_bfloat162 _hxa_397 = amax_b; + __nv_bfloat162 _hxb_397 = __as_bf16x2(cols[13]); + unsigned int _hxr_397; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_397) : "r"(*reinterpret_cast(&_hxa_397)), "r"(*reinterpret_cast(&_hxb_397))); + __nv_bfloat162 _max_396 = *reinterpret_cast(&_hxr_397); + amax_b = _max_396; + __nv_bfloat162 _hxa_399 = amax_a; + __nv_bfloat162 _hxb_399 = __as_bf16x2(cols[6]); + unsigned int _hxr_399; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_399) : "r"(*reinterpret_cast(&_hxa_399)), "r"(*reinterpret_cast(&_hxb_399))); + __nv_bfloat162 _max_398 = *reinterpret_cast(&_hxr_399); + amax_a = _max_398; + __nv_bfloat162 _hxa_401 = amax_b; + __nv_bfloat162 _hxb_401 = __as_bf16x2(cols[14]); + unsigned int _hxr_401; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_401) : "r"(*reinterpret_cast(&_hxa_401)), "r"(*reinterpret_cast(&_hxb_401))); + __nv_bfloat162 _max_400 = *reinterpret_cast(&_hxr_401); + amax_b = _max_400; + __nv_bfloat162 _hxa_403 = amax_a; + __nv_bfloat162 _hxb_403 = __as_bf16x2(cols[7]); + unsigned int _hxr_403; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_403) : "r"(*reinterpret_cast(&_hxa_403)), "r"(*reinterpret_cast(&_hxb_403))); + __nv_bfloat162 _max_402 = *reinterpret_cast(&_hxr_403); + amax_a = _max_402; + __nv_bfloat162 _hxa_405 = amax_b; + __nv_bfloat162 _hxb_405 = __as_bf16x2(cols[15]); + unsigned int _hxr_405; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_405) : "r"(*reinterpret_cast(&_hxa_405)), "r"(*reinterpret_cast(&_hxb_405))); + __nv_bfloat162 _max_404 = *reinterpret_cast(&_hxr_405); + amax_b = _max_404; + unsigned int mag_a = __as_u32(amax_a) & 2147450879; + unsigned int mag_b = __as_u32(amax_b) & 2147450879; + unsigned int lows_ab = mag_a & 65535 | mag_b << 16; + unsigned int highs_ab = mag_a >> 16 | mag_b & 4294901760u; + __nv_bfloat162 col_max = __hmax2_nan(__as_bf16x2(lows_ab), __as_bf16x2(highs_ab)); + unsigned int peer1 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(col_max), 1); + col_max = __hmax2_nan(col_max, __as_bf16x2(peer1)); + unsigned int col_max_bits = __as_u32(col_max); + unsigned int amax_bits = (col_max_bits & 65535) << 16; + int scale_i32 = (int)(amax_bits + 2031616 >> 23) - 8; + if (scale_i32 < 0) { + scale_i32 = 0; + } + if ((amax_bits & 2139095040) == 2139095040) { + scale_i32 = 255; + } + unsigned int scale = (unsigned int)scale_i32; + unsigned int inv = 254 - scale << 7; + if (scale == 255) { + inv = 32704; + } + unsigned int inv2 = inv | inv << 16; + unsigned int amax_bits_0 = (col_max_bits >> 16 & 65535) << 16; + int scale_i32_1 = (int)(amax_bits_0 + 2031616 >> 23) - 8; + if (scale_i32_1 < 0) { + scale_i32_1 = 0; + } + if ((amax_bits_0 & 2139095040) == 2139095040) { + scale_i32_1 = 255; + } + unsigned int scale_2 = (unsigned int)scale_i32_1; + unsigned int inv_3 = 254 - scale_2 << 7; + if (scale_2 == 255) { + inv_3 = 32704; + } + unsigned int inv2_4 = inv_3 | inv_3 << 16; + int gcol_a = bx * 128 + warp_1 * 16 + (lane_1 >> 2); + int gcol_b = gcol_a + 8; + int out_row = by * 64 + (lane_1 & 3) * 16; + unsigned int words_a[4]; + unsigned int words_b[4]; + uint32_t _bf16x2_mul_0; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_0) : "r"(cols[0]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_0; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_0) : "r"(_bf16x2_mul_0)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_0); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_0) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_1; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_1) : "r"(cols[1]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_407; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_407) : "r"(_bf16x2_mul_1)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_1); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_407) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_406; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_406) : "h"(_e4m3x2_bf16x2_0), "h"(_cvt_satfinite_e4m3x2_bf16x2_407)); + words_a[0] = _mov_b32_pack_u16x2_406; + uint32_t _bf16x2_mul_2; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_2) : "r"(cols[8]), "r"(inv2_4)); + uint16_t _e4m3x2_bf16x2_1; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_1) : "r"(_bf16x2_mul_2)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_2); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_1) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_3; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_3) : "r"(cols[9]), "r"(inv2_4)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_409; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_409) : "r"(_bf16x2_mul_3)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_3); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_409) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_408; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_408) : "h"(_e4m3x2_bf16x2_1), "h"(_cvt_satfinite_e4m3x2_bf16x2_409)); + words_b[0] = _mov_b32_pack_u16x2_408; + uint32_t _bf16x2_mul_4; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_4) : "r"(cols[2]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_2; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_2) : "r"(_bf16x2_mul_4)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_4); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_2) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_5; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_5) : "r"(cols[3]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_411; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_411) : "r"(_bf16x2_mul_5)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_5); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_411) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_410; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_410) : "h"(_e4m3x2_bf16x2_2), "h"(_cvt_satfinite_e4m3x2_bf16x2_411)); + words_a[1] = _mov_b32_pack_u16x2_410; + uint32_t _bf16x2_mul_6; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_6) : "r"(cols[10]), "r"(inv2_4)); + uint16_t _e4m3x2_bf16x2_3; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_3) : "r"(_bf16x2_mul_6)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_6); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_3) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_7; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_7) : "r"(cols[11]), "r"(inv2_4)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_413; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_413) : "r"(_bf16x2_mul_7)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_7); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_413) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_412; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_412) : "h"(_e4m3x2_bf16x2_3), "h"(_cvt_satfinite_e4m3x2_bf16x2_413)); + words_b[1] = _mov_b32_pack_u16x2_412; + uint32_t _bf16x2_mul_8; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_8) : "r"(cols[4]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_4; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_4) : "r"(_bf16x2_mul_8)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_8); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_4) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_9; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_9) : "r"(cols[5]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_415; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_415) : "r"(_bf16x2_mul_9)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_9); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_415) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_414; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_414) : "h"(_e4m3x2_bf16x2_4), "h"(_cvt_satfinite_e4m3x2_bf16x2_415)); + words_a[2] = _mov_b32_pack_u16x2_414; + uint32_t _bf16x2_mul_10; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_10) : "r"(cols[12]), "r"(inv2_4)); + uint16_t _e4m3x2_bf16x2_5; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_5) : "r"(_bf16x2_mul_10)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_10); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_5) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_11; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_11) : "r"(cols[13]), "r"(inv2_4)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_417; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_417) : "r"(_bf16x2_mul_11)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_11); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_417) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_416; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_416) : "h"(_e4m3x2_bf16x2_5), "h"(_cvt_satfinite_e4m3x2_bf16x2_417)); + words_b[2] = _mov_b32_pack_u16x2_416; + uint32_t _bf16x2_mul_12; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_12) : "r"(cols[6]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_6; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_6) : "r"(_bf16x2_mul_12)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_12); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_6) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_13; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_13) : "r"(cols[7]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_419; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_419) : "r"(_bf16x2_mul_13)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_13); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_419) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_418; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_418) : "h"(_e4m3x2_bf16x2_6), "h"(_cvt_satfinite_e4m3x2_bf16x2_419)); + words_a[3] = _mov_b32_pack_u16x2_418; + uint32_t _bf16x2_mul_14; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_14) : "r"(cols[14]), "r"(inv2_4)); + uint16_t _e4m3x2_bf16x2_7; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_7) : "r"(_bf16x2_mul_14)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_14); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_7) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_15; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_15) : "r"(cols[15]), "r"(inv2_4)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_421; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_421) : "r"(_bf16x2_mul_15)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_15); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_421) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_420; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_420) : "h"(_e4m3x2_bf16x2_7), "h"(_cvt_satfinite_e4m3x2_bf16x2_421)); + words_b[3] = _mov_b32_pack_u16x2_420; + reinterpret_cast(col_output + (gcol_a * M + out_row))[0] = reinterpret_cast(words_a + 0)[0]; + reinterpret_cast(col_output + (gcol_b * M + out_row))[0] = reinterpret_cast(words_b + 0)[0]; + if ((lane_1 & 1) == 0) { + int row_tiles = M >> 7; + int rblk = by * 2 + ((lane_1 & 3) >> 1); + int col_scale_a = ((gcol_a >> 7) * row_tiles + (rblk >> 2)) * 512 + (gcol_a & 31) * 16 + (gcol_a >> 5 & 3) * 4 + (rblk & 3); + int col_scale_b = ((gcol_b >> 7) * row_tiles + (rblk >> 2)) * 512 + (gcol_b & 31) * 16 + (gcol_b >> 5 & 3) * 4 + (rblk & 3); + *(reinterpret_cast(col_scales + col_scale_a) + (0)) = (unsigned char)(scale); + *(reinterpret_cast(col_scales + col_scale_b) + (0)) = (unsigned char)(scale_2); + } +} + +} // extern "C" diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_383546f101950463dcf8_binding.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_383546f101950463dcf8_binding.cu new file mode 100644 index 00000000000..7dcd8727de6 --- /dev/null +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_383546f101950463dcf8_binding.cu @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2026 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gated_act_mxfp8_383546f101950463dcf8, for the device source that defines it. Do not edit. +#include +#include + +#include "cake_ffi_checks.h" + +#include + +extern "C" __global__ void kernel_cake_gated_act_mxfp8_383546f101950463dcf8(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K); + +namespace cake_binding_kernel_cake_gated_act_mxfp8_383546f101950463dcf8 { + +using tvm::ffi::TensorView; + +constexpr const char* KERNEL_NAME = "kernel_cake_gated_act_mxfp8_383546f101950463dcf8"; + +void CheckCuda(cudaError_t status, const char* what) { + TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) + << KERNEL_NAME << ": " << what << " failed: " << cudaGetErrorString(status); +} + +void* DataPtr(const TensorView& t) { return static_cast(t.data_ptr()) + t.byte_offset(); } + +void Run(TensorView a_gated_input, TensorView a_grad, TensorView a_row_output, TensorView a_col_output, TensorView a_row_scales, TensorView a_col_scales, int64_t a_M, int64_t a_K, int64_t grid_x, int64_t grid_y, int64_t grid_z) { + DLDevice dev = a_gated_input.device(); + tvm::ffi::CUDADeviceGuard device_guard(dev.device_id); + check_cuda_tensor(a_gated_input, "gated_input"); + check_dtype(a_gated_input, DLDataType{kDLBfloat, 16, 1}, "gated_input"); + check_cuda_tensor(a_grad, "grad"); + check_dtype(a_grad, DLDataType{kDLBfloat, 16, 1}, "grad"); + check_cuda_tensor(a_row_output, "row_output"); + check_dtype(a_row_output, DLDataType{kDLUInt, 8, 1}, "row_output"); + check_cuda_tensor(a_col_output, "col_output"); + check_dtype(a_col_output, DLDataType{kDLUInt, 8, 1}, "col_output"); + check_cuda_tensor(a_row_scales, "row_scales"); + check_dtype(a_row_scales, DLDataType{kDLUInt, 8, 1}, "row_scales"); + check_cuda_tensor(a_col_scales, "col_scales"); + check_dtype(a_col_scales, DLDataType{kDLUInt, 8, 1}, "col_scales"); + TVM_FFI_CHECK(a_M >= -2147483648LL && a_M <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'M' = " << a_M + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_K >= -2147483648LL && a_K <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'K' = " << a_K + << " is outside the i32 range [-2147483648, 2147483647]"; + check_contiguous(a_gated_input, "gated_input"); + check_contiguous(a_grad, "grad"); + check_same_device(a_grad, a_gated_input, "grad", "gated_input"); + check_contiguous(a_row_output, "row_output"); + check_same_device(a_row_output, a_gated_input, "row_output", "gated_input"); + check_contiguous(a_col_output, "col_output"); + check_same_device(a_col_output, a_gated_input, "col_output", "gated_input"); + check_contiguous(a_row_scales, "row_scales"); + check_same_device(a_row_scales, a_gated_input, "row_scales", "gated_input"); + check_contiguous(a_col_scales, "col_scales"); + check_same_device(a_col_scales, a_gated_input, "col_scales", "gated_input"); + TVM_FFI_CHECK(grid_x > 0 && grid_y > 0 && grid_z > 0, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") must be positive"; + TVM_FFI_CHECK(grid_x <= 2147483647 && grid_y <= 65535 && grid_z <= 65535, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") exceeds CUDA's limits"; + cudaStream_t stream = static_cast(TVMFFIEnvGetStream(dev.device_type, dev.device_id)); + void* v_gated_input = DataPtr(a_gated_input); + void* v_grad = DataPtr(a_grad); + void* v_row_output = DataPtr(a_row_output); + void* v_col_output = DataPtr(a_col_output); + void* v_row_scales = DataPtr(a_row_scales); + void* v_col_scales = DataPtr(a_col_scales); + int32_t v_M = static_cast(a_M); + int32_t v_K = static_cast(a_K); + void* kernel_args[] = {&v_gated_input, &v_grad, &v_row_output, &v_col_output, &v_row_scales, &v_col_scales, &v_M, &v_K}; + cudaLaunchConfig_t config{}; + config.gridDim = dim3(static_cast(grid_x), static_cast(grid_y), static_cast(grid_z)); + config.blockDim = dim3(128, 1, 1); + config.dynamicSmemBytes = 4096; + config.stream = stream; + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gated_act_mxfp8_383546f101950463dcf8), kernel_args), + "cudaLaunchKernelExC"); +} + +} // namespace cake_binding_kernel_cake_gated_act_mxfp8_383546f101950463dcf8 + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gated_act_mxfp8_383546f101950463dcf8::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_383546f101950463dcf8_kernel.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_383546f101950463dcf8_kernel.cu new file mode 100644 index 00000000000..781600d220c --- /dev/null +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_383546f101950463dcf8_kernel.cu @@ -0,0 +1,1057 @@ +/* + * Copyright (c) 2026 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +#if defined(__CUDACC_RTC__) +typedef unsigned long long uint64_t; +#else +typedef unsigned long uint64_t; +#endif +static_assert(sizeof(uint64_t) == 8, "Cake requires an LP64 CUDA host ABI"); +typedef signed int int32_t; +typedef short int int16_t; +struct __align__(64) CakeTensorMap64 { uint64_t opaque[16]; }; +static_assert(sizeof(CakeTensorMap64) == 128, "64-aligned tensor-map ABI size"); +static_assert(alignof(CakeTensorMap64) == 64, "64-aligned tensor-map ABI alignment"); + +#if defined(__CUDACC_RTC__) +typedef struct __align__(128) { uint64_t opaque[16]; } CUtensorMap; +#else +#include +#endif + +static_assert(sizeof(CUtensorMap) == 128, "CUtensorMap CUDA ABI must be 128 bytes"); +#include +#include + +__device__ __forceinline__ int make_warp_uniform(int x) { + int result; + asm volatile("shfl.sync.idx.b32 %0, %1, 0, 0x1F, 0xFFFFFFFF;" + : "=r"(result) : "r"(x)); + return result; +} + +#define CAKE_INF CUDART_INF_F +#define NUM_MAIN_STAGES 1 +#define SMEM_TILES_OFF 0 +#define SMEM_TILES_STAGE_BYTES 4096 +#define SMEM_TILES_STRIDE 4096 +#define SMEM_TOTAL 4096 +#define THREADS 128 +#define LAUNCH_MIN_BLOCKS 12 + +#include + +__device__ __forceinline__ float approx_exp2(float x) { + float y; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + +__device__ __forceinline__ float max_noftz(float a, float b) { + float c; + asm("max.f32 %0, %1, %2;" : "=f"(c) : "f"(a), "f"(b)); + return c; +} + + +__device__ __forceinline__ float2 sub_f32x2_noftz(float2 a, float2 b) { + float2 r; + asm("sub.f32x2 %0, %1, %2;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(unsigned long long*)&a), "l"(*(unsigned long long*)&b)); + return r; +} + +__device__ __forceinline__ float2 mul_f32x2_noftz(float2 a, float2 b) { + float2 r; + asm("mul.rn.f32x2 %0, %1, %2;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(unsigned long long*)&a), "l"(*(unsigned long long*)&b)); + return r; +} + + +// ex2_emulation_f32x2 defined in softmax_frag_exp2_cast helper (or standalone) + +__device__ __forceinline__ float2 fma_f32x2_rn_noftz(float2 a, float2 b, float2 c) { + float2 r; + asm("fma.rn.f32x2 %0, %1, %2, %3;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(const unsigned long long*)&a), + "l"(*(const unsigned long long*)&b), + "l"(*(const unsigned long long*)&c)); + return r; +} + + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + +__device__ __forceinline__ __nv_bfloat162 __as_bf16x2(unsigned int v) { + __nv_bfloat162_raw raw; + raw.x = static_cast(v); + raw.y = static_cast(v >> 16); + return __nv_bfloat162(raw); +} + +__device__ __forceinline__ float2 cake_bf16x2_to_f32x2(__nv_bfloat162 v) { + const unsigned int w = *reinterpret_cast(&v); + return make_float2(__uint_as_float(w << 16), __uint_as_float(w & 0xffff0000u)); +} + +__device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { + float r; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} + +extern "C" { + +__global__ __launch_bounds__(128, LAUNCH_MIN_BLOCKS) void +kernel_cake_gated_act_mxfp8_383546f101950463dcf8(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K) +{ + const int tid = threadIdx.x; + const int warp = make_warp_uniform(tid / 32); + const int lane = tid % 32; + + extern __shared__ __align__(1024) char smem_raw[]; + int smem; +#if __CUDA_ARCH__ == 1000 + asm volatile("{ .reg .u64 smem_ptr; cvta.to.shared.u64 smem_ptr, %1; cvt.u32.u64 %0, smem_ptr; }" : "=r"(smem) : "l"(smem_raw)); + smem = make_warp_uniform(smem); +#else + smem = (int)(unsigned long long)__cvta_generic_to_shared(smem_raw); +#endif + + const int bid = blockIdx.x; + const int num_bids = gridDim.x; + + const int cta_rank = 0; + + // Kernel setup ops + uint8_t* tiles = reinterpret_cast(smem_raw + 0); + const int tiles_addr = smem + 0; + + // === Task calls (declaration order) === + int quarter = tid & 3; + int row = tid >> 2; + int bx = blockIdx.x; + int by = blockIdx.y; + int grow = by * 32 + row; + int col = bx * 32 + quarter * 8; + int gate_index = grow * (2 * K) + col; + unsigned int _vec_load_0[4]; + { + uint32_t _uv4_0_0; + uint32_t _uv4_0_1; + uint32_t _uv4_0_2; + uint32_t _uv4_0_3; + asm volatile("ld.global.L2::cache_hint.v4.b32 {%0, %1, %2, %3}, [%4], %5;" + : "=r"(_uv4_0_0), "=r"(_uv4_0_1), "=r"(_uv4_0_2), "=r"(_uv4_0_3) : "l"((const void*)(gated_input + gate_index + (0))), "l"(0x12F0000000000000ULL) : "memory"); + _vec_load_0[0 + 0] = _uv4_0_0; + _vec_load_0[0 + 1] = _uv4_0_1; + _vec_load_0[0 + 2] = _uv4_0_2; + _vec_load_0[0 + 3] = _uv4_0_3; + } + unsigned int _vec_load_1[4]; + { + uint32_t _uv4_1_0; + uint32_t _uv4_1_1; + uint32_t _uv4_1_2; + uint32_t _uv4_1_3; + asm volatile("ld.global.L2::cache_hint.v4.b32 {%0, %1, %2, %3}, [%4], %5;" + : "=r"(_uv4_1_0), "=r"(_uv4_1_1), "=r"(_uv4_1_2), "=r"(_uv4_1_3) : "l"((const void*)(gated_input + (gate_index + K) + (0))), "l"(0x12F0000000000000ULL) : "memory"); + _vec_load_1[0 + 0] = _uv4_1_0; + _vec_load_1[0 + 1] = _uv4_1_1; + _vec_load_1[0 + 2] = _uv4_1_2; + _vec_load_1[0 + 3] = _uv4_1_3; + } + unsigned int _vec_load_2[4]; + { + uint32_t _uv4_2_0; + uint32_t _uv4_2_1; + uint32_t _uv4_2_2; + uint32_t _uv4_2_3; + asm volatile("ld.global.L2::cache_hint.v4.b32 {%0, %1, %2, %3}, [%4], %5;" + : "=r"(_uv4_2_0), "=r"(_uv4_2_1), "=r"(_uv4_2_2), "=r"(_uv4_2_3) : "l"((const void*)(grad + (grow * K + col) + (0))), "l"(0x12F0000000000000ULL) : "memory"); + _vec_load_2[0 + 0] = _uv4_2_0; + _vec_load_2[0 + 1] = _uv4_2_1; + _vec_load_2[0 + 2] = _uv4_2_2; + _vec_load_2[0 + 3] = _uv4_2_3; + } + unsigned int dgate[4]; + unsigned int dup[4]; + __nv_bfloat162 lowest = __as_bf16x2(_vec_load_0[0]); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[1])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[2])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[3])); + float2 _cvt_f32_0 = cake_bf16x2_to_f32x2(lowest); + float _min_4; + asm("min.NaN.f32 %0, %1, %2;" : "=f"(_min_4) : "f"(_cvt_f32_0.x), "f"(_cvt_f32_0.y)); + int _warp_vote_3 = __all_sync(0xFFFFFFFF, _min_4 > -87.0f); + if (_warp_vote_3 != 0) { + float2 _cvt_f32_1 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[0])); + float2 _cvt_f32_2 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[0])); + float2 _cvt_f32_3 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[0])); + float e0_1 = approx_exp2((-_cvt_f32_1.x) * 1.442695f); + float e1_1 = approx_exp2((-_cvt_f32_1.y) * 1.442695f); + float d0_1 = 1.0f + e0_1; + float d1_1 = 1.0f + e1_1; + float r0_1 = approx_rcp(d0_1); + float r1_1 = approx_rcp(d1_1); + float _fma_6; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_6) : "f"(-d0_1), "f"(r0_1), "f"(1.0f)); + float _fma_5; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_5) : "f"(r0_1), "f"(_fma_6), "f"(r0_1)); + float _fma_8; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_8) : "f"(-d1_1), "f"(r1_1), "f"(1.0f)); + float _fma_7; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_7) : "f"(r1_1), "f"(_fma_8), "f"(r1_1)); + float2 s_0 = make_float2(_fma_5, _fma_7); + float2 act_0; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_0) : "l"(*(const unsigned long long*)&_cvt_f32_1), "l"(*(const unsigned long long*)&s_0)); + float2 _f32x2_mul_a_9 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_0, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_0), s_0), _cvt_f32_3); + float2 _mul_0; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_9), "l"(*(const unsigned long long*)&_cvt_f32_2)); + float2 _mul_1; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_1) : "l"(*(const unsigned long long*)&act_0), "l"(*(const unsigned long long*)&_cvt_f32_3)); + __nv_bfloat162 _bf16x2_0 = __float22bfloat162_rn(make_float2(_mul_0.x, _mul_0.y)); + __nv_bfloat162 _bf16x2_1 = __float22bfloat162_rn(make_float2(_mul_1.x, _mul_1.y)); + dgate[0] = __as_u32(_bf16x2_0); + dup[0] = __as_u32(_bf16x2_1); + float2 _cvt_f32_4 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[1])); + float2 _cvt_f32_5 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[1])); + float2 _cvt_f32_6 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[1])); + float e0_3 = approx_exp2((-_cvt_f32_4.x) * 1.442695f); + float e1_3 = approx_exp2((-_cvt_f32_4.y) * 1.442695f); + float d0_3 = 1.0f + e0_3; + float d1_3 = 1.0f + e1_3; + float r0_3 = approx_rcp(d0_3); + float r1_3 = approx_rcp(d1_3); + float _fma_11; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_11) : "f"(-d0_3), "f"(r0_3), "f"(1.0f)); + float _fma_10; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_10) : "f"(r0_3), "f"(_fma_11), "f"(r0_3)); + float _fma_13; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_13) : "f"(-d1_3), "f"(r1_3), "f"(1.0f)); + float _fma_12; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_12) : "f"(r1_3), "f"(_fma_13), "f"(r1_3)); + float2 s_2 = make_float2(_fma_10, _fma_12); + float2 act_2; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_2) : "l"(*(const unsigned long long*)&_cvt_f32_4), "l"(*(const unsigned long long*)&s_2)); + float2 _f32x2_mul_a_14 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_2, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_2), s_2), _cvt_f32_6); + float2 _mul_2; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_14), "l"(*(const unsigned long long*)&_cvt_f32_5)); + float2 _mul_3; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_3) : "l"(*(const unsigned long long*)&act_2), "l"(*(const unsigned long long*)&_cvt_f32_6)); + __nv_bfloat162 _bf16x2_2 = __float22bfloat162_rn(make_float2(_mul_2.x, _mul_2.y)); + __nv_bfloat162 _bf16x2_3 = __float22bfloat162_rn(make_float2(_mul_3.x, _mul_3.y)); + dgate[1] = __as_u32(_bf16x2_2); + dup[1] = __as_u32(_bf16x2_3); + float2 _cvt_f32_7 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[2])); + float2 _cvt_f32_8 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[2])); + float2 _cvt_f32_9 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[2])); + float e0_5 = approx_exp2((-_cvt_f32_7.x) * 1.442695f); + float e1_5 = approx_exp2((-_cvt_f32_7.y) * 1.442695f); + float d0_5 = 1.0f + e0_5; + float d1_5 = 1.0f + e1_5; + float r0_5 = approx_rcp(d0_5); + float r1_5 = approx_rcp(d1_5); + float _fma_16; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_16) : "f"(-d0_5), "f"(r0_5), "f"(1.0f)); + float _fma_15; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_15) : "f"(r0_5), "f"(_fma_16), "f"(r0_5)); + float _fma_18; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_18) : "f"(-d1_5), "f"(r1_5), "f"(1.0f)); + float _fma_17; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_17) : "f"(r1_5), "f"(_fma_18), "f"(r1_5)); + float2 s_4 = make_float2(_fma_15, _fma_17); + float2 act_4; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_4) : "l"(*(const unsigned long long*)&_cvt_f32_7), "l"(*(const unsigned long long*)&s_4)); + float2 _f32x2_mul_a_19 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_4, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_4), s_4), _cvt_f32_9); + float2 _mul_4; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_19), "l"(*(const unsigned long long*)&_cvt_f32_8)); + float2 _mul_5; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_5) : "l"(*(const unsigned long long*)&act_4), "l"(*(const unsigned long long*)&_cvt_f32_9)); + __nv_bfloat162 _bf16x2_4 = __float22bfloat162_rn(make_float2(_mul_4.x, _mul_4.y)); + __nv_bfloat162 _bf16x2_5 = __float22bfloat162_rn(make_float2(_mul_5.x, _mul_5.y)); + dgate[2] = __as_u32(_bf16x2_4); + dup[2] = __as_u32(_bf16x2_5); + float2 _cvt_f32_10 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[3])); + float2 _cvt_f32_11 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[3])); + float2 _cvt_f32_12 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[3])); + float e0_7 = approx_exp2((-_cvt_f32_10.x) * 1.442695f); + float e1_7 = approx_exp2((-_cvt_f32_10.y) * 1.442695f); + float d0_7 = 1.0f + e0_7; + float d1_7 = 1.0f + e1_7; + float r0_7 = approx_rcp(d0_7); + float r1_7 = approx_rcp(d1_7); + float _fma_21; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_21) : "f"(-d0_7), "f"(r0_7), "f"(1.0f)); + float _fma_20; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_20) : "f"(r0_7), "f"(_fma_21), "f"(r0_7)); + float _fma_23; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_23) : "f"(-d1_7), "f"(r1_7), "f"(1.0f)); + float _fma_22; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_22) : "f"(r1_7), "f"(_fma_23), "f"(r1_7)); + float2 s_6 = make_float2(_fma_20, _fma_22); + float2 act_6; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_6) : "l"(*(const unsigned long long*)&_cvt_f32_10), "l"(*(const unsigned long long*)&s_6)); + float2 _f32x2_mul_a_24 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_6, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_6), s_6), _cvt_f32_12); + float2 _mul_6; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_24), "l"(*(const unsigned long long*)&_cvt_f32_11)); + float2 _mul_7; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_7) : "l"(*(const unsigned long long*)&act_6), "l"(*(const unsigned long long*)&_cvt_f32_12)); + __nv_bfloat162 _bf16x2_6 = __float22bfloat162_rn(make_float2(_mul_6.x, _mul_6.y)); + __nv_bfloat162 _bf16x2_7 = __float22bfloat162_rn(make_float2(_mul_7.x, _mul_7.y)); + dgate[3] = __as_u32(_bf16x2_6); + dup[3] = __as_u32(_bf16x2_7); + } else { + int band = 0; + float2 _cvt_f32_13 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[0])); + float2 _cvt_f32_14 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[0])); + float2 _cvt_f32_15 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[0])); + float e0_9 = approx_exp2((-_cvt_f32_13.x) * 1.442695f); + float e1_9 = approx_exp2((-_cvt_f32_13.y) * 1.442695f); + float d0_9 = 1.0f + e0_9; + float d1_9 = 1.0f + e1_9; + float r0_9 = approx_rcp(d0_9); + float r1_9 = approx_rcp(d1_9); + float _fma_25; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_25) : "f"(-d0_9), "f"(r0_9), "f"(1.0f)); + float _fma_0; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_0) : "f"(r0_9), "f"(_fma_25), "f"(r0_9)); + float _fma_26; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_26) : "f"(-d1_9), "f"(r1_9), "f"(1.0f)); + float _fma_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_1) : "f"(r1_9), "f"(_fma_26), "f"(r1_9)); + float t0_9 = _fma_0; + float t1_9 = _fma_1; + band = band | ((fminf(t0_9, t1_9) == 0.0f) ? 1 : 0); + float2 s_8 = make_float2(max_noftz(t0_9, 0.0f), max_noftz(t1_9, 0.0f)); + float2 act_8; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_8) : "l"(*(const unsigned long long*)&_cvt_f32_13), "l"(*(const unsigned long long*)&s_8)); + float2 _f32x2_mul_a_27 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_8, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_8), s_8), _cvt_f32_15); + float2 _mul_8; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_27), "l"(*(const unsigned long long*)&_cvt_f32_14)); + float2 _mul_9; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_9) : "l"(*(const unsigned long long*)&act_8), "l"(*(const unsigned long long*)&_cvt_f32_15)); + __nv_bfloat162 _bf16x2_8 = __float22bfloat162_rn(make_float2(_mul_8.x, _mul_8.y)); + __nv_bfloat162 _bf16x2_9 = __float22bfloat162_rn(make_float2(_mul_9.x, _mul_9.y)); + dgate[0] = __as_u32(_bf16x2_8); + dup[0] = __as_u32(_bf16x2_9); + float2 _cvt_f32_16 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[1])); + float2 _cvt_f32_17 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[1])); + float2 _cvt_f32_18 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[1])); + float e0_11 = approx_exp2((-_cvt_f32_16.x) * 1.442695f); + float e1_11 = approx_exp2((-_cvt_f32_16.y) * 1.442695f); + float d0_11 = 1.0f + e0_11; + float d1_11 = 1.0f + e1_11; + float r0_11 = approx_rcp(d0_11); + float r1_11 = approx_rcp(d1_11); + float _fma_28; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_28) : "f"(-d0_11), "f"(r0_11), "f"(1.0f)); + float _fma_2; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_2) : "f"(r0_11), "f"(_fma_28), "f"(r0_11)); + float _fma_29; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_29) : "f"(-d1_11), "f"(r1_11), "f"(1.0f)); + float _fma_3; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_3) : "f"(r1_11), "f"(_fma_29), "f"(r1_11)); + float t0_11 = _fma_2; + float t1_11 = _fma_3; + band = band | ((fminf(t0_11, t1_11) == 0.0f) ? 2 : 0); + float2 s_10 = make_float2(max_noftz(t0_11, 0.0f), max_noftz(t1_11, 0.0f)); + float2 act_10; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_10) : "l"(*(const unsigned long long*)&_cvt_f32_16), "l"(*(const unsigned long long*)&s_10)); + float2 _f32x2_mul_a_30 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_10, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_10), s_10), _cvt_f32_18); + float2 _mul_10; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_30), "l"(*(const unsigned long long*)&_cvt_f32_17)); + float2 _mul_11; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_11) : "l"(*(const unsigned long long*)&act_10), "l"(*(const unsigned long long*)&_cvt_f32_18)); + __nv_bfloat162 _bf16x2_10 = __float22bfloat162_rn(make_float2(_mul_10.x, _mul_10.y)); + __nv_bfloat162 _bf16x2_11 = __float22bfloat162_rn(make_float2(_mul_11.x, _mul_11.y)); + dgate[1] = __as_u32(_bf16x2_10); + dup[1] = __as_u32(_bf16x2_11); + float2 _cvt_f32_19 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[2])); + float2 _cvt_f32_20 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[2])); + float2 _cvt_f32_21 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[2])); + float e0_13 = approx_exp2((-_cvt_f32_19.x) * 1.442695f); + float e1_13 = approx_exp2((-_cvt_f32_19.y) * 1.442695f); + float d0_13 = 1.0f + e0_13; + float d1_13 = 1.0f + e1_13; + float r0_13 = approx_rcp(d0_13); + float r1_13 = approx_rcp(d1_13); + float _fma_31; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_31) : "f"(-d0_13), "f"(r0_13), "f"(1.0f)); + float _fma_4; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_4) : "f"(r0_13), "f"(_fma_31), "f"(r0_13)); + float _fma_32; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_32) : "f"(-d1_13), "f"(r1_13), "f"(1.0f)); + float _fma_5_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_5_1) : "f"(r1_13), "f"(_fma_32), "f"(r1_13)); + float t0_13 = _fma_4; + float t1_13 = _fma_5_1; + band = band | ((fminf(t0_13, t1_13) == 0.0f) ? 4 : 0); + float2 s_12 = make_float2(max_noftz(t0_13, 0.0f), max_noftz(t1_13, 0.0f)); + float2 act_12; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_12) : "l"(*(const unsigned long long*)&_cvt_f32_19), "l"(*(const unsigned long long*)&s_12)); + float2 _f32x2_mul_a_33 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_12, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_12), s_12), _cvt_f32_21); + float2 _mul_12; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_33), "l"(*(const unsigned long long*)&_cvt_f32_20)); + float2 _mul_13; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_13) : "l"(*(const unsigned long long*)&act_12), "l"(*(const unsigned long long*)&_cvt_f32_21)); + __nv_bfloat162 _bf16x2_12 = __float22bfloat162_rn(make_float2(_mul_12.x, _mul_12.y)); + __nv_bfloat162 _bf16x2_13 = __float22bfloat162_rn(make_float2(_mul_13.x, _mul_13.y)); + dgate[2] = __as_u32(_bf16x2_12); + dup[2] = __as_u32(_bf16x2_13); + float2 _cvt_f32_22 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[3])); + float2 _cvt_f32_23 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[3])); + float2 _cvt_f32_24 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[3])); + float e0_15 = approx_exp2((-_cvt_f32_22.x) * 1.442695f); + float e1_15 = approx_exp2((-_cvt_f32_22.y) * 1.442695f); + float d0_15 = 1.0f + e0_15; + float d1_15 = 1.0f + e1_15; + float r0_15 = approx_rcp(d0_15); + float r1_15 = approx_rcp(d1_15); + float _fma_34; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_34) : "f"(-d0_15), "f"(r0_15), "f"(1.0f)); + float _fma_6_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_6_1) : "f"(r0_15), "f"(_fma_34), "f"(r0_15)); + float _fma_35; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_35) : "f"(-d1_15), "f"(r1_15), "f"(1.0f)); + float _fma_7_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_7_1) : "f"(r1_15), "f"(_fma_35), "f"(r1_15)); + float t0_15 = _fma_6_1; + float t1_15 = _fma_7_1; + band = band | ((fminf(t0_15, t1_15) == 0.0f) ? 8 : 0); + float2 s_14 = make_float2(max_noftz(t0_15, 0.0f), max_noftz(t1_15, 0.0f)); + float2 act_14; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_14) : "l"(*(const unsigned long long*)&_cvt_f32_22), "l"(*(const unsigned long long*)&s_14)); + float2 _f32x2_mul_a_36 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_14, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_14), s_14), _cvt_f32_24); + float2 _mul_14; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_36), "l"(*(const unsigned long long*)&_cvt_f32_23)); + float2 _mul_15; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_15) : "l"(*(const unsigned long long*)&act_14), "l"(*(const unsigned long long*)&_cvt_f32_24)); + __nv_bfloat162 _bf16x2_14 = __float22bfloat162_rn(make_float2(_mul_14.x, _mul_14.y)); + __nv_bfloat162 _bf16x2_15 = __float22bfloat162_rn(make_float2(_mul_15.x, _mul_15.y)); + dgate[3] = __as_u32(_bf16x2_14); + dup[3] = __as_u32(_bf16x2_15); + if (band != 0) { + unsigned int _popc_0 = (unsigned int)__popc(band); + #pragma unroll 1 + for (int _j = 0; _j < _popc_0; _j++) { + int _find_first_set_37 = __ffs(band); + int j = _find_first_set_37 - 1; + band = band & band - 1; + unsigned int _vec_load_3[1]; + { + _vec_load_3[0] = *reinterpret_cast(gated_input + (gate_index + 2 * j) + 0); + } + unsigned int _vec_load_4[1]; + { + _vec_load_4[0] = *reinterpret_cast(gated_input + (gate_index + K + 2 * j) + 0); + } + unsigned int _vec_load_5[1]; + { + _vec_load_5[0] = *reinterpret_cast(grad + (grow * K + col + 2 * j) + 0); + } + float2 _cvt_f32_25 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[0])); + float2 _cvt_f32_26 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[0])); + float2 _cvt_f32_27 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[0])); + float e0_17 = approx_exp2((-_cvt_f32_25.x) * 1.442695f); + float e1_17 = approx_exp2((-_cvt_f32_25.y) * 1.442695f); + float d0_17 = 1.0f + e0_17; + float d1_17 = 1.0f + e1_17; + float r0_17 = approx_rcp(d0_17); + float r1_17 = approx_rcp(d1_17); + float _fma_38; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_38) : "f"(-d0_17), "f"(r0_17), "f"(1.0f)); + float _fma_8_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_8_1) : "f"(r0_17), "f"(_fma_38), "f"(r0_17)); + float _fma_39; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_39) : "f"(-d1_17), "f"(r1_17), "f"(1.0f)); + float _fma_9; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_9) : "f"(r1_17), "f"(_fma_39), "f"(r1_17)); + float t0_17 = _fma_8_1; + float t1_17 = _fma_9; + float d4_17a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_17a) : "f"(d0_17), "f"(0.25f)); + float r4_17a = approx_rcp(d4_17a); + float _fma_41; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_41) : "f"(-d4_17a), "f"(r4_17a), "f"(1.0f)); + float _fma_40; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_40) : "f"(r4_17a), "f"(_fma_41), "f"(r4_17a)); + float big_17a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_17a) : "f"(mul_f32_rn_noftz(_fma_40, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_42; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_42) : "f"(big_17a), "f"(8.388608e+06f)); + float whole_17a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_17a) : "f"(_add_42), "f"(8.388608e+06f)); + float above_17a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_17a) : "f"(-d4_17a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_17a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_44; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_44) : "f"(big_17a), "f"(whole_17a)); + float _abs_43; + asm("abs.f32 %0, %1;" : "=f"(_abs_43) : "f"(_sub_44)); + float _add_45; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_45) : "f"(big_17a), "f"(0.5f)); + float _sub_46; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_46) : "f"(big_17a), "f"(0.5f)); + float rounded_17a = ((_abs_43 == 0.5f) ? ((above_17a > 0.0f) ? _add_45 : _sub_46) : whole_17a); + float u0_17 = ((d0_17 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_17a, 7.888609e-31f), 1.7763568e-15f) : t0_17); + float d4_17b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_17b) : "f"(d1_17), "f"(0.25f)); + float r4_17b = approx_rcp(d4_17b); + float _fma_48; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_48) : "f"(-d4_17b), "f"(r4_17b), "f"(1.0f)); + float _fma_47; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_47) : "f"(r4_17b), "f"(_fma_48), "f"(r4_17b)); + float big_17b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_17b) : "f"(mul_f32_rn_noftz(_fma_47, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_49; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_49) : "f"(big_17b), "f"(8.388608e+06f)); + float whole_17b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_17b) : "f"(_add_49), "f"(8.388608e+06f)); + float above_17b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_17b) : "f"(-d4_17b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_17b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_51; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_51) : "f"(big_17b), "f"(whole_17b)); + float _abs_50; + asm("abs.f32 %0, %1;" : "=f"(_abs_50) : "f"(_sub_51)); + float _add_52; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_52) : "f"(big_17b), "f"(0.5f)); + float _sub_53; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_53) : "f"(big_17b), "f"(0.5f)); + float rounded_17b = ((_abs_50 == 0.5f) ? ((above_17b > 0.0f) ? _add_52 : _sub_53) : whole_17b); + float u1_17 = ((d1_17 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_17b, 7.888609e-31f), 1.7763568e-15f) : t1_17); + float2 s_16 = make_float2(max_noftz(u0_17, 0.0f), max_noftz(u1_17, 0.0f)); + float2 act_16; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_16) : "l"(*(const unsigned long long*)&_cvt_f32_25), "l"(*(const unsigned long long*)&s_16)); + float2 _f32x2_mul_a_54 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_16, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_16), s_16), _cvt_f32_27); + float2 _mul_16; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_16) : "l"(*(const unsigned long long*)&_f32x2_mul_a_54), "l"(*(const unsigned long long*)&_cvt_f32_26)); + float2 _mul_17; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_17) : "l"(*(const unsigned long long*)&act_16), "l"(*(const unsigned long long*)&_cvt_f32_27)); + __nv_bfloat162 _bf16x2_16 = __float22bfloat162_rn(make_float2(_mul_16.x, _mul_16.y)); + __nv_bfloat162 _bf16x2_17 = __float22bfloat162_rn(make_float2(_mul_17.x, _mul_17.y)); + unsigned int dgate_new = __as_u32(_bf16x2_16); + unsigned int dup_new = __as_u32(_bf16x2_17); + dgate[0] = ((j == 0) ? dgate_new : dgate[0]); + dup[0] = ((j == 0) ? dup_new : dup[0]); + dgate[1] = ((j == 1) ? dgate_new : dgate[1]); + dup[1] = ((j == 1) ? dup_new : dup[1]); + dgate[2] = ((j == 2) ? dgate_new : dgate[2]); + dup[2] = ((j == 2) ? dup_new : dup[2]); + dgate[3] = ((j == 3) ? dgate_new : dgate[3]); + dup[3] = ((j == 3) ? dup_new : dup[3]); + } + } + } + unsigned int chunk_addr = tiles_addr + (unsigned int)(row * 64) + (((unsigned int)quarter ^ (unsigned int)(row >> 3 & 3)) << 4); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(chunk_addr), "r"(dgate[0]), "r"(dgate[1]), "r"(dgate[2]), "r"(dgate[3]) : "memory"); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(chunk_addr + 2048), "r"(dup[0]), "r"(dup[1]), "r"(dup[2]), "r"(dup[3]) : "memory"); + int scale_cols = K >> 6; + int out_index = grow * (2 * K) + col; + __nv_bfloat162 amax = __as_bf16x2(0); + __nv_bfloat162 _hxa_56 = amax; + __nv_bfloat162 _hxb_56 = __as_bf16x2(dgate[0]); + unsigned int _hxr_56; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_56) : "r"(*reinterpret_cast(&_hxa_56)), "r"(*reinterpret_cast(&_hxb_56))); + __nv_bfloat162 _max_55 = *reinterpret_cast(&_hxr_56); + amax = _max_55; + __nv_bfloat162 _hxa_58 = amax; + __nv_bfloat162 _hxb_58 = __as_bf16x2(dgate[1]); + unsigned int _hxr_58; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_58) : "r"(*reinterpret_cast(&_hxa_58)), "r"(*reinterpret_cast(&_hxb_58))); + __nv_bfloat162 _max_57 = *reinterpret_cast(&_hxr_58); + amax = _max_57; + __nv_bfloat162 _hxa_60 = amax; + __nv_bfloat162 _hxb_60 = __as_bf16x2(dgate[2]); + unsigned int _hxr_60; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_60) : "r"(*reinterpret_cast(&_hxa_60)), "r"(*reinterpret_cast(&_hxb_60))); + __nv_bfloat162 _max_59 = *reinterpret_cast(&_hxr_60); + amax = _max_59; + __nv_bfloat162 _hxa_62 = amax; + __nv_bfloat162 _hxb_62 = __as_bf16x2(dgate[3]); + unsigned int _hxr_62; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_62) : "r"(*reinterpret_cast(&_hxa_62)), "r"(*reinterpret_cast(&_hxb_62))); + __nv_bfloat162 _max_61 = *reinterpret_cast(&_hxr_62); + amax = _max_61; + unsigned int mag = __as_u32(amax) & 2147450879; + unsigned int peer1 = __shfl_xor_sync(0xFFFFFFFF, mag, 1); + __nv_bfloat162 pair_max = __hmax2_nan(__as_bf16x2(mag), __as_bf16x2(peer1)); + unsigned int peer2 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(pair_max), 2); + __nv_bfloat162 quad_max = __hmax2_nan(pair_max, __as_bf16x2(peer2)); + unsigned int folded = __as_u32(__hmax2_nan(quad_max, __as_bf16x2(__as_u32(quad_max) >> 16))); + unsigned int amax_bits = (folded & 65535) << 16; + int scale_i32 = (int)(amax_bits + 2031616 >> 23) - 8; + if (scale_i32 < 0) { + scale_i32 = 0; + } + if ((amax_bits & 2139095040) == 2139095040) { + scale_i32 = 255; + } + unsigned int scale = (unsigned int)scale_i32; + unsigned int inv = 254 - scale << 7; + if (scale == 255) { + inv = 32704; + } + unsigned int inv2 = inv | inv << 16; + int scale_index = ((grow >> 7) * scale_cols + (bx >> 2)) * 512 + (grow & 31) * 16 + (grow >> 5 & 3) * 4 + (bx & 3); + if (quarter == 0) { + *(reinterpret_cast(row_scales + scale_index) + (0)) = (unsigned char)(scale); + } + unsigned int words[2]; + uint32_t _bf16x2_mul_0; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_0) : "r"(dgate[0]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_0; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_0) : "r"(_bf16x2_mul_0)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_0); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_0) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_1; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_1) : "r"(dgate[1]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_64; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_64) : "r"(_bf16x2_mul_1)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_1); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_64) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_63; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_63) : "h"(_e4m3x2_bf16x2_0), "h"(_cvt_satfinite_e4m3x2_bf16x2_64)); + words[0] = _mov_b32_pack_u16x2_63; + uint32_t _bf16x2_mul_2; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_2) : "r"(dgate[2]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_1; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_1) : "r"(_bf16x2_mul_2)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_2); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_1) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_3; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_3) : "r"(dgate[3]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_66; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_66) : "r"(_bf16x2_mul_3)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_3); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_66) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_65; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_65) : "h"(_e4m3x2_bf16x2_1), "h"(_cvt_satfinite_e4m3x2_bf16x2_66)); + words[1] = _mov_b32_pack_u16x2_65; + reinterpret_cast(row_output + out_index)[0] = reinterpret_cast(words + 0)[0]; + __nv_bfloat162 amax_0 = __as_bf16x2(0); + __nv_bfloat162 _hxa_68 = amax_0; + __nv_bfloat162 _hxb_68 = __as_bf16x2(dup[0]); + unsigned int _hxr_68; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_68) : "r"(*reinterpret_cast(&_hxa_68)), "r"(*reinterpret_cast(&_hxb_68))); + __nv_bfloat162 _max_67 = *reinterpret_cast(&_hxr_68); + amax_0 = _max_67; + __nv_bfloat162 _hxa_70 = amax_0; + __nv_bfloat162 _hxb_70 = __as_bf16x2(dup[1]); + unsigned int _hxr_70; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_70) : "r"(*reinterpret_cast(&_hxa_70)), "r"(*reinterpret_cast(&_hxb_70))); + __nv_bfloat162 _max_69 = *reinterpret_cast(&_hxr_70); + amax_0 = _max_69; + __nv_bfloat162 _hxa_72 = amax_0; + __nv_bfloat162 _hxb_72 = __as_bf16x2(dup[2]); + unsigned int _hxr_72; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_72) : "r"(*reinterpret_cast(&_hxa_72)), "r"(*reinterpret_cast(&_hxb_72))); + __nv_bfloat162 _max_71 = *reinterpret_cast(&_hxr_72); + amax_0 = _max_71; + __nv_bfloat162 _hxa_74 = amax_0; + __nv_bfloat162 _hxb_74 = __as_bf16x2(dup[3]); + unsigned int _hxr_74; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_74) : "r"(*reinterpret_cast(&_hxa_74)), "r"(*reinterpret_cast(&_hxb_74))); + __nv_bfloat162 _max_73 = *reinterpret_cast(&_hxr_74); + amax_0 = _max_73; + unsigned int mag_1 = __as_u32(amax_0) & 2147450879; + unsigned int peer1_2 = __shfl_xor_sync(0xFFFFFFFF, mag_1, 1); + __nv_bfloat162 pair_max_3 = __hmax2_nan(__as_bf16x2(mag_1), __as_bf16x2(peer1_2)); + unsigned int peer2_4 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(pair_max_3), 2); + __nv_bfloat162 quad_max_5 = __hmax2_nan(pair_max_3, __as_bf16x2(peer2_4)); + unsigned int folded_6 = __as_u32(__hmax2_nan(quad_max_5, __as_bf16x2(__as_u32(quad_max_5) >> 16))); + unsigned int amax_bits_7 = (folded_6 & 65535) << 16; + int scale_i32_8 = (int)(amax_bits_7 + 2031616 >> 23) - 8; + if (scale_i32_8 < 0) { + scale_i32_8 = 0; + } + if ((amax_bits_7 & 2139095040) == 2139095040) { + scale_i32_8 = 255; + } + unsigned int scale_9 = (unsigned int)scale_i32_8; + unsigned int inv_10 = 254 - scale_9 << 7; + if (scale_9 == 255) { + inv_10 = 32704; + } + unsigned int inv2_11 = inv_10 | inv_10 << 16; + int scale_index_12 = ((grow >> 7) * scale_cols + (bx + (K >> 5) >> 2)) * 512 + (grow & 31) * 16 + (grow >> 5 & 3) * 4 + (bx + (K >> 5) & 3); + if (quarter == 0) { + *(reinterpret_cast(row_scales + scale_index_12) + (0)) = (unsigned char)(scale_9); + } + unsigned int words_13[2]; + uint32_t _bf16x2_mul_4; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_4) : "r"(dup[0]), "r"(inv2_11)); + uint16_t _e4m3x2_bf16x2_2; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_2) : "r"(_bf16x2_mul_4)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_4); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_2) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_5; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_5) : "r"(dup[1]), "r"(inv2_11)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_76; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_76) : "r"(_bf16x2_mul_5)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_5); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_76) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_75; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_75) : "h"(_e4m3x2_bf16x2_2), "h"(_cvt_satfinite_e4m3x2_bf16x2_76)); + words_13[0] = _mov_b32_pack_u16x2_75; + uint32_t _bf16x2_mul_6; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_6) : "r"(dup[2]), "r"(inv2_11)); + uint16_t _e4m3x2_bf16x2_3; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_3) : "r"(_bf16x2_mul_6)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_6); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_3) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_7; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_7) : "r"(dup[3]), "r"(inv2_11)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_78; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_78) : "r"(_bf16x2_mul_7)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_7); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_78) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_77; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_77) : "h"(_e4m3x2_bf16x2_3), "h"(_cvt_satfinite_e4m3x2_bf16x2_78)); + words_13[1] = _mov_b32_pack_u16x2_77; + reinterpret_cast(row_output + (out_index + K))[0] = reinterpret_cast(words_13 + 0)[0]; + __syncthreads(); + int lane_1 = tid & 31; + int warp_1 = tid >> 5; + int k8 = lane_1 & 7; + int srow = (k8 >> 1) * 8 + (lane_1 >> 3) * 2 + (k8 & 1); + unsigned int sswizzle = (unsigned int)(srow >> 3 & 3); + unsigned int cols[4]; + unsigned int src = tiles_addr + (unsigned int)(srow * 64) + (((unsigned int)warp_1 ^ sswizzle) << 4); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols[0]), "=r"(cols[1]), "=r"(cols[2]), "=r"(cols[3]) + : "r"(src) + : "memory"); + __nv_bfloat162 amax_14 = __as_bf16x2(0); + __nv_bfloat162 _hxa_80 = amax_14; + __nv_bfloat162 _hxb_80 = __as_bf16x2(cols[0]); + unsigned int _hxr_80; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_80) : "r"(*reinterpret_cast(&_hxa_80)), "r"(*reinterpret_cast(&_hxb_80))); + __nv_bfloat162 _max_79 = *reinterpret_cast(&_hxr_80); + amax_14 = _max_79; + __nv_bfloat162 _hxa_82 = amax_14; + __nv_bfloat162 _hxb_82 = __as_bf16x2(cols[1]); + unsigned int _hxr_82; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_82) : "r"(*reinterpret_cast(&_hxa_82)), "r"(*reinterpret_cast(&_hxb_82))); + __nv_bfloat162 _max_81 = *reinterpret_cast(&_hxr_82); + amax_14 = _max_81; + __nv_bfloat162 _hxa_84 = amax_14; + __nv_bfloat162 _hxb_84 = __as_bf16x2(cols[2]); + unsigned int _hxr_84; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_84) : "r"(*reinterpret_cast(&_hxa_84)), "r"(*reinterpret_cast(&_hxb_84))); + __nv_bfloat162 _max_83 = *reinterpret_cast(&_hxr_84); + amax_14 = _max_83; + __nv_bfloat162 _hxa_86 = amax_14; + __nv_bfloat162 _hxb_86 = __as_bf16x2(cols[3]); + unsigned int _hxr_86; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_86) : "r"(*reinterpret_cast(&_hxa_86)), "r"(*reinterpret_cast(&_hxb_86))); + __nv_bfloat162 _max_85 = *reinterpret_cast(&_hxr_86); + amax_14 = _max_85; + unsigned int mag_15 = __as_u32(amax_14) & 2147450879; + __nv_bfloat162 col_max = __hmax2_nan(__as_bf16x2(mag_15), __as_bf16x2(mag_15 >> 16)); + unsigned int peer1_16 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(col_max), 1); + col_max = __hmax2_nan(col_max, __as_bf16x2(peer1_16)); + unsigned int peer2_17 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(col_max), 2); + col_max = __hmax2_nan(col_max, __as_bf16x2(peer2_17)); + unsigned int amax_bits_18 = (__as_u32(col_max) & 65535) << 16; + int scale_i32_19 = (int)(amax_bits_18 + 2031616 >> 23) - 8; + if (scale_i32_19 < 0) { + scale_i32_19 = 0; + } + if ((amax_bits_18 & 2139095040) == 2139095040) { + scale_i32_19 = 255; + } + unsigned int scale_20 = (unsigned int)scale_i32_19; + unsigned int inv_21 = 254 - scale_20 << 7; + if (scale_20 == 255) { + inv_21 = 32704; + } + unsigned int inv2_22 = inv_21 | inv_21 << 16; + int gcol = bx * 32 + warp_1 * 8 + (lane_1 >> 2); + int out_row = by * 32 + (lane_1 & 3) * 8; + unsigned int words_23[2]; + uint32_t _bf16x2_mul_8; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_8) : "r"(cols[0]), "r"(inv2_22)); + uint16_t _e4m3x2_bf16x2_4; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_4) : "r"(_bf16x2_mul_8)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_8); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_4) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_9; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_9) : "r"(cols[1]), "r"(inv2_22)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_88; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_88) : "r"(_bf16x2_mul_9)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_9); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_88) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_87; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_87) : "h"(_e4m3x2_bf16x2_4), "h"(_cvt_satfinite_e4m3x2_bf16x2_88)); + words_23[0] = _mov_b32_pack_u16x2_87; + uint32_t _bf16x2_mul_10; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_10) : "r"(cols[2]), "r"(inv2_22)); + uint16_t _e4m3x2_bf16x2_5; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_5) : "r"(_bf16x2_mul_10)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_10); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_5) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_11; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_11) : "r"(cols[3]), "r"(inv2_22)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_90; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_90) : "r"(_bf16x2_mul_11)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_11); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_90) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_89; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_89) : "h"(_e4m3x2_bf16x2_5), "h"(_cvt_satfinite_e4m3x2_bf16x2_90)); + words_23[1] = _mov_b32_pack_u16x2_89; + reinterpret_cast(col_output + (gcol * M + out_row))[0] = reinterpret_cast(words_23 + 0)[0]; + if ((lane_1 & 3) == 0) { + int scale_index_0 = ((gcol >> 7) * (M >> 7) + (by >> 2)) * 512 + (gcol & 31) * 16 + (gcol >> 5 & 3) * 4 + (by & 3); + *(reinterpret_cast(col_scales + scale_index_0) + (0)) = (unsigned char)(scale_20); + } + int lane_24 = tid & 31; + int warp_25 = tid >> 5; + int k8_26 = lane_24 & 7; + int srow_27 = (k8_26 >> 1) * 8 + (lane_24 >> 3) * 2 + (k8_26 & 1); + unsigned int sswizzle_28 = (unsigned int)(srow_27 >> 3 & 3); + unsigned int cols_29[4]; + unsigned int src_30 = tiles_addr + 2048 + (unsigned int)(srow_27 * 64) + (((unsigned int)warp_25 ^ sswizzle_28) << 4); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols_29[0]), "=r"(cols_29[1]), "=r"(cols_29[2]), "=r"(cols_29[3]) + : "r"(src_30) + : "memory"); + __nv_bfloat162 amax_31 = __as_bf16x2(0); + __nv_bfloat162 _hxa_92 = amax_31; + __nv_bfloat162 _hxb_92 = __as_bf16x2(cols_29[0]); + unsigned int _hxr_92; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_92) : "r"(*reinterpret_cast(&_hxa_92)), "r"(*reinterpret_cast(&_hxb_92))); + __nv_bfloat162 _max_91 = *reinterpret_cast(&_hxr_92); + amax_31 = _max_91; + __nv_bfloat162 _hxa_94 = amax_31; + __nv_bfloat162 _hxb_94 = __as_bf16x2(cols_29[1]); + unsigned int _hxr_94; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_94) : "r"(*reinterpret_cast(&_hxa_94)), "r"(*reinterpret_cast(&_hxb_94))); + __nv_bfloat162 _max_93 = *reinterpret_cast(&_hxr_94); + amax_31 = _max_93; + __nv_bfloat162 _hxa_96 = amax_31; + __nv_bfloat162 _hxb_96 = __as_bf16x2(cols_29[2]); + unsigned int _hxr_96; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_96) : "r"(*reinterpret_cast(&_hxa_96)), "r"(*reinterpret_cast(&_hxb_96))); + __nv_bfloat162 _max_95 = *reinterpret_cast(&_hxr_96); + amax_31 = _max_95; + __nv_bfloat162 _hxa_98 = amax_31; + __nv_bfloat162 _hxb_98 = __as_bf16x2(cols_29[3]); + unsigned int _hxr_98; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_98) : "r"(*reinterpret_cast(&_hxa_98)), "r"(*reinterpret_cast(&_hxb_98))); + __nv_bfloat162 _max_97 = *reinterpret_cast(&_hxr_98); + amax_31 = _max_97; + unsigned int mag_32 = __as_u32(amax_31) & 2147450879; + __nv_bfloat162 col_max_33 = __hmax2_nan(__as_bf16x2(mag_32), __as_bf16x2(mag_32 >> 16)); + unsigned int peer1_34 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(col_max_33), 1); + col_max_33 = __hmax2_nan(col_max_33, __as_bf16x2(peer1_34)); + unsigned int peer2_35 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(col_max_33), 2); + col_max_33 = __hmax2_nan(col_max_33, __as_bf16x2(peer2_35)); + unsigned int amax_bits_36 = (__as_u32(col_max_33) & 65535) << 16; + int scale_i32_37 = (int)(amax_bits_36 + 2031616 >> 23) - 8; + if (scale_i32_37 < 0) { + scale_i32_37 = 0; + } + if ((amax_bits_36 & 2139095040) == 2139095040) { + scale_i32_37 = 255; + } + unsigned int scale_38 = (unsigned int)scale_i32_37; + unsigned int inv_39 = 254 - scale_38 << 7; + if (scale_38 == 255) { + inv_39 = 32704; + } + unsigned int inv2_40 = inv_39 | inv_39 << 16; + int gcol_41 = bx * 32 + K + warp_25 * 8 + (lane_24 >> 2); + int out_row_42 = by * 32 + (lane_24 & 3) * 8; + unsigned int words_43[2]; + uint32_t _bf16x2_mul_12; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_12) : "r"(cols_29[0]), "r"(inv2_40)); + uint16_t _e4m3x2_bf16x2_6; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_6) : "r"(_bf16x2_mul_12)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_12); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_6) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_13; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_13) : "r"(cols_29[1]), "r"(inv2_40)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_100; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_100) : "r"(_bf16x2_mul_13)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_13); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_100) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_99; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_99) : "h"(_e4m3x2_bf16x2_6), "h"(_cvt_satfinite_e4m3x2_bf16x2_100)); + words_43[0] = _mov_b32_pack_u16x2_99; + uint32_t _bf16x2_mul_14; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_14) : "r"(cols_29[2]), "r"(inv2_40)); + uint16_t _e4m3x2_bf16x2_7; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_7) : "r"(_bf16x2_mul_14)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_14); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_7) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_15; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_15) : "r"(cols_29[3]), "r"(inv2_40)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_102; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_102) : "r"(_bf16x2_mul_15)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_15); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_102) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_101; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_101) : "h"(_e4m3x2_bf16x2_7), "h"(_cvt_satfinite_e4m3x2_bf16x2_102)); + words_43[1] = _mov_b32_pack_u16x2_101; + reinterpret_cast(col_output + (gcol_41 * M + out_row_42))[0] = reinterpret_cast(words_43 + 0)[0]; + if ((lane_24 & 3) == 0) { + int scale_index_0_1 = ((gcol_41 >> 7) * (M >> 7) + (by >> 2)) * 512 + (gcol_41 & 31) * 16 + (gcol_41 >> 5 & 3) * 4 + (by & 3); + *(reinterpret_cast(col_scales + scale_index_0_1) + (0)) = (unsigned char)(scale_38); + } +} + +} // extern "C" diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_077a568b810590574917_binding.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_3e0ac746df31b17317e0_binding.cu similarity index 91% rename from flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_077a568b810590574917_binding.cu rename to flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_3e0ac746df31b17317e0_binding.cu index d6736995a7c..3cf34a923e8 100644 --- a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_077a568b810590574917_binding.cu +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_3e0ac746df31b17317e0_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gated_act_mxfp8_077a568b810590574917, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gated_act_mxfp8_3e0ac746df31b17317e0, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K); +extern "C" __global__ void kernel_cake_gated_act_mxfp8_3e0ac746df31b17317e0(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K); -namespace cake_binding_kernel_cake_gated_act_mxfp8_077a568b810590574917 { +namespace cake_binding_kernel_cake_gated_act_mxfp8_3e0ac746df31b17317e0 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gated_act_mxfp8_077a568b810590574917"; +constexpr const char* KERNEL_NAME = "kernel_cake_gated_act_mxfp8_3e0ac746df31b17317e0"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -88,11 +88,11 @@ void Run(TensorView a_gated_input, TensorView a_grad, TensorView a_row_output, T config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 0; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gated_act_mxfp8_077a568b810590574917), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gated_act_mxfp8_3e0ac746df31b17317e0), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gated_act_mxfp8_077a568b810590574917 +} // namespace cake_binding_kernel_cake_gated_act_mxfp8_3e0ac746df31b17317e0 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gated_act_mxfp8_077a568b810590574917::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gated_act_mxfp8_3e0ac746df31b17317e0::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_077a568b810590574917_kernel.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_3e0ac746df31b17317e0_kernel.cu similarity index 65% rename from flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_077a568b810590574917_kernel.cu rename to flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_3e0ac746df31b17317e0_kernel.cu index f60c4c1f9e2..92dae7d1a57 100644 --- a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_077a568b810590574917_kernel.cu +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_3e0ac746df31b17317e0_kernel.cu @@ -50,7 +50,7 @@ __device__ __forceinline__ int make_warp_uniform(int x) { #define CAKE_INF CUDART_INF_F #define NUM_MAIN_STAGES 1 #define THREADS 128 -#define LAUNCH_MIN_BLOCKS 12 +#define LAUNCH_MIN_BLOCKS 10 #include @@ -141,7 +141,7 @@ __device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { extern "C" { __global__ __launch_bounds__(128, LAUNCH_MIN_BLOCKS) void -kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K) +kernel_cake_gated_act_mxfp8_3e0ac746df31b17317e0(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -243,26 +243,24 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_1 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[0])); float2 _cvt_f32_2 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[0])); float2 _cvt_f32_3 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[0])); - float e0_1 = approx_exp2((-_cvt_f32_1.x) * 1.442695f); - float e1_1 = approx_exp2((-_cvt_f32_1.y) * 1.442695f); - float d0_1 = 1.0f + e0_1; - float d1_1 = 1.0f + e1_1; + float2 _f32x2_mul_b_5 = make_float2(-1.442695f, -1.442695f); + float2 t_1; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_1) : "l"(*(const unsigned long long*)&_cvt_f32_1), "l"(*(const unsigned long long*)&_f32x2_mul_b_5)); + float e0_1 = approx_exp2(t_1.x); + float e1_1 = approx_exp2(t_1.y); + float2 nd_1 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_1, e1_1)); + float d0_1 = -nd_1.x; + float d1_1 = -nd_1.y; float r0_1 = approx_rcp(d0_1); float r1_1 = approx_rcp(d1_1); - float _fma_6; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_6) : "f"(-d0_1), "f"(r0_1), "f"(1.0f)); - float _fma_5; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_5) : "f"(r0_1), "f"(_fma_6), "f"(r0_1)); - float _fma_8; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_8) : "f"(-d1_1), "f"(r1_1), "f"(1.0f)); - float _fma_7; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_7) : "f"(r1_1), "f"(_fma_8), "f"(r1_1)); - float2 s_0 = make_float2(_fma_5, _fma_7); + float2 r_1 = make_float2(r0_1, r1_1); + float2 s_1 = fma_f32x2_rn_noftz(r_1, fma_f32x2_rn_noftz(nd_1, r_1, make_float2(1.0f, 1.0f)), r_1); + float2 s_0 = s_1; float2 act_0; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_0) : "l"(*(const unsigned long long*)&_cvt_f32_1), "l"(*(const unsigned long long*)&s_0)); - float2 _f32x2_mul_a_9 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_0, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_0), s_0), _cvt_f32_3); + float2 _f32x2_mul_a_6 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_0, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_0), s_0), _cvt_f32_3); float2 _mul_0; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_9), "l"(*(const unsigned long long*)&_cvt_f32_2)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_6), "l"(*(const unsigned long long*)&_cvt_f32_2)); float2 _mul_1; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_1) : "l"(*(const unsigned long long*)&act_0), "l"(*(const unsigned long long*)&_cvt_f32_3)); __nv_bfloat162 _bf16x2_0 = __float22bfloat162_rn(make_float2(_mul_0.x, _mul_0.y)); @@ -272,26 +270,24 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_4 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[1])); float2 _cvt_f32_5 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[1])); float2 _cvt_f32_6 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[1])); - float e0_3 = approx_exp2((-_cvt_f32_4.x) * 1.442695f); - float e1_3 = approx_exp2((-_cvt_f32_4.y) * 1.442695f); - float d0_3 = 1.0f + e0_3; - float d1_3 = 1.0f + e1_3; + float2 _f32x2_mul_b_7 = make_float2(-1.442695f, -1.442695f); + float2 t_3; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_3) : "l"(*(const unsigned long long*)&_cvt_f32_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_7)); + float e0_3 = approx_exp2(t_3.x); + float e1_3 = approx_exp2(t_3.y); + float2 nd_3 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_3, e1_3)); + float d0_3 = -nd_3.x; + float d1_3 = -nd_3.y; float r0_3 = approx_rcp(d0_3); float r1_3 = approx_rcp(d1_3); - float _fma_11; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_11) : "f"(-d0_3), "f"(r0_3), "f"(1.0f)); - float _fma_10; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_10) : "f"(r0_3), "f"(_fma_11), "f"(r0_3)); - float _fma_13; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_13) : "f"(-d1_3), "f"(r1_3), "f"(1.0f)); - float _fma_12; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_12) : "f"(r1_3), "f"(_fma_13), "f"(r1_3)); - float2 s_2 = make_float2(_fma_10, _fma_12); + float2 r_3 = make_float2(r0_3, r1_3); + float2 s_3 = fma_f32x2_rn_noftz(r_3, fma_f32x2_rn_noftz(nd_3, r_3, make_float2(1.0f, 1.0f)), r_3); + float2 s_2 = s_3; float2 act_2; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_2) : "l"(*(const unsigned long long*)&_cvt_f32_4), "l"(*(const unsigned long long*)&s_2)); - float2 _f32x2_mul_a_14 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_2, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_2), s_2), _cvt_f32_6); + float2 _f32x2_mul_a_8 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_2, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_2), s_2), _cvt_f32_6); float2 _mul_2; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_14), "l"(*(const unsigned long long*)&_cvt_f32_5)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_8), "l"(*(const unsigned long long*)&_cvt_f32_5)); float2 _mul_3; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_3) : "l"(*(const unsigned long long*)&act_2), "l"(*(const unsigned long long*)&_cvt_f32_6)); __nv_bfloat162 _bf16x2_2 = __float22bfloat162_rn(make_float2(_mul_2.x, _mul_2.y)); @@ -301,26 +297,24 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_7 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[2])); float2 _cvt_f32_8 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[2])); float2 _cvt_f32_9 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[2])); - float e0_5 = approx_exp2((-_cvt_f32_7.x) * 1.442695f); - float e1_5 = approx_exp2((-_cvt_f32_7.y) * 1.442695f); - float d0_5 = 1.0f + e0_5; - float d1_5 = 1.0f + e1_5; + float2 _f32x2_mul_b_9 = make_float2(-1.442695f, -1.442695f); + float2 t_5; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_5) : "l"(*(const unsigned long long*)&_cvt_f32_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_9)); + float e0_5 = approx_exp2(t_5.x); + float e1_5 = approx_exp2(t_5.y); + float2 nd_5 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_5, e1_5)); + float d0_5 = -nd_5.x; + float d1_5 = -nd_5.y; float r0_5 = approx_rcp(d0_5); float r1_5 = approx_rcp(d1_5); - float _fma_16; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_16) : "f"(-d0_5), "f"(r0_5), "f"(1.0f)); - float _fma_15; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_15) : "f"(r0_5), "f"(_fma_16), "f"(r0_5)); - float _fma_18; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_18) : "f"(-d1_5), "f"(r1_5), "f"(1.0f)); - float _fma_17; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_17) : "f"(r1_5), "f"(_fma_18), "f"(r1_5)); - float2 s_4 = make_float2(_fma_15, _fma_17); + float2 r_5 = make_float2(r0_5, r1_5); + float2 s_5 = fma_f32x2_rn_noftz(r_5, fma_f32x2_rn_noftz(nd_5, r_5, make_float2(1.0f, 1.0f)), r_5); + float2 s_4 = s_5; float2 act_4; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_4) : "l"(*(const unsigned long long*)&_cvt_f32_7), "l"(*(const unsigned long long*)&s_4)); - float2 _f32x2_mul_a_19 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_4, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_4), s_4), _cvt_f32_9); + float2 _f32x2_mul_a_10 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_4, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_4), s_4), _cvt_f32_9); float2 _mul_4; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_19), "l"(*(const unsigned long long*)&_cvt_f32_8)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_10), "l"(*(const unsigned long long*)&_cvt_f32_8)); float2 _mul_5; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_5) : "l"(*(const unsigned long long*)&act_4), "l"(*(const unsigned long long*)&_cvt_f32_9)); __nv_bfloat162 _bf16x2_4 = __float22bfloat162_rn(make_float2(_mul_4.x, _mul_4.y)); @@ -330,26 +324,24 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_10 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[3])); float2 _cvt_f32_11 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[3])); float2 _cvt_f32_12 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[3])); - float e0_7 = approx_exp2((-_cvt_f32_10.x) * 1.442695f); - float e1_7 = approx_exp2((-_cvt_f32_10.y) * 1.442695f); - float d0_7 = 1.0f + e0_7; - float d1_7 = 1.0f + e1_7; + float2 _f32x2_mul_b_11 = make_float2(-1.442695f, -1.442695f); + float2 t_7; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_7) : "l"(*(const unsigned long long*)&_cvt_f32_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_11)); + float e0_7 = approx_exp2(t_7.x); + float e1_7 = approx_exp2(t_7.y); + float2 nd_7 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_7, e1_7)); + float d0_7 = -nd_7.x; + float d1_7 = -nd_7.y; float r0_7 = approx_rcp(d0_7); float r1_7 = approx_rcp(d1_7); - float _fma_21; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_21) : "f"(-d0_7), "f"(r0_7), "f"(1.0f)); - float _fma_20; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_20) : "f"(r0_7), "f"(_fma_21), "f"(r0_7)); - float _fma_23; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_23) : "f"(-d1_7), "f"(r1_7), "f"(1.0f)); - float _fma_22; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_22) : "f"(r1_7), "f"(_fma_23), "f"(r1_7)); - float2 s_6 = make_float2(_fma_20, _fma_22); + float2 r_7 = make_float2(r0_7, r1_7); + float2 s_7 = fma_f32x2_rn_noftz(r_7, fma_f32x2_rn_noftz(nd_7, r_7, make_float2(1.0f, 1.0f)), r_7); + float2 s_6 = s_7; float2 act_6; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_6) : "l"(*(const unsigned long long*)&_cvt_f32_10), "l"(*(const unsigned long long*)&s_6)); - float2 _f32x2_mul_a_24 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_6, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_6), s_6), _cvt_f32_12); + float2 _f32x2_mul_a_12 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_6, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_6), s_6), _cvt_f32_12); float2 _mul_6; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_24), "l"(*(const unsigned long long*)&_cvt_f32_11)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_12), "l"(*(const unsigned long long*)&_cvt_f32_11)); float2 _mul_7; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_7) : "l"(*(const unsigned long long*)&act_6), "l"(*(const unsigned long long*)&_cvt_f32_12)); __nv_bfloat162 _bf16x2_6 = __float22bfloat162_rn(make_float2(_mul_6.x, _mul_6.y)); @@ -359,26 +351,24 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_13 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[4])); float2 _cvt_f32_14 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[4])); float2 _cvt_f32_15 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[4])); - float e0_9 = approx_exp2((-_cvt_f32_13.x) * 1.442695f); - float e1_9 = approx_exp2((-_cvt_f32_13.y) * 1.442695f); - float d0_9 = 1.0f + e0_9; - float d1_9 = 1.0f + e1_9; + float2 _f32x2_mul_b_13 = make_float2(-1.442695f, -1.442695f); + float2 t_9; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_9) : "l"(*(const unsigned long long*)&_cvt_f32_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_13)); + float e0_9 = approx_exp2(t_9.x); + float e1_9 = approx_exp2(t_9.y); + float2 nd_9 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_9, e1_9)); + float d0_9 = -nd_9.x; + float d1_9 = -nd_9.y; float r0_9 = approx_rcp(d0_9); float r1_9 = approx_rcp(d1_9); - float _fma_26; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_26) : "f"(-d0_9), "f"(r0_9), "f"(1.0f)); - float _fma_25; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_25) : "f"(r0_9), "f"(_fma_26), "f"(r0_9)); - float _fma_28; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_28) : "f"(-d1_9), "f"(r1_9), "f"(1.0f)); - float _fma_27; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_27) : "f"(r1_9), "f"(_fma_28), "f"(r1_9)); - float2 s_8 = make_float2(_fma_25, _fma_27); + float2 r_9 = make_float2(r0_9, r1_9); + float2 s_9 = fma_f32x2_rn_noftz(r_9, fma_f32x2_rn_noftz(nd_9, r_9, make_float2(1.0f, 1.0f)), r_9); + float2 s_8 = s_9; float2 act_8; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_8) : "l"(*(const unsigned long long*)&_cvt_f32_13), "l"(*(const unsigned long long*)&s_8)); - float2 _f32x2_mul_a_29 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_8, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_8), s_8), _cvt_f32_15); + float2 _f32x2_mul_a_14 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_8, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_8), s_8), _cvt_f32_15); float2 _mul_8; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_29), "l"(*(const unsigned long long*)&_cvt_f32_14)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_14), "l"(*(const unsigned long long*)&_cvt_f32_14)); float2 _mul_9; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_9) : "l"(*(const unsigned long long*)&act_8), "l"(*(const unsigned long long*)&_cvt_f32_15)); __nv_bfloat162 _bf16x2_8 = __float22bfloat162_rn(make_float2(_mul_8.x, _mul_8.y)); @@ -388,26 +378,24 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_16 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[5])); float2 _cvt_f32_17 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[5])); float2 _cvt_f32_18 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[5])); - float e0_11 = approx_exp2((-_cvt_f32_16.x) * 1.442695f); - float e1_11 = approx_exp2((-_cvt_f32_16.y) * 1.442695f); - float d0_11 = 1.0f + e0_11; - float d1_11 = 1.0f + e1_11; + float2 _f32x2_mul_b_15 = make_float2(-1.442695f, -1.442695f); + float2 t_11; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_11) : "l"(*(const unsigned long long*)&_cvt_f32_16), "l"(*(const unsigned long long*)&_f32x2_mul_b_15)); + float e0_11 = approx_exp2(t_11.x); + float e1_11 = approx_exp2(t_11.y); + float2 nd_11 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_11, e1_11)); + float d0_11 = -nd_11.x; + float d1_11 = -nd_11.y; float r0_11 = approx_rcp(d0_11); float r1_11 = approx_rcp(d1_11); - float _fma_31; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_31) : "f"(-d0_11), "f"(r0_11), "f"(1.0f)); - float _fma_30; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_30) : "f"(r0_11), "f"(_fma_31), "f"(r0_11)); - float _fma_33; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_33) : "f"(-d1_11), "f"(r1_11), "f"(1.0f)); - float _fma_32; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_32) : "f"(r1_11), "f"(_fma_33), "f"(r1_11)); - float2 s_10 = make_float2(_fma_30, _fma_32); + float2 r_11 = make_float2(r0_11, r1_11); + float2 s_11 = fma_f32x2_rn_noftz(r_11, fma_f32x2_rn_noftz(nd_11, r_11, make_float2(1.0f, 1.0f)), r_11); + float2 s_10 = s_11; float2 act_10; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_10) : "l"(*(const unsigned long long*)&_cvt_f32_16), "l"(*(const unsigned long long*)&s_10)); - float2 _f32x2_mul_a_34 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_10, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_10), s_10), _cvt_f32_18); + float2 _f32x2_mul_a_16 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_10, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_10), s_10), _cvt_f32_18); float2 _mul_10; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_34), "l"(*(const unsigned long long*)&_cvt_f32_17)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_16), "l"(*(const unsigned long long*)&_cvt_f32_17)); float2 _mul_11; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_11) : "l"(*(const unsigned long long*)&act_10), "l"(*(const unsigned long long*)&_cvt_f32_18)); __nv_bfloat162 _bf16x2_10 = __float22bfloat162_rn(make_float2(_mul_10.x, _mul_10.y)); @@ -417,26 +405,24 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_19 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[6])); float2 _cvt_f32_20 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[6])); float2 _cvt_f32_21 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[6])); - float e0_13 = approx_exp2((-_cvt_f32_19.x) * 1.442695f); - float e1_13 = approx_exp2((-_cvt_f32_19.y) * 1.442695f); - float d0_13 = 1.0f + e0_13; - float d1_13 = 1.0f + e1_13; + float2 _f32x2_mul_b_17 = make_float2(-1.442695f, -1.442695f); + float2 t_13; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_13) : "l"(*(const unsigned long long*)&_cvt_f32_19), "l"(*(const unsigned long long*)&_f32x2_mul_b_17)); + float e0_13 = approx_exp2(t_13.x); + float e1_13 = approx_exp2(t_13.y); + float2 nd_13 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_13, e1_13)); + float d0_13 = -nd_13.x; + float d1_13 = -nd_13.y; float r0_13 = approx_rcp(d0_13); float r1_13 = approx_rcp(d1_13); - float _fma_36; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_36) : "f"(-d0_13), "f"(r0_13), "f"(1.0f)); - float _fma_35; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_35) : "f"(r0_13), "f"(_fma_36), "f"(r0_13)); - float _fma_38; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_38) : "f"(-d1_13), "f"(r1_13), "f"(1.0f)); - float _fma_37; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_37) : "f"(r1_13), "f"(_fma_38), "f"(r1_13)); - float2 s_12 = make_float2(_fma_35, _fma_37); + float2 r_13 = make_float2(r0_13, r1_13); + float2 s_13 = fma_f32x2_rn_noftz(r_13, fma_f32x2_rn_noftz(nd_13, r_13, make_float2(1.0f, 1.0f)), r_13); + float2 s_12 = s_13; float2 act_12; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_12) : "l"(*(const unsigned long long*)&_cvt_f32_19), "l"(*(const unsigned long long*)&s_12)); - float2 _f32x2_mul_a_39 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_12, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_12), s_12), _cvt_f32_21); + float2 _f32x2_mul_a_18 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_12, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_12), s_12), _cvt_f32_21); float2 _mul_12; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_39), "l"(*(const unsigned long long*)&_cvt_f32_20)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_18), "l"(*(const unsigned long long*)&_cvt_f32_20)); float2 _mul_13; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_13) : "l"(*(const unsigned long long*)&act_12), "l"(*(const unsigned long long*)&_cvt_f32_21)); __nv_bfloat162 _bf16x2_12 = __float22bfloat162_rn(make_float2(_mul_12.x, _mul_12.y)); @@ -446,26 +432,24 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_22 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[7])); float2 _cvt_f32_23 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[7])); float2 _cvt_f32_24 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[7])); - float e0_15 = approx_exp2((-_cvt_f32_22.x) * 1.442695f); - float e1_15 = approx_exp2((-_cvt_f32_22.y) * 1.442695f); - float d0_15 = 1.0f + e0_15; - float d1_15 = 1.0f + e1_15; + float2 _f32x2_mul_b_19 = make_float2(-1.442695f, -1.442695f); + float2 t_15; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_15) : "l"(*(const unsigned long long*)&_cvt_f32_22), "l"(*(const unsigned long long*)&_f32x2_mul_b_19)); + float e0_15 = approx_exp2(t_15.x); + float e1_15 = approx_exp2(t_15.y); + float2 nd_15 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_15, e1_15)); + float d0_15 = -nd_15.x; + float d1_15 = -nd_15.y; float r0_15 = approx_rcp(d0_15); float r1_15 = approx_rcp(d1_15); - float _fma_41; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_41) : "f"(-d0_15), "f"(r0_15), "f"(1.0f)); - float _fma_40; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_40) : "f"(r0_15), "f"(_fma_41), "f"(r0_15)); - float _fma_43; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_43) : "f"(-d1_15), "f"(r1_15), "f"(1.0f)); - float _fma_42; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_42) : "f"(r1_15), "f"(_fma_43), "f"(r1_15)); - float2 s_14 = make_float2(_fma_40, _fma_42); + float2 r_15 = make_float2(r0_15, r1_15); + float2 s_15 = fma_f32x2_rn_noftz(r_15, fma_f32x2_rn_noftz(nd_15, r_15, make_float2(1.0f, 1.0f)), r_15); + float2 s_14 = s_15; float2 act_14; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_14) : "l"(*(const unsigned long long*)&_cvt_f32_22), "l"(*(const unsigned long long*)&s_14)); - float2 _f32x2_mul_a_44 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_14, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_14), s_14), _cvt_f32_24); + float2 _f32x2_mul_a_20 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_14, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_14), s_14), _cvt_f32_24); float2 _mul_14; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_44), "l"(*(const unsigned long long*)&_cvt_f32_23)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_20), "l"(*(const unsigned long long*)&_cvt_f32_23)); float2 _mul_15; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_15) : "l"(*(const unsigned long long*)&act_14), "l"(*(const unsigned long long*)&_cvt_f32_24)); __nv_bfloat162 _bf16x2_14 = __float22bfloat162_rn(make_float2(_mul_14.x, _mul_14.y)); @@ -477,29 +461,27 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_25 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[0])); float2 _cvt_f32_26 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[0])); float2 _cvt_f32_27 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[0])); - float e0_17 = approx_exp2((-_cvt_f32_25.x) * 1.442695f); - float e1_17 = approx_exp2((-_cvt_f32_25.y) * 1.442695f); - float d0_17 = 1.0f + e0_17; - float d1_17 = 1.0f + e1_17; + float2 _f32x2_mul_b_21 = make_float2(-1.442695f, -1.442695f); + float2 t_17; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_17) : "l"(*(const unsigned long long*)&_cvt_f32_25), "l"(*(const unsigned long long*)&_f32x2_mul_b_21)); + float e0_17 = approx_exp2(t_17.x); + float e1_17 = approx_exp2(t_17.y); + float2 nd_17 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_17, e1_17)); + float d0_17 = -nd_17.x; + float d1_17 = -nd_17.y; float r0_17 = approx_rcp(d0_17); float r1_17 = approx_rcp(d1_17); - float _fma_45; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_45) : "f"(-d0_17), "f"(r0_17), "f"(1.0f)); - float _fma_0; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_0) : "f"(r0_17), "f"(_fma_45), "f"(r0_17)); - float _fma_46; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_46) : "f"(-d1_17), "f"(r1_17), "f"(1.0f)); - float _fma_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_1) : "f"(r1_17), "f"(_fma_46), "f"(r1_17)); - float t0_17 = _fma_0; - float t1_17 = _fma_1; + float2 r_17 = make_float2(r0_17, r1_17); + float2 s_17 = fma_f32x2_rn_noftz(r_17, fma_f32x2_rn_noftz(nd_17, r_17, make_float2(1.0f, 1.0f)), r_17); + float t0_17 = s_17.x; + float t1_17 = s_17.y; band = band | ((fminf(t0_17, t1_17) == 0.0f) ? 1 : 0); float2 s_16 = make_float2(max_noftz(t0_17, 0.0f), max_noftz(t1_17, 0.0f)); float2 act_16; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_16) : "l"(*(const unsigned long long*)&_cvt_f32_25), "l"(*(const unsigned long long*)&s_16)); - float2 _f32x2_mul_a_47 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_16, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_16), s_16), _cvt_f32_27); + float2 _f32x2_mul_a_22 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_16, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_16), s_16), _cvt_f32_27); float2 _mul_16; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_16) : "l"(*(const unsigned long long*)&_f32x2_mul_a_47), "l"(*(const unsigned long long*)&_cvt_f32_26)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_16) : "l"(*(const unsigned long long*)&_f32x2_mul_a_22), "l"(*(const unsigned long long*)&_cvt_f32_26)); float2 _mul_17; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_17) : "l"(*(const unsigned long long*)&act_16), "l"(*(const unsigned long long*)&_cvt_f32_27)); __nv_bfloat162 _bf16x2_16 = __float22bfloat162_rn(make_float2(_mul_16.x, _mul_16.y)); @@ -509,29 +491,27 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_28 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[1])); float2 _cvt_f32_29 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[1])); float2 _cvt_f32_30 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[1])); - float e0_19 = approx_exp2((-_cvt_f32_28.x) * 1.442695f); - float e1_19 = approx_exp2((-_cvt_f32_28.y) * 1.442695f); - float d0_19 = 1.0f + e0_19; - float d1_19 = 1.0f + e1_19; + float2 _f32x2_mul_b_23 = make_float2(-1.442695f, -1.442695f); + float2 t_19; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_19) : "l"(*(const unsigned long long*)&_cvt_f32_28), "l"(*(const unsigned long long*)&_f32x2_mul_b_23)); + float e0_19 = approx_exp2(t_19.x); + float e1_19 = approx_exp2(t_19.y); + float2 nd_19 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_19, e1_19)); + float d0_19 = -nd_19.x; + float d1_19 = -nd_19.y; float r0_19 = approx_rcp(d0_19); float r1_19 = approx_rcp(d1_19); - float _fma_48; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_48) : "f"(-d0_19), "f"(r0_19), "f"(1.0f)); - float _fma_2; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_2) : "f"(r0_19), "f"(_fma_48), "f"(r0_19)); - float _fma_49; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_49) : "f"(-d1_19), "f"(r1_19), "f"(1.0f)); - float _fma_3; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_3) : "f"(r1_19), "f"(_fma_49), "f"(r1_19)); - float t0_19 = _fma_2; - float t1_19 = _fma_3; + float2 r_19 = make_float2(r0_19, r1_19); + float2 s_19 = fma_f32x2_rn_noftz(r_19, fma_f32x2_rn_noftz(nd_19, r_19, make_float2(1.0f, 1.0f)), r_19); + float t0_19 = s_19.x; + float t1_19 = s_19.y; band = band | ((fminf(t0_19, t1_19) == 0.0f) ? 2 : 0); float2 s_18 = make_float2(max_noftz(t0_19, 0.0f), max_noftz(t1_19, 0.0f)); float2 act_18; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_18) : "l"(*(const unsigned long long*)&_cvt_f32_28), "l"(*(const unsigned long long*)&s_18)); - float2 _f32x2_mul_a_50 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_18, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_18), s_18), _cvt_f32_30); + float2 _f32x2_mul_a_24 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_18, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_18), s_18), _cvt_f32_30); float2 _mul_18; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_18) : "l"(*(const unsigned long long*)&_f32x2_mul_a_50), "l"(*(const unsigned long long*)&_cvt_f32_29)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_18) : "l"(*(const unsigned long long*)&_f32x2_mul_a_24), "l"(*(const unsigned long long*)&_cvt_f32_29)); float2 _mul_19; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_19) : "l"(*(const unsigned long long*)&act_18), "l"(*(const unsigned long long*)&_cvt_f32_30)); __nv_bfloat162 _bf16x2_18 = __float22bfloat162_rn(make_float2(_mul_18.x, _mul_18.y)); @@ -541,29 +521,27 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_31 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[2])); float2 _cvt_f32_32 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[2])); float2 _cvt_f32_33 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[2])); - float e0_21 = approx_exp2((-_cvt_f32_31.x) * 1.442695f); - float e1_21 = approx_exp2((-_cvt_f32_31.y) * 1.442695f); - float d0_21 = 1.0f + e0_21; - float d1_21 = 1.0f + e1_21; + float2 _f32x2_mul_b_25 = make_float2(-1.442695f, -1.442695f); + float2 t_21; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_21) : "l"(*(const unsigned long long*)&_cvt_f32_31), "l"(*(const unsigned long long*)&_f32x2_mul_b_25)); + float e0_21 = approx_exp2(t_21.x); + float e1_21 = approx_exp2(t_21.y); + float2 nd_21 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_21, e1_21)); + float d0_21 = -nd_21.x; + float d1_21 = -nd_21.y; float r0_21 = approx_rcp(d0_21); float r1_21 = approx_rcp(d1_21); - float _fma_51; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_51) : "f"(-d0_21), "f"(r0_21), "f"(1.0f)); - float _fma_4; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_4) : "f"(r0_21), "f"(_fma_51), "f"(r0_21)); - float _fma_52; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_52) : "f"(-d1_21), "f"(r1_21), "f"(1.0f)); - float _fma_5_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_5_1) : "f"(r1_21), "f"(_fma_52), "f"(r1_21)); - float t0_21 = _fma_4; - float t1_21 = _fma_5_1; + float2 r_21 = make_float2(r0_21, r1_21); + float2 s_21 = fma_f32x2_rn_noftz(r_21, fma_f32x2_rn_noftz(nd_21, r_21, make_float2(1.0f, 1.0f)), r_21); + float t0_21 = s_21.x; + float t1_21 = s_21.y; band = band | ((fminf(t0_21, t1_21) == 0.0f) ? 4 : 0); float2 s_20 = make_float2(max_noftz(t0_21, 0.0f), max_noftz(t1_21, 0.0f)); float2 act_20; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_20) : "l"(*(const unsigned long long*)&_cvt_f32_31), "l"(*(const unsigned long long*)&s_20)); - float2 _f32x2_mul_a_53 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_20, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_20), s_20), _cvt_f32_33); + float2 _f32x2_mul_a_26 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_20, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_20), s_20), _cvt_f32_33); float2 _mul_20; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_53), "l"(*(const unsigned long long*)&_cvt_f32_32)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_26), "l"(*(const unsigned long long*)&_cvt_f32_32)); float2 _mul_21; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_21) : "l"(*(const unsigned long long*)&act_20), "l"(*(const unsigned long long*)&_cvt_f32_33)); __nv_bfloat162 _bf16x2_20 = __float22bfloat162_rn(make_float2(_mul_20.x, _mul_20.y)); @@ -573,29 +551,27 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_34 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[3])); float2 _cvt_f32_35 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[3])); float2 _cvt_f32_36 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[3])); - float e0_23 = approx_exp2((-_cvt_f32_34.x) * 1.442695f); - float e1_23 = approx_exp2((-_cvt_f32_34.y) * 1.442695f); - float d0_23 = 1.0f + e0_23; - float d1_23 = 1.0f + e1_23; + float2 _f32x2_mul_b_27 = make_float2(-1.442695f, -1.442695f); + float2 t_23; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_23) : "l"(*(const unsigned long long*)&_cvt_f32_34), "l"(*(const unsigned long long*)&_f32x2_mul_b_27)); + float e0_23 = approx_exp2(t_23.x); + float e1_23 = approx_exp2(t_23.y); + float2 nd_23 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_23, e1_23)); + float d0_23 = -nd_23.x; + float d1_23 = -nd_23.y; float r0_23 = approx_rcp(d0_23); float r1_23 = approx_rcp(d1_23); - float _fma_54; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_54) : "f"(-d0_23), "f"(r0_23), "f"(1.0f)); - float _fma_6_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_6_1) : "f"(r0_23), "f"(_fma_54), "f"(r0_23)); - float _fma_55; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_55) : "f"(-d1_23), "f"(r1_23), "f"(1.0f)); - float _fma_7_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_7_1) : "f"(r1_23), "f"(_fma_55), "f"(r1_23)); - float t0_23 = _fma_6_1; - float t1_23 = _fma_7_1; + float2 r_23 = make_float2(r0_23, r1_23); + float2 s_23 = fma_f32x2_rn_noftz(r_23, fma_f32x2_rn_noftz(nd_23, r_23, make_float2(1.0f, 1.0f)), r_23); + float t0_23 = s_23.x; + float t1_23 = s_23.y; band = band | ((fminf(t0_23, t1_23) == 0.0f) ? 8 : 0); float2 s_22 = make_float2(max_noftz(t0_23, 0.0f), max_noftz(t1_23, 0.0f)); float2 act_22; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_22) : "l"(*(const unsigned long long*)&_cvt_f32_34), "l"(*(const unsigned long long*)&s_22)); - float2 _f32x2_mul_a_56 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_22, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_22), s_22), _cvt_f32_36); + float2 _f32x2_mul_a_28 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_22, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_22), s_22), _cvt_f32_36); float2 _mul_22; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_56), "l"(*(const unsigned long long*)&_cvt_f32_35)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_28), "l"(*(const unsigned long long*)&_cvt_f32_35)); float2 _mul_23; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_23) : "l"(*(const unsigned long long*)&act_22), "l"(*(const unsigned long long*)&_cvt_f32_36)); __nv_bfloat162 _bf16x2_22 = __float22bfloat162_rn(make_float2(_mul_22.x, _mul_22.y)); @@ -605,29 +581,27 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_37 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[4])); float2 _cvt_f32_38 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[4])); float2 _cvt_f32_39 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[4])); - float e0_25 = approx_exp2((-_cvt_f32_37.x) * 1.442695f); - float e1_25 = approx_exp2((-_cvt_f32_37.y) * 1.442695f); - float d0_25 = 1.0f + e0_25; - float d1_25 = 1.0f + e1_25; + float2 _f32x2_mul_b_29 = make_float2(-1.442695f, -1.442695f); + float2 t_25; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_25) : "l"(*(const unsigned long long*)&_cvt_f32_37), "l"(*(const unsigned long long*)&_f32x2_mul_b_29)); + float e0_25 = approx_exp2(t_25.x); + float e1_25 = approx_exp2(t_25.y); + float2 nd_25 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_25, e1_25)); + float d0_25 = -nd_25.x; + float d1_25 = -nd_25.y; float r0_25 = approx_rcp(d0_25); float r1_25 = approx_rcp(d1_25); - float _fma_57; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_57) : "f"(-d0_25), "f"(r0_25), "f"(1.0f)); - float _fma_8_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_8_1) : "f"(r0_25), "f"(_fma_57), "f"(r0_25)); - float _fma_58; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_58) : "f"(-d1_25), "f"(r1_25), "f"(1.0f)); - float _fma_9; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_9) : "f"(r1_25), "f"(_fma_58), "f"(r1_25)); - float t0_25 = _fma_8_1; - float t1_25 = _fma_9; + float2 r_25 = make_float2(r0_25, r1_25); + float2 s_25 = fma_f32x2_rn_noftz(r_25, fma_f32x2_rn_noftz(nd_25, r_25, make_float2(1.0f, 1.0f)), r_25); + float t0_25 = s_25.x; + float t1_25 = s_25.y; band = band | ((fminf(t0_25, t1_25) == 0.0f) ? 16 : 0); float2 s_24 = make_float2(max_noftz(t0_25, 0.0f), max_noftz(t1_25, 0.0f)); float2 act_24; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_24) : "l"(*(const unsigned long long*)&_cvt_f32_37), "l"(*(const unsigned long long*)&s_24)); - float2 _f32x2_mul_a_59 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_24, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_24), s_24), _cvt_f32_39); + float2 _f32x2_mul_a_30 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_24, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_24), s_24), _cvt_f32_39); float2 _mul_24; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_24) : "l"(*(const unsigned long long*)&_f32x2_mul_a_59), "l"(*(const unsigned long long*)&_cvt_f32_38)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_24) : "l"(*(const unsigned long long*)&_f32x2_mul_a_30), "l"(*(const unsigned long long*)&_cvt_f32_38)); float2 _mul_25; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_25) : "l"(*(const unsigned long long*)&act_24), "l"(*(const unsigned long long*)&_cvt_f32_39)); __nv_bfloat162 _bf16x2_24 = __float22bfloat162_rn(make_float2(_mul_24.x, _mul_24.y)); @@ -637,29 +611,27 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_40 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[5])); float2 _cvt_f32_41 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[5])); float2 _cvt_f32_42 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[5])); - float e0_27 = approx_exp2((-_cvt_f32_40.x) * 1.442695f); - float e1_27 = approx_exp2((-_cvt_f32_40.y) * 1.442695f); - float d0_27 = 1.0f + e0_27; - float d1_27 = 1.0f + e1_27; + float2 _f32x2_mul_b_31 = make_float2(-1.442695f, -1.442695f); + float2 t_27; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_27) : "l"(*(const unsigned long long*)&_cvt_f32_40), "l"(*(const unsigned long long*)&_f32x2_mul_b_31)); + float e0_27 = approx_exp2(t_27.x); + float e1_27 = approx_exp2(t_27.y); + float2 nd_27 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_27, e1_27)); + float d0_27 = -nd_27.x; + float d1_27 = -nd_27.y; float r0_27 = approx_rcp(d0_27); float r1_27 = approx_rcp(d1_27); - float _fma_60; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_60) : "f"(-d0_27), "f"(r0_27), "f"(1.0f)); - float _fma_10_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_10_1) : "f"(r0_27), "f"(_fma_60), "f"(r0_27)); - float _fma_61; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_61) : "f"(-d1_27), "f"(r1_27), "f"(1.0f)); - float _fma_11_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_11_1) : "f"(r1_27), "f"(_fma_61), "f"(r1_27)); - float t0_27 = _fma_10_1; - float t1_27 = _fma_11_1; + float2 r_27 = make_float2(r0_27, r1_27); + float2 s_27 = fma_f32x2_rn_noftz(r_27, fma_f32x2_rn_noftz(nd_27, r_27, make_float2(1.0f, 1.0f)), r_27); + float t0_27 = s_27.x; + float t1_27 = s_27.y; band = band | ((fminf(t0_27, t1_27) == 0.0f) ? 32 : 0); float2 s_26 = make_float2(max_noftz(t0_27, 0.0f), max_noftz(t1_27, 0.0f)); float2 act_26; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_26) : "l"(*(const unsigned long long*)&_cvt_f32_40), "l"(*(const unsigned long long*)&s_26)); - float2 _f32x2_mul_a_62 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_26, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_26), s_26), _cvt_f32_42); + float2 _f32x2_mul_a_32 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_26, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_26), s_26), _cvt_f32_42); float2 _mul_26; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_26) : "l"(*(const unsigned long long*)&_f32x2_mul_a_62), "l"(*(const unsigned long long*)&_cvt_f32_41)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_26) : "l"(*(const unsigned long long*)&_f32x2_mul_a_32), "l"(*(const unsigned long long*)&_cvt_f32_41)); float2 _mul_27; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_27) : "l"(*(const unsigned long long*)&act_26), "l"(*(const unsigned long long*)&_cvt_f32_42)); __nv_bfloat162 _bf16x2_26 = __float22bfloat162_rn(make_float2(_mul_26.x, _mul_26.y)); @@ -669,29 +641,27 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_43 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[6])); float2 _cvt_f32_44 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[6])); float2 _cvt_f32_45 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[6])); - float e0_29 = approx_exp2((-_cvt_f32_43.x) * 1.442695f); - float e1_29 = approx_exp2((-_cvt_f32_43.y) * 1.442695f); - float d0_29 = 1.0f + e0_29; - float d1_29 = 1.0f + e1_29; + float2 _f32x2_mul_b_33 = make_float2(-1.442695f, -1.442695f); + float2 t_29; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_29) : "l"(*(const unsigned long long*)&_cvt_f32_43), "l"(*(const unsigned long long*)&_f32x2_mul_b_33)); + float e0_29 = approx_exp2(t_29.x); + float e1_29 = approx_exp2(t_29.y); + float2 nd_29 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_29, e1_29)); + float d0_29 = -nd_29.x; + float d1_29 = -nd_29.y; float r0_29 = approx_rcp(d0_29); float r1_29 = approx_rcp(d1_29); - float _fma_63; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_63) : "f"(-d0_29), "f"(r0_29), "f"(1.0f)); - float _fma_12_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_12_1) : "f"(r0_29), "f"(_fma_63), "f"(r0_29)); - float _fma_64; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_64) : "f"(-d1_29), "f"(r1_29), "f"(1.0f)); - float _fma_13_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_13_1) : "f"(r1_29), "f"(_fma_64), "f"(r1_29)); - float t0_29 = _fma_12_1; - float t1_29 = _fma_13_1; + float2 r_29 = make_float2(r0_29, r1_29); + float2 s_29 = fma_f32x2_rn_noftz(r_29, fma_f32x2_rn_noftz(nd_29, r_29, make_float2(1.0f, 1.0f)), r_29); + float t0_29 = s_29.x; + float t1_29 = s_29.y; band = band | ((fminf(t0_29, t1_29) == 0.0f) ? 64 : 0); float2 s_28 = make_float2(max_noftz(t0_29, 0.0f), max_noftz(t1_29, 0.0f)); float2 act_28; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_28) : "l"(*(const unsigned long long*)&_cvt_f32_43), "l"(*(const unsigned long long*)&s_28)); - float2 _f32x2_mul_a_65 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_28, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_28), s_28), _cvt_f32_45); + float2 _f32x2_mul_a_34 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_28, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_28), s_28), _cvt_f32_45); float2 _mul_28; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_28) : "l"(*(const unsigned long long*)&_f32x2_mul_a_65), "l"(*(const unsigned long long*)&_cvt_f32_44)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_28) : "l"(*(const unsigned long long*)&_f32x2_mul_a_34), "l"(*(const unsigned long long*)&_cvt_f32_44)); float2 _mul_29; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_29) : "l"(*(const unsigned long long*)&act_28), "l"(*(const unsigned long long*)&_cvt_f32_45)); __nv_bfloat162 _bf16x2_28 = __float22bfloat162_rn(make_float2(_mul_28.x, _mul_28.y)); @@ -701,29 +671,27 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_46 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[7])); float2 _cvt_f32_47 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[7])); float2 _cvt_f32_48 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[7])); - float e0_31 = approx_exp2((-_cvt_f32_46.x) * 1.442695f); - float e1_31 = approx_exp2((-_cvt_f32_46.y) * 1.442695f); - float d0_31 = 1.0f + e0_31; - float d1_31 = 1.0f + e1_31; + float2 _f32x2_mul_b_35 = make_float2(-1.442695f, -1.442695f); + float2 t_31; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_31) : "l"(*(const unsigned long long*)&_cvt_f32_46), "l"(*(const unsigned long long*)&_f32x2_mul_b_35)); + float e0_31 = approx_exp2(t_31.x); + float e1_31 = approx_exp2(t_31.y); + float2 nd_31 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_31, e1_31)); + float d0_31 = -nd_31.x; + float d1_31 = -nd_31.y; float r0_31 = approx_rcp(d0_31); float r1_31 = approx_rcp(d1_31); - float _fma_66; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_66) : "f"(-d0_31), "f"(r0_31), "f"(1.0f)); - float _fma_14; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_14) : "f"(r0_31), "f"(_fma_66), "f"(r0_31)); - float _fma_67; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_67) : "f"(-d1_31), "f"(r1_31), "f"(1.0f)); - float _fma_15_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_15_1) : "f"(r1_31), "f"(_fma_67), "f"(r1_31)); - float t0_31 = _fma_14; - float t1_31 = _fma_15_1; + float2 r_31 = make_float2(r0_31, r1_31); + float2 s_31 = fma_f32x2_rn_noftz(r_31, fma_f32x2_rn_noftz(nd_31, r_31, make_float2(1.0f, 1.0f)), r_31); + float t0_31 = s_31.x; + float t1_31 = s_31.y; band = band | ((fminf(t0_31, t1_31) == 0.0f) ? 128 : 0); float2 s_30 = make_float2(max_noftz(t0_31, 0.0f), max_noftz(t1_31, 0.0f)); float2 act_30; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_30) : "l"(*(const unsigned long long*)&_cvt_f32_46), "l"(*(const unsigned long long*)&s_30)); - float2 _f32x2_mul_a_68 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_30, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_30), s_30), _cvt_f32_48); + float2 _f32x2_mul_a_36 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_30, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_30), s_30), _cvt_f32_48); float2 _mul_30; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_30) : "l"(*(const unsigned long long*)&_f32x2_mul_a_68), "l"(*(const unsigned long long*)&_cvt_f32_47)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_30) : "l"(*(const unsigned long long*)&_f32x2_mul_a_36), "l"(*(const unsigned long long*)&_cvt_f32_47)); float2 _mul_31; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_31) : "l"(*(const unsigned long long*)&act_30), "l"(*(const unsigned long long*)&_cvt_f32_48)); __nv_bfloat162 _bf16x2_30 = __float22bfloat162_rn(make_float2(_mul_30.x, _mul_30.y)); @@ -749,78 +717,76 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat float2 _cvt_f32_49 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[0])); float2 _cvt_f32_50 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[0])); float2 _cvt_f32_51 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[0])); - float e0_33 = approx_exp2((-_cvt_f32_49.x) * 1.442695f); - float e1_33 = approx_exp2((-_cvt_f32_49.y) * 1.442695f); - float d0_33 = 1.0f + e0_33; - float d1_33 = 1.0f + e1_33; + float2 _f32x2_mul_b_37 = make_float2(-1.442695f, -1.442695f); + float2 t_33; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_33) : "l"(*(const unsigned long long*)&_cvt_f32_49), "l"(*(const unsigned long long*)&_f32x2_mul_b_37)); + float e0_33 = approx_exp2(t_33.x); + float e1_33 = approx_exp2(t_33.y); + float2 nd_33 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_33, e1_33)); + float d0_33 = -nd_33.x; + float d1_33 = -nd_33.y; float r0_33 = approx_rcp(d0_33); float r1_33 = approx_rcp(d1_33); - float _fma_69; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_69) : "f"(-d0_33), "f"(r0_33), "f"(1.0f)); - float _fma_16_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_16_1) : "f"(r0_33), "f"(_fma_69), "f"(r0_33)); - float _fma_70; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_70) : "f"(-d1_33), "f"(r1_33), "f"(1.0f)); - float _fma_17_1; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_17_1) : "f"(r1_33), "f"(_fma_70), "f"(r1_33)); - float t0_33 = _fma_16_1; - float t1_33 = _fma_17_1; + float2 r_33 = make_float2(r0_33, r1_33); + float2 s_33 = fma_f32x2_rn_noftz(r_33, fma_f32x2_rn_noftz(nd_33, r_33, make_float2(1.0f, 1.0f)), r_33); + float t0_33 = s_33.x; + float t1_33 = s_33.y; float d4_33a; asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_33a) : "f"(d0_33), "f"(0.25f)); float r4_33a = approx_rcp(d4_33a); - float _fma_72; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_72) : "f"(-d4_33a), "f"(r4_33a), "f"(1.0f)); - float _fma_71; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_71) : "f"(r4_33a), "f"(_fma_72), "f"(r4_33a)); + float _fma_39; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_39) : "f"(-d4_33a), "f"(r4_33a), "f"(1.0f)); + float _fma_38; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_38) : "f"(r4_33a), "f"(_fma_39), "f"(r4_33a)); float big_33a; - asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_33a) : "f"(mul_f32_rn_noftz(_fma_71, 1.2676506e+30f)), "f"(1.4073749e+14f)); - float _add_73; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_73) : "f"(big_33a), "f"(8.388608e+06f)); + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_33a) : "f"(mul_f32_rn_noftz(_fma_38, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_40; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_40) : "f"(big_33a), "f"(8.388608e+06f)); float whole_33a; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_33a) : "f"(_add_73), "f"(8.388608e+06f)); + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_33a) : "f"(_add_40), "f"(8.388608e+06f)); float above_33a; asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_33a) : "f"(-d4_33a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_33a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); - float _sub_75; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_75) : "f"(big_33a), "f"(whole_33a)); - float _abs_74; - asm("abs.f32 %0, %1;" : "=f"(_abs_74) : "f"(_sub_75)); - float _add_76; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_76) : "f"(big_33a), "f"(0.5f)); - float _sub_77; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_77) : "f"(big_33a), "f"(0.5f)); - float rounded_33a = ((_abs_74 == 0.5f) ? ((above_33a > 0.0f) ? _add_76 : _sub_77) : whole_33a); + float _sub_42; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_42) : "f"(big_33a), "f"(whole_33a)); + float _abs_41; + asm("abs.f32 %0, %1;" : "=f"(_abs_41) : "f"(_sub_42)); + float _add_43; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_43) : "f"(big_33a), "f"(0.5f)); + float _sub_44; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_44) : "f"(big_33a), "f"(0.5f)); + float rounded_33a = ((_abs_41 == 0.5f) ? ((above_33a > 0.0f) ? _add_43 : _sub_44) : whole_33a); float u0_33 = ((d0_33 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_33a, 7.888609e-31f), 1.7763568e-15f) : t0_33); float d4_33b; asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_33b) : "f"(d1_33), "f"(0.25f)); float r4_33b = approx_rcp(d4_33b); - float _fma_79; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_79) : "f"(-d4_33b), "f"(r4_33b), "f"(1.0f)); - float _fma_78; - asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_78) : "f"(r4_33b), "f"(_fma_79), "f"(r4_33b)); + float _fma_46; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_46) : "f"(-d4_33b), "f"(r4_33b), "f"(1.0f)); + float _fma_45; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_45) : "f"(r4_33b), "f"(_fma_46), "f"(r4_33b)); float big_33b; - asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_33b) : "f"(mul_f32_rn_noftz(_fma_78, 1.2676506e+30f)), "f"(1.4073749e+14f)); - float _add_80; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_80) : "f"(big_33b), "f"(8.388608e+06f)); + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_33b) : "f"(mul_f32_rn_noftz(_fma_45, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_47; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_47) : "f"(big_33b), "f"(8.388608e+06f)); float whole_33b; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_33b) : "f"(_add_80), "f"(8.388608e+06f)); + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_33b) : "f"(_add_47), "f"(8.388608e+06f)); float above_33b; asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_33b) : "f"(-d4_33b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_33b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); - float _sub_82; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_82) : "f"(big_33b), "f"(whole_33b)); - float _abs_81; - asm("abs.f32 %0, %1;" : "=f"(_abs_81) : "f"(_sub_82)); - float _add_83; - asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_83) : "f"(big_33b), "f"(0.5f)); - float _sub_84; - asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_84) : "f"(big_33b), "f"(0.5f)); - float rounded_33b = ((_abs_81 == 0.5f) ? ((above_33b > 0.0f) ? _add_83 : _sub_84) : whole_33b); + float _sub_49; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_49) : "f"(big_33b), "f"(whole_33b)); + float _abs_48; + asm("abs.f32 %0, %1;" : "=f"(_abs_48) : "f"(_sub_49)); + float _add_50; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_50) : "f"(big_33b), "f"(0.5f)); + float _sub_51; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_51) : "f"(big_33b), "f"(0.5f)); + float rounded_33b = ((_abs_48 == 0.5f) ? ((above_33b > 0.0f) ? _add_50 : _sub_51) : whole_33b); float u1_33 = ((d1_33 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_33b, 7.888609e-31f), 1.7763568e-15f) : t1_33); float2 s_32 = make_float2(max_noftz(u0_33, 0.0f), max_noftz(u1_33, 0.0f)); float2 act_32; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_32) : "l"(*(const unsigned long long*)&_cvt_f32_49), "l"(*(const unsigned long long*)&s_32)); - float2 _f32x2_mul_a_85 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_32, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_32), s_32), _cvt_f32_51); + float2 _f32x2_mul_a_52 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_32, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_32), s_32), _cvt_f32_51); float2 _mul_32; - asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_32) : "l"(*(const unsigned long long*)&_f32x2_mul_a_85), "l"(*(const unsigned long long*)&_cvt_f32_50)); + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_32) : "l"(*(const unsigned long long*)&_f32x2_mul_a_52), "l"(*(const unsigned long long*)&_cvt_f32_50)); float2 _mul_33; asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_33) : "l"(*(const unsigned long long*)&act_32), "l"(*(const unsigned long long*)&_cvt_f32_51)); __nv_bfloat162 _bf16x2_32 = __float22bfloat162_rn(make_float2(_mul_32.x, _mul_32.y)); @@ -851,54 +817,54 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat int scale_col = bx * 2 + blk; int out_index = grow * (2 * K) + col; __nv_bfloat162 amax = __as_bf16x2(0); - __nv_bfloat162 _hxa_87 = amax; - __nv_bfloat162 _hxb_87 = __as_bf16x2(dgate[0]); - unsigned int _hxr_87; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_87) : "r"(*reinterpret_cast(&_hxa_87)), "r"(*reinterpret_cast(&_hxb_87))); - __nv_bfloat162 _max_86 = *reinterpret_cast(&_hxr_87); - amax = _max_86; - __nv_bfloat162 _hxa_89 = amax; - __nv_bfloat162 _hxb_89 = __as_bf16x2(dgate[1]); - unsigned int _hxr_89; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_89) : "r"(*reinterpret_cast(&_hxa_89)), "r"(*reinterpret_cast(&_hxb_89))); - __nv_bfloat162 _max_88 = *reinterpret_cast(&_hxr_89); - amax = _max_88; - __nv_bfloat162 _hxa_91 = amax; - __nv_bfloat162 _hxb_91 = __as_bf16x2(dgate[2]); - unsigned int _hxr_91; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_91) : "r"(*reinterpret_cast(&_hxa_91)), "r"(*reinterpret_cast(&_hxb_91))); - __nv_bfloat162 _max_90 = *reinterpret_cast(&_hxr_91); - amax = _max_90; - __nv_bfloat162 _hxa_93 = amax; - __nv_bfloat162 _hxb_93 = __as_bf16x2(dgate[3]); - unsigned int _hxr_93; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_93) : "r"(*reinterpret_cast(&_hxa_93)), "r"(*reinterpret_cast(&_hxb_93))); - __nv_bfloat162 _max_92 = *reinterpret_cast(&_hxr_93); - amax = _max_92; - __nv_bfloat162 _hxa_95 = amax; - __nv_bfloat162 _hxb_95 = __as_bf16x2(dgate[4]); - unsigned int _hxr_95; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_95) : "r"(*reinterpret_cast(&_hxa_95)), "r"(*reinterpret_cast(&_hxb_95))); - __nv_bfloat162 _max_94 = *reinterpret_cast(&_hxr_95); - amax = _max_94; - __nv_bfloat162 _hxa_97 = amax; - __nv_bfloat162 _hxb_97 = __as_bf16x2(dgate[5]); - unsigned int _hxr_97; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_97) : "r"(*reinterpret_cast(&_hxa_97)), "r"(*reinterpret_cast(&_hxb_97))); - __nv_bfloat162 _max_96 = *reinterpret_cast(&_hxr_97); - amax = _max_96; - __nv_bfloat162 _hxa_99 = amax; - __nv_bfloat162 _hxb_99 = __as_bf16x2(dgate[6]); - unsigned int _hxr_99; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_99) : "r"(*reinterpret_cast(&_hxa_99)), "r"(*reinterpret_cast(&_hxb_99))); - __nv_bfloat162 _max_98 = *reinterpret_cast(&_hxr_99); - amax = _max_98; - __nv_bfloat162 _hxa_101 = amax; - __nv_bfloat162 _hxb_101 = __as_bf16x2(dgate[7]); - unsigned int _hxr_101; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_101) : "r"(*reinterpret_cast(&_hxa_101)), "r"(*reinterpret_cast(&_hxb_101))); - __nv_bfloat162 _max_100 = *reinterpret_cast(&_hxr_101); - amax = _max_100; + __nv_bfloat162 _hxa_54 = amax; + __nv_bfloat162 _hxb_54 = __as_bf16x2(dgate[0]); + unsigned int _hxr_54; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_54) : "r"(*reinterpret_cast(&_hxa_54)), "r"(*reinterpret_cast(&_hxb_54))); + __nv_bfloat162 _max_53 = *reinterpret_cast(&_hxr_54); + amax = _max_53; + __nv_bfloat162 _hxa_56 = amax; + __nv_bfloat162 _hxb_56 = __as_bf16x2(dgate[1]); + unsigned int _hxr_56; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_56) : "r"(*reinterpret_cast(&_hxa_56)), "r"(*reinterpret_cast(&_hxb_56))); + __nv_bfloat162 _max_55 = *reinterpret_cast(&_hxr_56); + amax = _max_55; + __nv_bfloat162 _hxa_58 = amax; + __nv_bfloat162 _hxb_58 = __as_bf16x2(dgate[2]); + unsigned int _hxr_58; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_58) : "r"(*reinterpret_cast(&_hxa_58)), "r"(*reinterpret_cast(&_hxb_58))); + __nv_bfloat162 _max_57 = *reinterpret_cast(&_hxr_58); + amax = _max_57; + __nv_bfloat162 _hxa_60 = amax; + __nv_bfloat162 _hxb_60 = __as_bf16x2(dgate[3]); + unsigned int _hxr_60; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_60) : "r"(*reinterpret_cast(&_hxa_60)), "r"(*reinterpret_cast(&_hxb_60))); + __nv_bfloat162 _max_59 = *reinterpret_cast(&_hxr_60); + amax = _max_59; + __nv_bfloat162 _hxa_62 = amax; + __nv_bfloat162 _hxb_62 = __as_bf16x2(dgate[4]); + unsigned int _hxr_62; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_62) : "r"(*reinterpret_cast(&_hxa_62)), "r"(*reinterpret_cast(&_hxb_62))); + __nv_bfloat162 _max_61 = *reinterpret_cast(&_hxr_62); + amax = _max_61; + __nv_bfloat162 _hxa_64 = amax; + __nv_bfloat162 _hxb_64 = __as_bf16x2(dgate[5]); + unsigned int _hxr_64; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_64) : "r"(*reinterpret_cast(&_hxa_64)), "r"(*reinterpret_cast(&_hxb_64))); + __nv_bfloat162 _max_63 = *reinterpret_cast(&_hxr_64); + amax = _max_63; + __nv_bfloat162 _hxa_66 = amax; + __nv_bfloat162 _hxb_66 = __as_bf16x2(dgate[6]); + unsigned int _hxr_66; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_66) : "r"(*reinterpret_cast(&_hxa_66)), "r"(*reinterpret_cast(&_hxb_66))); + __nv_bfloat162 _max_65 = *reinterpret_cast(&_hxr_66); + amax = _max_65; + __nv_bfloat162 _hxa_68 = amax; + __nv_bfloat162 _hxb_68 = __as_bf16x2(dgate[7]); + unsigned int _hxr_68; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_68) : "r"(*reinterpret_cast(&_hxa_68)), "r"(*reinterpret_cast(&_hxb_68))); + __nv_bfloat162 _max_67 = *reinterpret_cast(&_hxr_68); + amax = _max_67; unsigned int mag = __as_u32(amax) & 2147450879; unsigned int peer = __shfl_xor_sync(0xFFFFFFFF, mag, 1); __nv_bfloat162 both = __hmax2_nan(__as_bf16x2(mag), __as_bf16x2(peer)); @@ -937,20 +903,20 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat #endif uint32_t _bf16x2_mul_1; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_1) : "r"(dgate[1]), "r"(inv2)); - uint16_t _cvt_satfinite_e4m3x2_bf16x2_103; + uint16_t _cvt_satfinite_e4m3x2_bf16x2_70; #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) - asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_103) : "r"(_bf16x2_mul_1)); + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_70) : "r"(_bf16x2_mul_1)); #else { const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_1); const float _lo = __uint_as_float(_bf16x2 << 16); const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); - asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_103) : "f"(_hi), "f"(_lo)); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_70) : "f"(_hi), "f"(_lo)); } #endif - uint32_t _mov_b32_pack_u16x2_102; - asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_102) : "h"(_e4m3x2_bf16x2_0), "h"(_cvt_satfinite_e4m3x2_bf16x2_103)); - words[0] = _mov_b32_pack_u16x2_102; + uint32_t _mov_b32_pack_u16x2_69; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_69) : "h"(_e4m3x2_bf16x2_0), "h"(_cvt_satfinite_e4m3x2_bf16x2_70)); + words[0] = _mov_b32_pack_u16x2_69; uint32_t _bf16x2_mul_2; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_2) : "r"(dgate[2]), "r"(inv2)); uint16_t _e4m3x2_bf16x2_1; @@ -966,20 +932,20 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat #endif uint32_t _bf16x2_mul_3; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_3) : "r"(dgate[3]), "r"(inv2)); - uint16_t _cvt_satfinite_e4m3x2_bf16x2_105; + uint16_t _cvt_satfinite_e4m3x2_bf16x2_72; #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) - asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_105) : "r"(_bf16x2_mul_3)); + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_72) : "r"(_bf16x2_mul_3)); #else { const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_3); const float _lo = __uint_as_float(_bf16x2 << 16); const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); - asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_105) : "f"(_hi), "f"(_lo)); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_72) : "f"(_hi), "f"(_lo)); } #endif - uint32_t _mov_b32_pack_u16x2_104; - asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_104) : "h"(_e4m3x2_bf16x2_1), "h"(_cvt_satfinite_e4m3x2_bf16x2_105)); - words[1] = _mov_b32_pack_u16x2_104; + uint32_t _mov_b32_pack_u16x2_71; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_71) : "h"(_e4m3x2_bf16x2_1), "h"(_cvt_satfinite_e4m3x2_bf16x2_72)); + words[1] = _mov_b32_pack_u16x2_71; uint32_t _bf16x2_mul_4; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_4) : "r"(dgate[4]), "r"(inv2)); uint16_t _e4m3x2_bf16x2_2; @@ -995,20 +961,20 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat #endif uint32_t _bf16x2_mul_5; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_5) : "r"(dgate[5]), "r"(inv2)); - uint16_t _cvt_satfinite_e4m3x2_bf16x2_107; + uint16_t _cvt_satfinite_e4m3x2_bf16x2_74; #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) - asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_107) : "r"(_bf16x2_mul_5)); + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_74) : "r"(_bf16x2_mul_5)); #else { const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_5); const float _lo = __uint_as_float(_bf16x2 << 16); const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); - asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_107) : "f"(_hi), "f"(_lo)); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_74) : "f"(_hi), "f"(_lo)); } #endif - uint32_t _mov_b32_pack_u16x2_106; - asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_106) : "h"(_e4m3x2_bf16x2_2), "h"(_cvt_satfinite_e4m3x2_bf16x2_107)); - words[2] = _mov_b32_pack_u16x2_106; + uint32_t _mov_b32_pack_u16x2_73; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_73) : "h"(_e4m3x2_bf16x2_2), "h"(_cvt_satfinite_e4m3x2_bf16x2_74)); + words[2] = _mov_b32_pack_u16x2_73; uint32_t _bf16x2_mul_6; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_6) : "r"(dgate[6]), "r"(inv2)); uint16_t _e4m3x2_bf16x2_3; @@ -1024,70 +990,70 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat #endif uint32_t _bf16x2_mul_7; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_7) : "r"(dgate[7]), "r"(inv2)); - uint16_t _cvt_satfinite_e4m3x2_bf16x2_109; + uint16_t _cvt_satfinite_e4m3x2_bf16x2_76; #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) - asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_109) : "r"(_bf16x2_mul_7)); + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_76) : "r"(_bf16x2_mul_7)); #else { const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_7); const float _lo = __uint_as_float(_bf16x2 << 16); const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); - asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_109) : "f"(_hi), "f"(_lo)); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_76) : "f"(_hi), "f"(_lo)); } #endif - uint32_t _mov_b32_pack_u16x2_108; - asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_108) : "h"(_e4m3x2_bf16x2_3), "h"(_cvt_satfinite_e4m3x2_bf16x2_109)); - words[3] = _mov_b32_pack_u16x2_108; + uint32_t _mov_b32_pack_u16x2_75; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_75) : "h"(_e4m3x2_bf16x2_3), "h"(_cvt_satfinite_e4m3x2_bf16x2_76)); + words[3] = _mov_b32_pack_u16x2_75; reinterpret_cast(row_output + out_index)[0] = reinterpret_cast(words + 0)[0]; __nv_bfloat162 amax_0 = __as_bf16x2(0); - __nv_bfloat162 _hxa_111 = amax_0; - __nv_bfloat162 _hxb_111 = __as_bf16x2(dup[0]); - unsigned int _hxr_111; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_111) : "r"(*reinterpret_cast(&_hxa_111)), "r"(*reinterpret_cast(&_hxb_111))); - __nv_bfloat162 _max_110 = *reinterpret_cast(&_hxr_111); - amax_0 = _max_110; - __nv_bfloat162 _hxa_113 = amax_0; - __nv_bfloat162 _hxb_113 = __as_bf16x2(dup[1]); - unsigned int _hxr_113; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_113) : "r"(*reinterpret_cast(&_hxa_113)), "r"(*reinterpret_cast(&_hxb_113))); - __nv_bfloat162 _max_112 = *reinterpret_cast(&_hxr_113); - amax_0 = _max_112; - __nv_bfloat162 _hxa_115 = amax_0; - __nv_bfloat162 _hxb_115 = __as_bf16x2(dup[2]); - unsigned int _hxr_115; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_115) : "r"(*reinterpret_cast(&_hxa_115)), "r"(*reinterpret_cast(&_hxb_115))); - __nv_bfloat162 _max_114 = *reinterpret_cast(&_hxr_115); - amax_0 = _max_114; - __nv_bfloat162 _hxa_117 = amax_0; - __nv_bfloat162 _hxb_117 = __as_bf16x2(dup[3]); - unsigned int _hxr_117; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_117) : "r"(*reinterpret_cast(&_hxa_117)), "r"(*reinterpret_cast(&_hxb_117))); - __nv_bfloat162 _max_116 = *reinterpret_cast(&_hxr_117); - amax_0 = _max_116; - __nv_bfloat162 _hxa_119 = amax_0; - __nv_bfloat162 _hxb_119 = __as_bf16x2(dup[4]); - unsigned int _hxr_119; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_119) : "r"(*reinterpret_cast(&_hxa_119)), "r"(*reinterpret_cast(&_hxb_119))); - __nv_bfloat162 _max_118 = *reinterpret_cast(&_hxr_119); - amax_0 = _max_118; - __nv_bfloat162 _hxa_121 = amax_0; - __nv_bfloat162 _hxb_121 = __as_bf16x2(dup[5]); - unsigned int _hxr_121; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_121) : "r"(*reinterpret_cast(&_hxa_121)), "r"(*reinterpret_cast(&_hxb_121))); - __nv_bfloat162 _max_120 = *reinterpret_cast(&_hxr_121); - amax_0 = _max_120; - __nv_bfloat162 _hxa_123 = amax_0; - __nv_bfloat162 _hxb_123 = __as_bf16x2(dup[6]); - unsigned int _hxr_123; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_123) : "r"(*reinterpret_cast(&_hxa_123)), "r"(*reinterpret_cast(&_hxb_123))); - __nv_bfloat162 _max_122 = *reinterpret_cast(&_hxr_123); - amax_0 = _max_122; - __nv_bfloat162 _hxa_125 = amax_0; - __nv_bfloat162 _hxb_125 = __as_bf16x2(dup[7]); - unsigned int _hxr_125; - asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_125) : "r"(*reinterpret_cast(&_hxa_125)), "r"(*reinterpret_cast(&_hxb_125))); - __nv_bfloat162 _max_124 = *reinterpret_cast(&_hxr_125); - amax_0 = _max_124; + __nv_bfloat162 _hxa_78 = amax_0; + __nv_bfloat162 _hxb_78 = __as_bf16x2(dup[0]); + unsigned int _hxr_78; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_78) : "r"(*reinterpret_cast(&_hxa_78)), "r"(*reinterpret_cast(&_hxb_78))); + __nv_bfloat162 _max_77 = *reinterpret_cast(&_hxr_78); + amax_0 = _max_77; + __nv_bfloat162 _hxa_80 = amax_0; + __nv_bfloat162 _hxb_80 = __as_bf16x2(dup[1]); + unsigned int _hxr_80; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_80) : "r"(*reinterpret_cast(&_hxa_80)), "r"(*reinterpret_cast(&_hxb_80))); + __nv_bfloat162 _max_79 = *reinterpret_cast(&_hxr_80); + amax_0 = _max_79; + __nv_bfloat162 _hxa_82 = amax_0; + __nv_bfloat162 _hxb_82 = __as_bf16x2(dup[2]); + unsigned int _hxr_82; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_82) : "r"(*reinterpret_cast(&_hxa_82)), "r"(*reinterpret_cast(&_hxb_82))); + __nv_bfloat162 _max_81 = *reinterpret_cast(&_hxr_82); + amax_0 = _max_81; + __nv_bfloat162 _hxa_84 = amax_0; + __nv_bfloat162 _hxb_84 = __as_bf16x2(dup[3]); + unsigned int _hxr_84; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_84) : "r"(*reinterpret_cast(&_hxa_84)), "r"(*reinterpret_cast(&_hxb_84))); + __nv_bfloat162 _max_83 = *reinterpret_cast(&_hxr_84); + amax_0 = _max_83; + __nv_bfloat162 _hxa_86 = amax_0; + __nv_bfloat162 _hxb_86 = __as_bf16x2(dup[4]); + unsigned int _hxr_86; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_86) : "r"(*reinterpret_cast(&_hxa_86)), "r"(*reinterpret_cast(&_hxb_86))); + __nv_bfloat162 _max_85 = *reinterpret_cast(&_hxr_86); + amax_0 = _max_85; + __nv_bfloat162 _hxa_88 = amax_0; + __nv_bfloat162 _hxb_88 = __as_bf16x2(dup[5]); + unsigned int _hxr_88; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_88) : "r"(*reinterpret_cast(&_hxa_88)), "r"(*reinterpret_cast(&_hxb_88))); + __nv_bfloat162 _max_87 = *reinterpret_cast(&_hxr_88); + amax_0 = _max_87; + __nv_bfloat162 _hxa_90 = amax_0; + __nv_bfloat162 _hxb_90 = __as_bf16x2(dup[6]); + unsigned int _hxr_90; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_90) : "r"(*reinterpret_cast(&_hxa_90)), "r"(*reinterpret_cast(&_hxb_90))); + __nv_bfloat162 _max_89 = *reinterpret_cast(&_hxr_90); + amax_0 = _max_89; + __nv_bfloat162 _hxa_92 = amax_0; + __nv_bfloat162 _hxb_92 = __as_bf16x2(dup[7]); + unsigned int _hxr_92; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_92) : "r"(*reinterpret_cast(&_hxa_92)), "r"(*reinterpret_cast(&_hxb_92))); + __nv_bfloat162 _max_91 = *reinterpret_cast(&_hxr_92); + amax_0 = _max_91; unsigned int mag_1 = __as_u32(amax_0) & 2147450879; unsigned int peer_2 = __shfl_xor_sync(0xFFFFFFFF, mag_1, 1); __nv_bfloat162 both_3 = __hmax2_nan(__as_bf16x2(mag_1), __as_bf16x2(peer_2)); @@ -1126,20 +1092,20 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat #endif uint32_t _bf16x2_mul_9; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_9) : "r"(dup[1]), "r"(inv2_9)); - uint16_t _cvt_satfinite_e4m3x2_bf16x2_127; + uint16_t _cvt_satfinite_e4m3x2_bf16x2_94; #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) - asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_127) : "r"(_bf16x2_mul_9)); + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_94) : "r"(_bf16x2_mul_9)); #else { const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_9); const float _lo = __uint_as_float(_bf16x2 << 16); const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); - asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_127) : "f"(_hi), "f"(_lo)); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_94) : "f"(_hi), "f"(_lo)); } #endif - uint32_t _mov_b32_pack_u16x2_126; - asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_126) : "h"(_e4m3x2_bf16x2_4), "h"(_cvt_satfinite_e4m3x2_bf16x2_127)); - words_11[0] = _mov_b32_pack_u16x2_126; + uint32_t _mov_b32_pack_u16x2_93; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_93) : "h"(_e4m3x2_bf16x2_4), "h"(_cvt_satfinite_e4m3x2_bf16x2_94)); + words_11[0] = _mov_b32_pack_u16x2_93; uint32_t _bf16x2_mul_10; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_10) : "r"(dup[2]), "r"(inv2_9)); uint16_t _e4m3x2_bf16x2_5; @@ -1155,20 +1121,20 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat #endif uint32_t _bf16x2_mul_11; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_11) : "r"(dup[3]), "r"(inv2_9)); - uint16_t _cvt_satfinite_e4m3x2_bf16x2_129; + uint16_t _cvt_satfinite_e4m3x2_bf16x2_96; #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) - asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_129) : "r"(_bf16x2_mul_11)); + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_96) : "r"(_bf16x2_mul_11)); #else { const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_11); const float _lo = __uint_as_float(_bf16x2 << 16); const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); - asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_129) : "f"(_hi), "f"(_lo)); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_96) : "f"(_hi), "f"(_lo)); } #endif - uint32_t _mov_b32_pack_u16x2_128; - asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_128) : "h"(_e4m3x2_bf16x2_5), "h"(_cvt_satfinite_e4m3x2_bf16x2_129)); - words_11[1] = _mov_b32_pack_u16x2_128; + uint32_t _mov_b32_pack_u16x2_95; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_95) : "h"(_e4m3x2_bf16x2_5), "h"(_cvt_satfinite_e4m3x2_bf16x2_96)); + words_11[1] = _mov_b32_pack_u16x2_95; uint32_t _bf16x2_mul_12; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_12) : "r"(dup[4]), "r"(inv2_9)); uint16_t _e4m3x2_bf16x2_6; @@ -1184,20 +1150,20 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat #endif uint32_t _bf16x2_mul_13; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_13) : "r"(dup[5]), "r"(inv2_9)); - uint16_t _cvt_satfinite_e4m3x2_bf16x2_131; + uint16_t _cvt_satfinite_e4m3x2_bf16x2_98; #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) - asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_131) : "r"(_bf16x2_mul_13)); + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_98) : "r"(_bf16x2_mul_13)); #else { const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_13); const float _lo = __uint_as_float(_bf16x2 << 16); const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); - asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_131) : "f"(_hi), "f"(_lo)); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_98) : "f"(_hi), "f"(_lo)); } #endif - uint32_t _mov_b32_pack_u16x2_130; - asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_130) : "h"(_e4m3x2_bf16x2_6), "h"(_cvt_satfinite_e4m3x2_bf16x2_131)); - words_11[2] = _mov_b32_pack_u16x2_130; + uint32_t _mov_b32_pack_u16x2_97; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_97) : "h"(_e4m3x2_bf16x2_6), "h"(_cvt_satfinite_e4m3x2_bf16x2_98)); + words_11[2] = _mov_b32_pack_u16x2_97; uint32_t _bf16x2_mul_14; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_14) : "r"(dup[6]), "r"(inv2_9)); uint16_t _e4m3x2_bf16x2_7; @@ -1213,20 +1179,20 @@ kernel_cake_gated_act_mxfp8_077a568b810590574917(__nv_bfloat16* __restrict__ gat #endif uint32_t _bf16x2_mul_15; asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_15) : "r"(dup[7]), "r"(inv2_9)); - uint16_t _cvt_satfinite_e4m3x2_bf16x2_133; + uint16_t _cvt_satfinite_e4m3x2_bf16x2_100; #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) - asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_133) : "r"(_bf16x2_mul_15)); + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_100) : "r"(_bf16x2_mul_15)); #else { const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_15); const float _lo = __uint_as_float(_bf16x2 << 16); const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); - asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_133) : "f"(_hi), "f"(_lo)); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_100) : "f"(_hi), "f"(_lo)); } #endif - uint32_t _mov_b32_pack_u16x2_132; - asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_132) : "h"(_e4m3x2_bf16x2_7), "h"(_cvt_satfinite_e4m3x2_bf16x2_133)); - words_11[3] = _mov_b32_pack_u16x2_132; + uint32_t _mov_b32_pack_u16x2_99; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_99) : "h"(_e4m3x2_bf16x2_7), "h"(_cvt_satfinite_e4m3x2_bf16x2_100)); + words_11[3] = _mov_b32_pack_u16x2_99; reinterpret_cast(row_output + (out_index + K))[0] = reinterpret_cast(words_11 + 0)[0]; } diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_binding.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_binding.cu new file mode 100644 index 00000000000..65caba36c60 --- /dev/null +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_binding.cu @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2026 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gated_act_mxfp8_44395eef5de97bba1695, for the device source that defines it. Do not edit. +#include +#include + +#include "cake_ffi_checks.h" + +#include + +extern "C" __global__ void kernel_cake_gated_act_mxfp8_44395eef5de97bba1695(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K); + +namespace cake_binding_kernel_cake_gated_act_mxfp8_44395eef5de97bba1695 { + +using tvm::ffi::TensorView; + +constexpr const char* KERNEL_NAME = "kernel_cake_gated_act_mxfp8_44395eef5de97bba1695"; + +void CheckCuda(cudaError_t status, const char* what) { + TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) + << KERNEL_NAME << ": " << what << " failed: " << cudaGetErrorString(status); +} + +void* DataPtr(const TensorView& t) { return static_cast(t.data_ptr()) + t.byte_offset(); } + +void Run(TensorView a_gated_input, TensorView a_grad, TensorView a_row_output, TensorView a_col_output, TensorView a_row_scales, TensorView a_col_scales, int64_t a_M, int64_t a_K, int64_t grid_x, int64_t grid_y, int64_t grid_z) { + DLDevice dev = a_gated_input.device(); + tvm::ffi::CUDADeviceGuard device_guard(dev.device_id); + check_cuda_tensor(a_gated_input, "gated_input"); + check_dtype(a_gated_input, DLDataType{kDLBfloat, 16, 1}, "gated_input"); + check_cuda_tensor(a_grad, "grad"); + check_dtype(a_grad, DLDataType{kDLBfloat, 16, 1}, "grad"); + check_cuda_tensor(a_row_output, "row_output"); + check_dtype(a_row_output, DLDataType{kDLUInt, 8, 1}, "row_output"); + check_cuda_tensor(a_col_output, "col_output"); + check_dtype(a_col_output, DLDataType{kDLUInt, 8, 1}, "col_output"); + check_cuda_tensor(a_row_scales, "row_scales"); + check_dtype(a_row_scales, DLDataType{kDLUInt, 8, 1}, "row_scales"); + check_cuda_tensor(a_col_scales, "col_scales"); + check_dtype(a_col_scales, DLDataType{kDLUInt, 8, 1}, "col_scales"); + TVM_FFI_CHECK(a_M >= -2147483648LL && a_M <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'M' = " << a_M + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_K >= -2147483648LL && a_K <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'K' = " << a_K + << " is outside the i32 range [-2147483648, 2147483647]"; + check_contiguous(a_gated_input, "gated_input"); + check_contiguous(a_grad, "grad"); + check_same_device(a_grad, a_gated_input, "grad", "gated_input"); + check_contiguous(a_row_output, "row_output"); + check_same_device(a_row_output, a_gated_input, "row_output", "gated_input"); + check_contiguous(a_col_output, "col_output"); + check_same_device(a_col_output, a_gated_input, "col_output", "gated_input"); + check_contiguous(a_row_scales, "row_scales"); + check_same_device(a_row_scales, a_gated_input, "row_scales", "gated_input"); + check_contiguous(a_col_scales, "col_scales"); + check_same_device(a_col_scales, a_gated_input, "col_scales", "gated_input"); + TVM_FFI_CHECK(grid_x > 0 && grid_y > 0 && grid_z > 0, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") must be positive"; + TVM_FFI_CHECK(grid_x <= 2147483647 && grid_y <= 65535 && grid_z <= 65535, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") exceeds CUDA's limits"; + cudaStream_t stream = static_cast(TVMFFIEnvGetStream(dev.device_type, dev.device_id)); + void* v_gated_input = DataPtr(a_gated_input); + void* v_grad = DataPtr(a_grad); + void* v_row_output = DataPtr(a_row_output); + void* v_col_output = DataPtr(a_col_output); + void* v_row_scales = DataPtr(a_row_scales); + void* v_col_scales = DataPtr(a_col_scales); + int32_t v_M = static_cast(a_M); + int32_t v_K = static_cast(a_K); + void* kernel_args[] = {&v_gated_input, &v_grad, &v_row_output, &v_col_output, &v_row_scales, &v_col_scales, &v_M, &v_K}; + cudaLaunchConfig_t config{}; + config.gridDim = dim3(static_cast(grid_x), static_cast(grid_y), static_cast(grid_z)); + config.blockDim = dim3(128, 1, 1); + config.dynamicSmemBytes = 16384; + config.stream = stream; + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gated_act_mxfp8_44395eef5de97bba1695), kernel_args), + "cudaLaunchKernelExC"); +} + +} // namespace cake_binding_kernel_cake_gated_act_mxfp8_44395eef5de97bba1695 + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gated_act_mxfp8_44395eef5de97bba1695::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_kernel.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_kernel.cu new file mode 100644 index 00000000000..891f8368889 --- /dev/null +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_44395eef5de97bba1695_kernel.cu @@ -0,0 +1,2989 @@ +/* + * Copyright (c) 2026 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +#if defined(__CUDACC_RTC__) +typedef unsigned long long uint64_t; +#else +typedef unsigned long uint64_t; +#endif +static_assert(sizeof(uint64_t) == 8, "Cake requires an LP64 CUDA host ABI"); +typedef signed int int32_t; +typedef short int int16_t; +struct __align__(64) CakeTensorMap64 { uint64_t opaque[16]; }; +static_assert(sizeof(CakeTensorMap64) == 128, "64-aligned tensor-map ABI size"); +static_assert(alignof(CakeTensorMap64) == 64, "64-aligned tensor-map ABI alignment"); + +#if defined(__CUDACC_RTC__) +typedef struct __align__(128) { uint64_t opaque[16]; } CUtensorMap; +#else +#include +#endif + +static_assert(sizeof(CUtensorMap) == 128, "CUtensorMap CUDA ABI must be 128 bytes"); +#include +#include + +__device__ __forceinline__ int make_warp_uniform(int x) { + int result; + asm volatile("shfl.sync.idx.b32 %0, %1, 0, 0x1F, 0xFFFFFFFF;" + : "=r"(result) : "r"(x)); + return result; +} + +#define CAKE_INF CUDART_INF_F +#define NUM_MAIN_STAGES 1 +#define SMEM_TILES_OFF 0 +#define SMEM_TILES_STAGE_BYTES 16384 +#define SMEM_TILES_STRIDE 16384 +#define SMEM_TOTAL 16384 +#define THREADS 128 +#define LAUNCH_MIN_BLOCKS 7 + +#include + +__device__ __forceinline__ float approx_exp2(float x) { + float y; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + +__device__ __forceinline__ float max_noftz(float a, float b) { + float c; + asm("max.f32 %0, %1, %2;" : "=f"(c) : "f"(a), "f"(b)); + return c; +} + + +__device__ __forceinline__ float2 sub_f32x2_noftz(float2 a, float2 b) { + float2 r; + asm("sub.f32x2 %0, %1, %2;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(unsigned long long*)&a), "l"(*(unsigned long long*)&b)); + return r; +} + +__device__ __forceinline__ float2 mul_f32x2_noftz(float2 a, float2 b) { + float2 r; + asm("mul.rn.f32x2 %0, %1, %2;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(unsigned long long*)&a), "l"(*(unsigned long long*)&b)); + return r; +} + + +// ex2_emulation_f32x2 defined in softmax_frag_exp2_cast helper (or standalone) + +__device__ __forceinline__ float2 fma_f32x2_rn_noftz(float2 a, float2 b, float2 c) { + float2 r; + asm("fma.rn.f32x2 %0, %1, %2, %3;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(const unsigned long long*)&a), + "l"(*(const unsigned long long*)&b), + "l"(*(const unsigned long long*)&c)); + return r; +} + + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + +__device__ __forceinline__ __nv_bfloat162 __as_bf16x2(unsigned int v) { + __nv_bfloat162_raw raw; + raw.x = static_cast(v); + raw.y = static_cast(v >> 16); + return __nv_bfloat162(raw); +} + +__device__ __forceinline__ float2 cake_bf16x2_to_f32x2(__nv_bfloat162 v) { + const unsigned int w = *reinterpret_cast(&v); + return make_float2(__uint_as_float(w << 16), __uint_as_float(w & 0xffff0000u)); +} + +__device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { + float r; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} + +__device__ __noinline__ unsigned long long band_pair(__nv_bfloat16* src, __nv_bfloat16* dsrc, int index, int grad_index, int k) +{ + unsigned int _vec_load_0[1]; + { + _vec_load_0[0] = *reinterpret_cast(src + index + 0); + } + unsigned int _vec_load_1[1]; + { + _vec_load_1[0] = *reinterpret_cast(src + (index + k) + 0); + } + unsigned int _vec_load_2[1]; + { + _vec_load_2[0] = *reinterpret_cast(dsrc + grad_index + 0); + } + float2 _cvt_f32_0 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[0])); + float2 _cvt_f32_1 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[0])); + float2 _cvt_f32_2 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[0])); + float2 _f32x2_mul_b_0 = make_float2(-1.442695f, -1.442695f); + float2 t_1; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_1) : "l"(*(const unsigned long long*)&_cvt_f32_0), "l"(*(const unsigned long long*)&_f32x2_mul_b_0)); + float e0_1 = approx_exp2(t_1.x); + float e1_1 = approx_exp2(t_1.y); + float2 nd_1 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_1, e1_1)); + float d0_1 = -nd_1.x; + float d1_1 = -nd_1.y; + float r0_1 = approx_rcp(d0_1); + float r1_1 = approx_rcp(d1_1); + float2 r_1 = make_float2(r0_1, r1_1); + float2 s_1 = fma_f32x2_rn_noftz(r_1, fma_f32x2_rn_noftz(nd_1, r_1, make_float2(1.0f, 1.0f)), r_1); + float t0_1 = s_1.x; + float t1_1 = s_1.y; + float d4_1a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_1a) : "f"(d0_1), "f"(0.25f)); + float r4_1a = approx_rcp(d4_1a); + float _fma_2; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_2) : "f"(-d4_1a), "f"(r4_1a), "f"(1.0f)); + float _fma_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_1) : "f"(r4_1a), "f"(_fma_2), "f"(r4_1a)); + float big_1a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_1a) : "f"(mul_f32_rn_noftz(_fma_1, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_3; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_3) : "f"(big_1a), "f"(8.388608e+06f)); + float whole_1a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_1a) : "f"(_add_3), "f"(8.388608e+06f)); + float above_1a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_1a) : "f"(-d4_1a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_1a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_5; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_5) : "f"(big_1a), "f"(whole_1a)); + float _abs_4; + asm("abs.f32 %0, %1;" : "=f"(_abs_4) : "f"(_sub_5)); + float _add_6; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_6) : "f"(big_1a), "f"(0.5f)); + float _sub_7; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_7) : "f"(big_1a), "f"(0.5f)); + float rounded_1a = ((_abs_4 == 0.5f) ? ((above_1a > 0.0f) ? _add_6 : _sub_7) : whole_1a); + float u0_1 = ((d0_1 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_1a, 7.888609e-31f), 1.7763568e-15f) : t0_1); + float d4_1b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_1b) : "f"(d1_1), "f"(0.25f)); + float r4_1b = approx_rcp(d4_1b); + float _fma_9; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_9) : "f"(-d4_1b), "f"(r4_1b), "f"(1.0f)); + float _fma_8; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_8) : "f"(r4_1b), "f"(_fma_9), "f"(r4_1b)); + float big_1b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_1b) : "f"(mul_f32_rn_noftz(_fma_8, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_10; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_10) : "f"(big_1b), "f"(8.388608e+06f)); + float whole_1b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_1b) : "f"(_add_10), "f"(8.388608e+06f)); + float above_1b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_1b) : "f"(-d4_1b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_1b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_12; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_12) : "f"(big_1b), "f"(whole_1b)); + float _abs_11; + asm("abs.f32 %0, %1;" : "=f"(_abs_11) : "f"(_sub_12)); + float _add_13; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_13) : "f"(big_1b), "f"(0.5f)); + float _sub_14; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_14) : "f"(big_1b), "f"(0.5f)); + float rounded_1b = ((_abs_11 == 0.5f) ? ((above_1b > 0.0f) ? _add_13 : _sub_14) : whole_1b); + float u1_1 = ((d1_1 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_1b, 7.888609e-31f), 1.7763568e-15f) : t1_1); + float2 s_0 = make_float2(max_noftz(u0_1, 0.0f), max_noftz(u1_1, 0.0f)); + float2 act_0; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_0) : "l"(*(const unsigned long long*)&_cvt_f32_0), "l"(*(const unsigned long long*)&s_0)); + float2 _f32x2_mul_a_15 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_0, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_0), s_0), _cvt_f32_2); + float2 _mul_0; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_15), "l"(*(const unsigned long long*)&_cvt_f32_1)); + float2 _mul_1; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_1) : "l"(*(const unsigned long long*)&act_0), "l"(*(const unsigned long long*)&_cvt_f32_2)); + unsigned long long band_pair_out = (unsigned long long)__as_u32(__float22bfloat162_rn(make_float2(_mul_0.x, _mul_0.y))) | (unsigned long long)__as_u32(__float22bfloat162_rn(make_float2(_mul_1.x, _mul_1.y))) << 32; + return band_pair_out; +} + +extern "C" { + +__global__ __launch_bounds__(128, LAUNCH_MIN_BLOCKS) void +kernel_cake_gated_act_mxfp8_44395eef5de97bba1695(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K) +{ + const int tid = threadIdx.x; + const int warp = make_warp_uniform(tid / 32); + const int lane = tid % 32; + + extern __shared__ __align__(1024) char smem_raw[]; + int smem; +#if __CUDA_ARCH__ == 1000 + asm volatile("{ .reg .u64 smem_ptr; cvta.to.shared.u64 smem_ptr, %1; cvt.u32.u64 %0, smem_ptr; }" : "=r"(smem) : "l"(smem_raw)); + smem = make_warp_uniform(smem); +#else + smem = (int)(unsigned long long)__cvta_generic_to_shared(smem_raw); +#endif + + const int bid = blockIdx.x; + const int num_bids = gridDim.x; + + const int cta_rank = 0; + + // Kernel setup ops + uint8_t* tiles = reinterpret_cast(smem_raw + 0); + const int tiles_addr = smem + 0; + + // === Task calls (declaration order) === + int half = tid & 1; + int blk = tid >> 1 & 1; + int row = tid >> 2; + int bx = blockIdx.x; + int by = blockIdx.y; + int grow = by * 64 + row; + int col = bx * 64 + blk * 32 + half * 16; + int gate_index = grow * (2 * K) + col; + unsigned int _vec_load_3[8]; + { + uint32_t _iv_0_0; + uint32_t _iv_0_1; + uint32_t _iv_0_2; + uint32_t _iv_0_3; + uint32_t _iv_0_4; + uint32_t _iv_0_5; + uint32_t _iv_0_6; + uint32_t _iv_0_7; + asm volatile("ld.global.L1::no_allocate.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(_iv_0_0), "=r"(_iv_0_1), "=r"(_iv_0_2), "=r"(_iv_0_3), "=r"(_iv_0_4), "=r"(_iv_0_5), "=r"(_iv_0_6), "=r"(_iv_0_7) : "l"((const void*)(gated_input + gate_index + (0))) : "memory"); + _vec_load_3[0 + 0] = (uint32_t)_iv_0_0; + _vec_load_3[0 + 1] = (uint32_t)_iv_0_1; + _vec_load_3[0 + 2] = (uint32_t)_iv_0_2; + _vec_load_3[0 + 3] = (uint32_t)_iv_0_3; + _vec_load_3[0 + 4] = (uint32_t)_iv_0_4; + _vec_load_3[0 + 5] = (uint32_t)_iv_0_5; + _vec_load_3[0 + 6] = (uint32_t)_iv_0_6; + _vec_load_3[0 + 7] = (uint32_t)_iv_0_7; + } + unsigned int _vec_load_4[8]; + { + uint32_t _iv_1_0; + uint32_t _iv_1_1; + uint32_t _iv_1_2; + uint32_t _iv_1_3; + uint32_t _iv_1_4; + uint32_t _iv_1_5; + uint32_t _iv_1_6; + uint32_t _iv_1_7; + asm volatile("ld.global.L1::no_allocate.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(_iv_1_0), "=r"(_iv_1_1), "=r"(_iv_1_2), "=r"(_iv_1_3), "=r"(_iv_1_4), "=r"(_iv_1_5), "=r"(_iv_1_6), "=r"(_iv_1_7) : "l"((const void*)(gated_input + (gate_index + K) + (0))) : "memory"); + _vec_load_4[0 + 0] = (uint32_t)_iv_1_0; + _vec_load_4[0 + 1] = (uint32_t)_iv_1_1; + _vec_load_4[0 + 2] = (uint32_t)_iv_1_2; + _vec_load_4[0 + 3] = (uint32_t)_iv_1_3; + _vec_load_4[0 + 4] = (uint32_t)_iv_1_4; + _vec_load_4[0 + 5] = (uint32_t)_iv_1_5; + _vec_load_4[0 + 6] = (uint32_t)_iv_1_6; + _vec_load_4[0 + 7] = (uint32_t)_iv_1_7; + } + unsigned int _vec_load_5[8]; + { + uint32_t _iv_2_0; + uint32_t _iv_2_1; + uint32_t _iv_2_2; + uint32_t _iv_2_3; + uint32_t _iv_2_4; + uint32_t _iv_2_5; + uint32_t _iv_2_6; + uint32_t _iv_2_7; + asm volatile("ld.global.L1::no_allocate.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(_iv_2_0), "=r"(_iv_2_1), "=r"(_iv_2_2), "=r"(_iv_2_3), "=r"(_iv_2_4), "=r"(_iv_2_5), "=r"(_iv_2_6), "=r"(_iv_2_7) : "l"((const void*)(grad + (grow * K + col) + (0))) : "memory"); + _vec_load_5[0 + 0] = (uint32_t)_iv_2_0; + _vec_load_5[0 + 1] = (uint32_t)_iv_2_1; + _vec_load_5[0 + 2] = (uint32_t)_iv_2_2; + _vec_load_5[0 + 3] = (uint32_t)_iv_2_3; + _vec_load_5[0 + 4] = (uint32_t)_iv_2_4; + _vec_load_5[0 + 5] = (uint32_t)_iv_2_5; + _vec_load_5[0 + 6] = (uint32_t)_iv_2_6; + _vec_load_5[0 + 7] = (uint32_t)_iv_2_7; + } + int grow_hi = grow + 32; + int gate_index_hi = grow_hi * (2 * K) + col; + unsigned int _vec_load_6[8]; + { + uint32_t _iv_3_0; + uint32_t _iv_3_1; + uint32_t _iv_3_2; + uint32_t _iv_3_3; + uint32_t _iv_3_4; + uint32_t _iv_3_5; + uint32_t _iv_3_6; + uint32_t _iv_3_7; + asm volatile("ld.global.L1::no_allocate.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(_iv_3_0), "=r"(_iv_3_1), "=r"(_iv_3_2), "=r"(_iv_3_3), "=r"(_iv_3_4), "=r"(_iv_3_5), "=r"(_iv_3_6), "=r"(_iv_3_7) : "l"((const void*)(gated_input + gate_index_hi + (0))) : "memory"); + _vec_load_6[0 + 0] = (uint32_t)_iv_3_0; + _vec_load_6[0 + 1] = (uint32_t)_iv_3_1; + _vec_load_6[0 + 2] = (uint32_t)_iv_3_2; + _vec_load_6[0 + 3] = (uint32_t)_iv_3_3; + _vec_load_6[0 + 4] = (uint32_t)_iv_3_4; + _vec_load_6[0 + 5] = (uint32_t)_iv_3_5; + _vec_load_6[0 + 6] = (uint32_t)_iv_3_6; + _vec_load_6[0 + 7] = (uint32_t)_iv_3_7; + } + unsigned int _vec_load_7[8]; + { + uint32_t _iv_4_0; + uint32_t _iv_4_1; + uint32_t _iv_4_2; + uint32_t _iv_4_3; + uint32_t _iv_4_4; + uint32_t _iv_4_5; + uint32_t _iv_4_6; + uint32_t _iv_4_7; + asm volatile("ld.global.L1::no_allocate.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(_iv_4_0), "=r"(_iv_4_1), "=r"(_iv_4_2), "=r"(_iv_4_3), "=r"(_iv_4_4), "=r"(_iv_4_5), "=r"(_iv_4_6), "=r"(_iv_4_7) : "l"((const void*)(gated_input + (gate_index_hi + K) + (0))) : "memory"); + _vec_load_7[0 + 0] = (uint32_t)_iv_4_0; + _vec_load_7[0 + 1] = (uint32_t)_iv_4_1; + _vec_load_7[0 + 2] = (uint32_t)_iv_4_2; + _vec_load_7[0 + 3] = (uint32_t)_iv_4_3; + _vec_load_7[0 + 4] = (uint32_t)_iv_4_4; + _vec_load_7[0 + 5] = (uint32_t)_iv_4_5; + _vec_load_7[0 + 6] = (uint32_t)_iv_4_6; + _vec_load_7[0 + 7] = (uint32_t)_iv_4_7; + } + unsigned int _vec_load_8[8]; + { + uint32_t _iv_5_0; + uint32_t _iv_5_1; + uint32_t _iv_5_2; + uint32_t _iv_5_3; + uint32_t _iv_5_4; + uint32_t _iv_5_5; + uint32_t _iv_5_6; + uint32_t _iv_5_7; + asm volatile("ld.global.L1::no_allocate.v8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(_iv_5_0), "=r"(_iv_5_1), "=r"(_iv_5_2), "=r"(_iv_5_3), "=r"(_iv_5_4), "=r"(_iv_5_5), "=r"(_iv_5_6), "=r"(_iv_5_7) : "l"((const void*)(grad + (grow_hi * K + col) + (0))) : "memory"); + _vec_load_8[0 + 0] = (uint32_t)_iv_5_0; + _vec_load_8[0 + 1] = (uint32_t)_iv_5_1; + _vec_load_8[0 + 2] = (uint32_t)_iv_5_2; + _vec_load_8[0 + 3] = (uint32_t)_iv_5_3; + _vec_load_8[0 + 4] = (uint32_t)_iv_5_4; + _vec_load_8[0 + 5] = (uint32_t)_iv_5_5; + _vec_load_8[0 + 6] = (uint32_t)_iv_5_6; + _vec_load_8[0 + 7] = (uint32_t)_iv_5_7; + } + unsigned int dgate[8]; + unsigned int dup[8]; + unsigned int dgate_hi[8]; + unsigned int dup_hi[8]; + __nv_bfloat162 lowest = __as_bf16x2(_vec_load_3[0]); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_3[1])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_3[2])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_3[3])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_3[4])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_3[5])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_3[6])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_3[7])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_6[0])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_6[1])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_6[2])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_6[3])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_6[4])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_6[5])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_6[6])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_6[7])); + float2 _cvt_f32_3 = cake_bf16x2_to_f32x2(lowest); + float _min_7; + asm("min.NaN.f32 %0, %1, %2;" : "=f"(_min_7) : "f"(_cvt_f32_3.x), "f"(_cvt_f32_3.y)); + int _warp_vote_6 = __all_sync(0xFFFFFFFF, _min_7 > -87.0f); + if (_warp_vote_6 != 0) { + float2 _cvt_f32_4 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[0])); + float2 _cvt_f32_5 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[0])); + float2 _cvt_f32_6 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[0])); + float2 _f32x2_mul_b_8 = make_float2(-1.442695f, -1.442695f); + float2 t_3; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_3) : "l"(*(const unsigned long long*)&_cvt_f32_4), "l"(*(const unsigned long long*)&_f32x2_mul_b_8)); + float e0_3 = approx_exp2(t_3.x); + float e1_3 = approx_exp2(t_3.y); + float2 nd_3 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_3, e1_3)); + float d0_3 = -nd_3.x; + float d1_3 = -nd_3.y; + float r0_3 = approx_rcp(d0_3); + float r1_3 = approx_rcp(d1_3); + float2 r_3 = make_float2(r0_3, r1_3); + float2 s_3 = fma_f32x2_rn_noftz(r_3, fma_f32x2_rn_noftz(nd_3, r_3, make_float2(1.0f, 1.0f)), r_3); + float2 s_2 = s_3; + float2 act_2; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_2) : "l"(*(const unsigned long long*)&_cvt_f32_4), "l"(*(const unsigned long long*)&s_2)); + float2 _f32x2_mul_a_9 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_2, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_2), s_2), _cvt_f32_6); + float2 _mul_2; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_9), "l"(*(const unsigned long long*)&_cvt_f32_5)); + float2 _mul_3; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_3) : "l"(*(const unsigned long long*)&act_2), "l"(*(const unsigned long long*)&_cvt_f32_6)); + __nv_bfloat162 _bf16x2_0 = __float22bfloat162_rn(make_float2(_mul_2.x, _mul_2.y)); + __nv_bfloat162 _bf16x2_1 = __float22bfloat162_rn(make_float2(_mul_3.x, _mul_3.y)); + dgate[0] = __as_u32(_bf16x2_0); + dup[0] = __as_u32(_bf16x2_1); + float2 _cvt_f32_7 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[1])); + float2 _cvt_f32_8 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[1])); + float2 _cvt_f32_9 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[1])); + float2 _f32x2_mul_b_10 = make_float2(-1.442695f, -1.442695f); + float2 t_5; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_5) : "l"(*(const unsigned long long*)&_cvt_f32_7), "l"(*(const unsigned long long*)&_f32x2_mul_b_10)); + float e0_5 = approx_exp2(t_5.x); + float e1_5 = approx_exp2(t_5.y); + float2 nd_5 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_5, e1_5)); + float d0_5 = -nd_5.x; + float d1_5 = -nd_5.y; + float r0_5 = approx_rcp(d0_5); + float r1_5 = approx_rcp(d1_5); + float2 r_5 = make_float2(r0_5, r1_5); + float2 s_5 = fma_f32x2_rn_noftz(r_5, fma_f32x2_rn_noftz(nd_5, r_5, make_float2(1.0f, 1.0f)), r_5); + float2 s_4 = s_5; + float2 act_4; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_4) : "l"(*(const unsigned long long*)&_cvt_f32_7), "l"(*(const unsigned long long*)&s_4)); + float2 _f32x2_mul_a_11 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_4, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_4), s_4), _cvt_f32_9); + float2 _mul_4; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_11), "l"(*(const unsigned long long*)&_cvt_f32_8)); + float2 _mul_5; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_5) : "l"(*(const unsigned long long*)&act_4), "l"(*(const unsigned long long*)&_cvt_f32_9)); + __nv_bfloat162 _bf16x2_2 = __float22bfloat162_rn(make_float2(_mul_4.x, _mul_4.y)); + __nv_bfloat162 _bf16x2_3 = __float22bfloat162_rn(make_float2(_mul_5.x, _mul_5.y)); + dgate[1] = __as_u32(_bf16x2_2); + dup[1] = __as_u32(_bf16x2_3); + float2 _cvt_f32_10 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[2])); + float2 _cvt_f32_11 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[2])); + float2 _cvt_f32_12 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[2])); + float2 _f32x2_mul_b_12 = make_float2(-1.442695f, -1.442695f); + float2 t_7; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_7) : "l"(*(const unsigned long long*)&_cvt_f32_10), "l"(*(const unsigned long long*)&_f32x2_mul_b_12)); + float e0_7 = approx_exp2(t_7.x); + float e1_7 = approx_exp2(t_7.y); + float2 nd_7 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_7, e1_7)); + float d0_7 = -nd_7.x; + float d1_7 = -nd_7.y; + float r0_7 = approx_rcp(d0_7); + float r1_7 = approx_rcp(d1_7); + float2 r_7 = make_float2(r0_7, r1_7); + float2 s_7 = fma_f32x2_rn_noftz(r_7, fma_f32x2_rn_noftz(nd_7, r_7, make_float2(1.0f, 1.0f)), r_7); + float2 s_6 = s_7; + float2 act_6; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_6) : "l"(*(const unsigned long long*)&_cvt_f32_10), "l"(*(const unsigned long long*)&s_6)); + float2 _f32x2_mul_a_13 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_6, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_6), s_6), _cvt_f32_12); + float2 _mul_6; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_13), "l"(*(const unsigned long long*)&_cvt_f32_11)); + float2 _mul_7; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_7) : "l"(*(const unsigned long long*)&act_6), "l"(*(const unsigned long long*)&_cvt_f32_12)); + __nv_bfloat162 _bf16x2_4 = __float22bfloat162_rn(make_float2(_mul_6.x, _mul_6.y)); + __nv_bfloat162 _bf16x2_5 = __float22bfloat162_rn(make_float2(_mul_7.x, _mul_7.y)); + dgate[2] = __as_u32(_bf16x2_4); + dup[2] = __as_u32(_bf16x2_5); + float2 _cvt_f32_13 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[3])); + float2 _cvt_f32_14 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[3])); + float2 _cvt_f32_15 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[3])); + float2 _f32x2_mul_b_14 = make_float2(-1.442695f, -1.442695f); + float2 t_9; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_9) : "l"(*(const unsigned long long*)&_cvt_f32_13), "l"(*(const unsigned long long*)&_f32x2_mul_b_14)); + float e0_9 = approx_exp2(t_9.x); + float e1_9 = approx_exp2(t_9.y); + float2 nd_9 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_9, e1_9)); + float d0_9 = -nd_9.x; + float d1_9 = -nd_9.y; + float r0_9 = approx_rcp(d0_9); + float r1_9 = approx_rcp(d1_9); + float2 r_9 = make_float2(r0_9, r1_9); + float2 s_9 = fma_f32x2_rn_noftz(r_9, fma_f32x2_rn_noftz(nd_9, r_9, make_float2(1.0f, 1.0f)), r_9); + float2 s_8 = s_9; + float2 act_8; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_8) : "l"(*(const unsigned long long*)&_cvt_f32_13), "l"(*(const unsigned long long*)&s_8)); + float2 _f32x2_mul_a_15 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_8, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_8), s_8), _cvt_f32_15); + float2 _mul_8; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_15), "l"(*(const unsigned long long*)&_cvt_f32_14)); + float2 _mul_9; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_9) : "l"(*(const unsigned long long*)&act_8), "l"(*(const unsigned long long*)&_cvt_f32_15)); + __nv_bfloat162 _bf16x2_6 = __float22bfloat162_rn(make_float2(_mul_8.x, _mul_8.y)); + __nv_bfloat162 _bf16x2_7 = __float22bfloat162_rn(make_float2(_mul_9.x, _mul_9.y)); + dgate[3] = __as_u32(_bf16x2_6); + dup[3] = __as_u32(_bf16x2_7); + float2 _cvt_f32_16 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[4])); + float2 _cvt_f32_17 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[4])); + float2 _cvt_f32_18 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[4])); + float2 _f32x2_mul_b_16 = make_float2(-1.442695f, -1.442695f); + float2 t_11; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_11) : "l"(*(const unsigned long long*)&_cvt_f32_16), "l"(*(const unsigned long long*)&_f32x2_mul_b_16)); + float e0_11 = approx_exp2(t_11.x); + float e1_11 = approx_exp2(t_11.y); + float2 nd_11 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_11, e1_11)); + float d0_11 = -nd_11.x; + float d1_11 = -nd_11.y; + float r0_11 = approx_rcp(d0_11); + float r1_11 = approx_rcp(d1_11); + float2 r_11 = make_float2(r0_11, r1_11); + float2 s_11 = fma_f32x2_rn_noftz(r_11, fma_f32x2_rn_noftz(nd_11, r_11, make_float2(1.0f, 1.0f)), r_11); + float2 s_10 = s_11; + float2 act_10; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_10) : "l"(*(const unsigned long long*)&_cvt_f32_16), "l"(*(const unsigned long long*)&s_10)); + float2 _f32x2_mul_a_17 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_10, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_10), s_10), _cvt_f32_18); + float2 _mul_10; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_17), "l"(*(const unsigned long long*)&_cvt_f32_17)); + float2 _mul_11; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_11) : "l"(*(const unsigned long long*)&act_10), "l"(*(const unsigned long long*)&_cvt_f32_18)); + __nv_bfloat162 _bf16x2_8 = __float22bfloat162_rn(make_float2(_mul_10.x, _mul_10.y)); + __nv_bfloat162 _bf16x2_9 = __float22bfloat162_rn(make_float2(_mul_11.x, _mul_11.y)); + dgate[4] = __as_u32(_bf16x2_8); + dup[4] = __as_u32(_bf16x2_9); + float2 _cvt_f32_19 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[5])); + float2 _cvt_f32_20 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[5])); + float2 _cvt_f32_21 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[5])); + float2 _f32x2_mul_b_18 = make_float2(-1.442695f, -1.442695f); + float2 t_13; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_13) : "l"(*(const unsigned long long*)&_cvt_f32_19), "l"(*(const unsigned long long*)&_f32x2_mul_b_18)); + float e0_13 = approx_exp2(t_13.x); + float e1_13 = approx_exp2(t_13.y); + float2 nd_13 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_13, e1_13)); + float d0_13 = -nd_13.x; + float d1_13 = -nd_13.y; + float r0_13 = approx_rcp(d0_13); + float r1_13 = approx_rcp(d1_13); + float2 r_13 = make_float2(r0_13, r1_13); + float2 s_13 = fma_f32x2_rn_noftz(r_13, fma_f32x2_rn_noftz(nd_13, r_13, make_float2(1.0f, 1.0f)), r_13); + float2 s_12 = s_13; + float2 act_12; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_12) : "l"(*(const unsigned long long*)&_cvt_f32_19), "l"(*(const unsigned long long*)&s_12)); + float2 _f32x2_mul_a_19 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_12, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_12), s_12), _cvt_f32_21); + float2 _mul_12; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_19), "l"(*(const unsigned long long*)&_cvt_f32_20)); + float2 _mul_13; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_13) : "l"(*(const unsigned long long*)&act_12), "l"(*(const unsigned long long*)&_cvt_f32_21)); + __nv_bfloat162 _bf16x2_10 = __float22bfloat162_rn(make_float2(_mul_12.x, _mul_12.y)); + __nv_bfloat162 _bf16x2_11 = __float22bfloat162_rn(make_float2(_mul_13.x, _mul_13.y)); + dgate[5] = __as_u32(_bf16x2_10); + dup[5] = __as_u32(_bf16x2_11); + float2 _cvt_f32_22 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[6])); + float2 _cvt_f32_23 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[6])); + float2 _cvt_f32_24 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[6])); + float2 _f32x2_mul_b_20 = make_float2(-1.442695f, -1.442695f); + float2 t_15; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_15) : "l"(*(const unsigned long long*)&_cvt_f32_22), "l"(*(const unsigned long long*)&_f32x2_mul_b_20)); + float e0_15 = approx_exp2(t_15.x); + float e1_15 = approx_exp2(t_15.y); + float2 nd_15 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_15, e1_15)); + float d0_15 = -nd_15.x; + float d1_15 = -nd_15.y; + float r0_15 = approx_rcp(d0_15); + float r1_15 = approx_rcp(d1_15); + float2 r_15 = make_float2(r0_15, r1_15); + float2 s_15 = fma_f32x2_rn_noftz(r_15, fma_f32x2_rn_noftz(nd_15, r_15, make_float2(1.0f, 1.0f)), r_15); + float2 s_14 = s_15; + float2 act_14; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_14) : "l"(*(const unsigned long long*)&_cvt_f32_22), "l"(*(const unsigned long long*)&s_14)); + float2 _f32x2_mul_a_21 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_14, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_14), s_14), _cvt_f32_24); + float2 _mul_14; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_21), "l"(*(const unsigned long long*)&_cvt_f32_23)); + float2 _mul_15; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_15) : "l"(*(const unsigned long long*)&act_14), "l"(*(const unsigned long long*)&_cvt_f32_24)); + __nv_bfloat162 _bf16x2_12 = __float22bfloat162_rn(make_float2(_mul_14.x, _mul_14.y)); + __nv_bfloat162 _bf16x2_13 = __float22bfloat162_rn(make_float2(_mul_15.x, _mul_15.y)); + dgate[6] = __as_u32(_bf16x2_12); + dup[6] = __as_u32(_bf16x2_13); + float2 _cvt_f32_25 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[7])); + float2 _cvt_f32_26 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[7])); + float2 _cvt_f32_27 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[7])); + float2 _f32x2_mul_b_22 = make_float2(-1.442695f, -1.442695f); + float2 t_17; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_17) : "l"(*(const unsigned long long*)&_cvt_f32_25), "l"(*(const unsigned long long*)&_f32x2_mul_b_22)); + float e0_17 = approx_exp2(t_17.x); + float e1_17 = approx_exp2(t_17.y); + float2 nd_17 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_17, e1_17)); + float d0_17 = -nd_17.x; + float d1_17 = -nd_17.y; + float r0_17 = approx_rcp(d0_17); + float r1_17 = approx_rcp(d1_17); + float2 r_17 = make_float2(r0_17, r1_17); + float2 s_17 = fma_f32x2_rn_noftz(r_17, fma_f32x2_rn_noftz(nd_17, r_17, make_float2(1.0f, 1.0f)), r_17); + float2 s_16 = s_17; + float2 act_16; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_16) : "l"(*(const unsigned long long*)&_cvt_f32_25), "l"(*(const unsigned long long*)&s_16)); + float2 _f32x2_mul_a_23 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_16, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_16), s_16), _cvt_f32_27); + float2 _mul_16; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_16) : "l"(*(const unsigned long long*)&_f32x2_mul_a_23), "l"(*(const unsigned long long*)&_cvt_f32_26)); + float2 _mul_17; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_17) : "l"(*(const unsigned long long*)&act_16), "l"(*(const unsigned long long*)&_cvt_f32_27)); + __nv_bfloat162 _bf16x2_14 = __float22bfloat162_rn(make_float2(_mul_16.x, _mul_16.y)); + __nv_bfloat162 _bf16x2_15 = __float22bfloat162_rn(make_float2(_mul_17.x, _mul_17.y)); + dgate[7] = __as_u32(_bf16x2_14); + dup[7] = __as_u32(_bf16x2_15); + float2 _cvt_f32_28 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[0])); + float2 _cvt_f32_29 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[0])); + float2 _cvt_f32_30 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[0])); + float2 _f32x2_mul_b_24 = make_float2(-1.442695f, -1.442695f); + float2 t_19; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_19) : "l"(*(const unsigned long long*)&_cvt_f32_28), "l"(*(const unsigned long long*)&_f32x2_mul_b_24)); + float e0_19 = approx_exp2(t_19.x); + float e1_19 = approx_exp2(t_19.y); + float2 nd_19 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_19, e1_19)); + float d0_19 = -nd_19.x; + float d1_19 = -nd_19.y; + float r0_19 = approx_rcp(d0_19); + float r1_19 = approx_rcp(d1_19); + float2 r_19 = make_float2(r0_19, r1_19); + float2 s_19 = fma_f32x2_rn_noftz(r_19, fma_f32x2_rn_noftz(nd_19, r_19, make_float2(1.0f, 1.0f)), r_19); + float2 s_18 = s_19; + float2 act_18; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_18) : "l"(*(const unsigned long long*)&_cvt_f32_28), "l"(*(const unsigned long long*)&s_18)); + float2 _f32x2_mul_a_25 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_18, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_18), s_18), _cvt_f32_30); + float2 _mul_18; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_18) : "l"(*(const unsigned long long*)&_f32x2_mul_a_25), "l"(*(const unsigned long long*)&_cvt_f32_29)); + float2 _mul_19; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_19) : "l"(*(const unsigned long long*)&act_18), "l"(*(const unsigned long long*)&_cvt_f32_30)); + __nv_bfloat162 _bf16x2_16 = __float22bfloat162_rn(make_float2(_mul_18.x, _mul_18.y)); + __nv_bfloat162 _bf16x2_17 = __float22bfloat162_rn(make_float2(_mul_19.x, _mul_19.y)); + dgate_hi[0] = __as_u32(_bf16x2_16); + dup_hi[0] = __as_u32(_bf16x2_17); + float2 _cvt_f32_31 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[1])); + float2 _cvt_f32_32 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[1])); + float2 _cvt_f32_33 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[1])); + float2 _f32x2_mul_b_26 = make_float2(-1.442695f, -1.442695f); + float2 t_21; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_21) : "l"(*(const unsigned long long*)&_cvt_f32_31), "l"(*(const unsigned long long*)&_f32x2_mul_b_26)); + float e0_21 = approx_exp2(t_21.x); + float e1_21 = approx_exp2(t_21.y); + float2 nd_21 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_21, e1_21)); + float d0_21 = -nd_21.x; + float d1_21 = -nd_21.y; + float r0_21 = approx_rcp(d0_21); + float r1_21 = approx_rcp(d1_21); + float2 r_21 = make_float2(r0_21, r1_21); + float2 s_21 = fma_f32x2_rn_noftz(r_21, fma_f32x2_rn_noftz(nd_21, r_21, make_float2(1.0f, 1.0f)), r_21); + float2 s_20 = s_21; + float2 act_20; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_20) : "l"(*(const unsigned long long*)&_cvt_f32_31), "l"(*(const unsigned long long*)&s_20)); + float2 _f32x2_mul_a_27 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_20, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_20), s_20), _cvt_f32_33); + float2 _mul_20; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_20) : "l"(*(const unsigned long long*)&_f32x2_mul_a_27), "l"(*(const unsigned long long*)&_cvt_f32_32)); + float2 _mul_21; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_21) : "l"(*(const unsigned long long*)&act_20), "l"(*(const unsigned long long*)&_cvt_f32_33)); + __nv_bfloat162 _bf16x2_18 = __float22bfloat162_rn(make_float2(_mul_20.x, _mul_20.y)); + __nv_bfloat162 _bf16x2_19 = __float22bfloat162_rn(make_float2(_mul_21.x, _mul_21.y)); + dgate_hi[1] = __as_u32(_bf16x2_18); + dup_hi[1] = __as_u32(_bf16x2_19); + float2 _cvt_f32_34 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[2])); + float2 _cvt_f32_35 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[2])); + float2 _cvt_f32_36 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[2])); + float2 _f32x2_mul_b_28 = make_float2(-1.442695f, -1.442695f); + float2 t_23; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_23) : "l"(*(const unsigned long long*)&_cvt_f32_34), "l"(*(const unsigned long long*)&_f32x2_mul_b_28)); + float e0_23 = approx_exp2(t_23.x); + float e1_23 = approx_exp2(t_23.y); + float2 nd_23 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_23, e1_23)); + float d0_23 = -nd_23.x; + float d1_23 = -nd_23.y; + float r0_23 = approx_rcp(d0_23); + float r1_23 = approx_rcp(d1_23); + float2 r_23 = make_float2(r0_23, r1_23); + float2 s_23 = fma_f32x2_rn_noftz(r_23, fma_f32x2_rn_noftz(nd_23, r_23, make_float2(1.0f, 1.0f)), r_23); + float2 s_22 = s_23; + float2 act_22; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_22) : "l"(*(const unsigned long long*)&_cvt_f32_34), "l"(*(const unsigned long long*)&s_22)); + float2 _f32x2_mul_a_29 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_22, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_22), s_22), _cvt_f32_36); + float2 _mul_22; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_22) : "l"(*(const unsigned long long*)&_f32x2_mul_a_29), "l"(*(const unsigned long long*)&_cvt_f32_35)); + float2 _mul_23; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_23) : "l"(*(const unsigned long long*)&act_22), "l"(*(const unsigned long long*)&_cvt_f32_36)); + __nv_bfloat162 _bf16x2_20 = __float22bfloat162_rn(make_float2(_mul_22.x, _mul_22.y)); + __nv_bfloat162 _bf16x2_21 = __float22bfloat162_rn(make_float2(_mul_23.x, _mul_23.y)); + dgate_hi[2] = __as_u32(_bf16x2_20); + dup_hi[2] = __as_u32(_bf16x2_21); + float2 _cvt_f32_37 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[3])); + float2 _cvt_f32_38 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[3])); + float2 _cvt_f32_39 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[3])); + float2 _f32x2_mul_b_30 = make_float2(-1.442695f, -1.442695f); + float2 t_25; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_25) : "l"(*(const unsigned long long*)&_cvt_f32_37), "l"(*(const unsigned long long*)&_f32x2_mul_b_30)); + float e0_25 = approx_exp2(t_25.x); + float e1_25 = approx_exp2(t_25.y); + float2 nd_25 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_25, e1_25)); + float d0_25 = -nd_25.x; + float d1_25 = -nd_25.y; + float r0_25 = approx_rcp(d0_25); + float r1_25 = approx_rcp(d1_25); + float2 r_25 = make_float2(r0_25, r1_25); + float2 s_25 = fma_f32x2_rn_noftz(r_25, fma_f32x2_rn_noftz(nd_25, r_25, make_float2(1.0f, 1.0f)), r_25); + float2 s_24 = s_25; + float2 act_24; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_24) : "l"(*(const unsigned long long*)&_cvt_f32_37), "l"(*(const unsigned long long*)&s_24)); + float2 _f32x2_mul_a_31 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_24, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_24), s_24), _cvt_f32_39); + float2 _mul_24; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_24) : "l"(*(const unsigned long long*)&_f32x2_mul_a_31), "l"(*(const unsigned long long*)&_cvt_f32_38)); + float2 _mul_25; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_25) : "l"(*(const unsigned long long*)&act_24), "l"(*(const unsigned long long*)&_cvt_f32_39)); + __nv_bfloat162 _bf16x2_22 = __float22bfloat162_rn(make_float2(_mul_24.x, _mul_24.y)); + __nv_bfloat162 _bf16x2_23 = __float22bfloat162_rn(make_float2(_mul_25.x, _mul_25.y)); + dgate_hi[3] = __as_u32(_bf16x2_22); + dup_hi[3] = __as_u32(_bf16x2_23); + float2 _cvt_f32_40 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[4])); + float2 _cvt_f32_41 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[4])); + float2 _cvt_f32_42 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[4])); + float2 _f32x2_mul_b_32 = make_float2(-1.442695f, -1.442695f); + float2 t_27; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_27) : "l"(*(const unsigned long long*)&_cvt_f32_40), "l"(*(const unsigned long long*)&_f32x2_mul_b_32)); + float e0_27 = approx_exp2(t_27.x); + float e1_27 = approx_exp2(t_27.y); + float2 nd_27 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_27, e1_27)); + float d0_27 = -nd_27.x; + float d1_27 = -nd_27.y; + float r0_27 = approx_rcp(d0_27); + float r1_27 = approx_rcp(d1_27); + float2 r_27 = make_float2(r0_27, r1_27); + float2 s_27 = fma_f32x2_rn_noftz(r_27, fma_f32x2_rn_noftz(nd_27, r_27, make_float2(1.0f, 1.0f)), r_27); + float2 s_26 = s_27; + float2 act_26; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_26) : "l"(*(const unsigned long long*)&_cvt_f32_40), "l"(*(const unsigned long long*)&s_26)); + float2 _f32x2_mul_a_33 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_26, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_26), s_26), _cvt_f32_42); + float2 _mul_26; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_26) : "l"(*(const unsigned long long*)&_f32x2_mul_a_33), "l"(*(const unsigned long long*)&_cvt_f32_41)); + float2 _mul_27; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_27) : "l"(*(const unsigned long long*)&act_26), "l"(*(const unsigned long long*)&_cvt_f32_42)); + __nv_bfloat162 _bf16x2_24 = __float22bfloat162_rn(make_float2(_mul_26.x, _mul_26.y)); + __nv_bfloat162 _bf16x2_25 = __float22bfloat162_rn(make_float2(_mul_27.x, _mul_27.y)); + dgate_hi[4] = __as_u32(_bf16x2_24); + dup_hi[4] = __as_u32(_bf16x2_25); + float2 _cvt_f32_43 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[5])); + float2 _cvt_f32_44 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[5])); + float2 _cvt_f32_45 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[5])); + float2 _f32x2_mul_b_34 = make_float2(-1.442695f, -1.442695f); + float2 t_29; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_29) : "l"(*(const unsigned long long*)&_cvt_f32_43), "l"(*(const unsigned long long*)&_f32x2_mul_b_34)); + float e0_29 = approx_exp2(t_29.x); + float e1_29 = approx_exp2(t_29.y); + float2 nd_29 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_29, e1_29)); + float d0_29 = -nd_29.x; + float d1_29 = -nd_29.y; + float r0_29 = approx_rcp(d0_29); + float r1_29 = approx_rcp(d1_29); + float2 r_29 = make_float2(r0_29, r1_29); + float2 s_29 = fma_f32x2_rn_noftz(r_29, fma_f32x2_rn_noftz(nd_29, r_29, make_float2(1.0f, 1.0f)), r_29); + float2 s_28 = s_29; + float2 act_28; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_28) : "l"(*(const unsigned long long*)&_cvt_f32_43), "l"(*(const unsigned long long*)&s_28)); + float2 _f32x2_mul_a_35 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_28, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_28), s_28), _cvt_f32_45); + float2 _mul_28; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_28) : "l"(*(const unsigned long long*)&_f32x2_mul_a_35), "l"(*(const unsigned long long*)&_cvt_f32_44)); + float2 _mul_29; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_29) : "l"(*(const unsigned long long*)&act_28), "l"(*(const unsigned long long*)&_cvt_f32_45)); + __nv_bfloat162 _bf16x2_26 = __float22bfloat162_rn(make_float2(_mul_28.x, _mul_28.y)); + __nv_bfloat162 _bf16x2_27 = __float22bfloat162_rn(make_float2(_mul_29.x, _mul_29.y)); + dgate_hi[5] = __as_u32(_bf16x2_26); + dup_hi[5] = __as_u32(_bf16x2_27); + float2 _cvt_f32_46 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[6])); + float2 _cvt_f32_47 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[6])); + float2 _cvt_f32_48 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[6])); + float2 _f32x2_mul_b_36 = make_float2(-1.442695f, -1.442695f); + float2 t_31; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_31) : "l"(*(const unsigned long long*)&_cvt_f32_46), "l"(*(const unsigned long long*)&_f32x2_mul_b_36)); + float e0_31 = approx_exp2(t_31.x); + float e1_31 = approx_exp2(t_31.y); + float2 nd_31 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_31, e1_31)); + float d0_31 = -nd_31.x; + float d1_31 = -nd_31.y; + float r0_31 = approx_rcp(d0_31); + float r1_31 = approx_rcp(d1_31); + float2 r_31 = make_float2(r0_31, r1_31); + float2 s_31 = fma_f32x2_rn_noftz(r_31, fma_f32x2_rn_noftz(nd_31, r_31, make_float2(1.0f, 1.0f)), r_31); + float2 s_30 = s_31; + float2 act_30; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_30) : "l"(*(const unsigned long long*)&_cvt_f32_46), "l"(*(const unsigned long long*)&s_30)); + float2 _f32x2_mul_a_37 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_30, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_30), s_30), _cvt_f32_48); + float2 _mul_30; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_30) : "l"(*(const unsigned long long*)&_f32x2_mul_a_37), "l"(*(const unsigned long long*)&_cvt_f32_47)); + float2 _mul_31; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_31) : "l"(*(const unsigned long long*)&act_30), "l"(*(const unsigned long long*)&_cvt_f32_48)); + __nv_bfloat162 _bf16x2_28 = __float22bfloat162_rn(make_float2(_mul_30.x, _mul_30.y)); + __nv_bfloat162 _bf16x2_29 = __float22bfloat162_rn(make_float2(_mul_31.x, _mul_31.y)); + dgate_hi[6] = __as_u32(_bf16x2_28); + dup_hi[6] = __as_u32(_bf16x2_29); + float2 _cvt_f32_49 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[7])); + float2 _cvt_f32_50 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[7])); + float2 _cvt_f32_51 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[7])); + float2 _f32x2_mul_b_38 = make_float2(-1.442695f, -1.442695f); + float2 t_33; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_33) : "l"(*(const unsigned long long*)&_cvt_f32_49), "l"(*(const unsigned long long*)&_f32x2_mul_b_38)); + float e0_33 = approx_exp2(t_33.x); + float e1_33 = approx_exp2(t_33.y); + float2 nd_33 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_33, e1_33)); + float d0_33 = -nd_33.x; + float d1_33 = -nd_33.y; + float r0_33 = approx_rcp(d0_33); + float r1_33 = approx_rcp(d1_33); + float2 r_33 = make_float2(r0_33, r1_33); + float2 s_33 = fma_f32x2_rn_noftz(r_33, fma_f32x2_rn_noftz(nd_33, r_33, make_float2(1.0f, 1.0f)), r_33); + float2 s_32 = s_33; + float2 act_32; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_32) : "l"(*(const unsigned long long*)&_cvt_f32_49), "l"(*(const unsigned long long*)&s_32)); + float2 _f32x2_mul_a_39 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_32, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_32), s_32), _cvt_f32_51); + float2 _mul_32; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_32) : "l"(*(const unsigned long long*)&_f32x2_mul_a_39), "l"(*(const unsigned long long*)&_cvt_f32_50)); + float2 _mul_33; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_33) : "l"(*(const unsigned long long*)&act_32), "l"(*(const unsigned long long*)&_cvt_f32_51)); + __nv_bfloat162 _bf16x2_30 = __float22bfloat162_rn(make_float2(_mul_32.x, _mul_32.y)); + __nv_bfloat162 _bf16x2_31 = __float22bfloat162_rn(make_float2(_mul_33.x, _mul_33.y)); + dgate_hi[7] = __as_u32(_bf16x2_30); + dup_hi[7] = __as_u32(_bf16x2_31); + } else { + int band = 0; + float2 _cvt_f32_52 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[0])); + float2 _cvt_f32_53 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[0])); + float2 _cvt_f32_54 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[0])); + float2 _f32x2_mul_b_40 = make_float2(-1.442695f, -1.442695f); + float2 t_35; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_35) : "l"(*(const unsigned long long*)&_cvt_f32_52), "l"(*(const unsigned long long*)&_f32x2_mul_b_40)); + float e0_35 = approx_exp2(t_35.x); + float e1_35 = approx_exp2(t_35.y); + float2 nd_35 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_35, e1_35)); + float d0_35 = -nd_35.x; + float d1_35 = -nd_35.y; + float r0_35 = approx_rcp(d0_35); + float r1_35 = approx_rcp(d1_35); + float2 r_35 = make_float2(r0_35, r1_35); + float2 s_35 = fma_f32x2_rn_noftz(r_35, fma_f32x2_rn_noftz(nd_35, r_35, make_float2(1.0f, 1.0f)), r_35); + float t0_35 = s_35.x; + float t1_35 = s_35.y; + band = band | ((fminf(t0_35, t1_35) == 0.0f) ? 1 : 0); + float2 s_34 = make_float2(max_noftz(t0_35, 0.0f), max_noftz(t1_35, 0.0f)); + float2 act_34; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_34) : "l"(*(const unsigned long long*)&_cvt_f32_52), "l"(*(const unsigned long long*)&s_34)); + float2 _f32x2_mul_a_41 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_34, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_34), s_34), _cvt_f32_54); + float2 _mul_34; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_34) : "l"(*(const unsigned long long*)&_f32x2_mul_a_41), "l"(*(const unsigned long long*)&_cvt_f32_53)); + float2 _mul_35; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_35) : "l"(*(const unsigned long long*)&act_34), "l"(*(const unsigned long long*)&_cvt_f32_54)); + __nv_bfloat162 _bf16x2_32 = __float22bfloat162_rn(make_float2(_mul_34.x, _mul_34.y)); + __nv_bfloat162 _bf16x2_33 = __float22bfloat162_rn(make_float2(_mul_35.x, _mul_35.y)); + dgate[0] = __as_u32(_bf16x2_32); + dup[0] = __as_u32(_bf16x2_33); + float2 _cvt_f32_55 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[1])); + float2 _cvt_f32_56 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[1])); + float2 _cvt_f32_57 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[1])); + float2 _f32x2_mul_b_42 = make_float2(-1.442695f, -1.442695f); + float2 t_37; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_37) : "l"(*(const unsigned long long*)&_cvt_f32_55), "l"(*(const unsigned long long*)&_f32x2_mul_b_42)); + float e0_37 = approx_exp2(t_37.x); + float e1_37 = approx_exp2(t_37.y); + float2 nd_37 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_37, e1_37)); + float d0_37 = -nd_37.x; + float d1_37 = -nd_37.y; + float r0_37 = approx_rcp(d0_37); + float r1_37 = approx_rcp(d1_37); + float2 r_37 = make_float2(r0_37, r1_37); + float2 s_37 = fma_f32x2_rn_noftz(r_37, fma_f32x2_rn_noftz(nd_37, r_37, make_float2(1.0f, 1.0f)), r_37); + float t0_37 = s_37.x; + float t1_37 = s_37.y; + band = band | ((fminf(t0_37, t1_37) == 0.0f) ? 2 : 0); + float2 s_36 = make_float2(max_noftz(t0_37, 0.0f), max_noftz(t1_37, 0.0f)); + float2 act_36; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_36) : "l"(*(const unsigned long long*)&_cvt_f32_55), "l"(*(const unsigned long long*)&s_36)); + float2 _f32x2_mul_a_43 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_36, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_36), s_36), _cvt_f32_57); + float2 _mul_36; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_36) : "l"(*(const unsigned long long*)&_f32x2_mul_a_43), "l"(*(const unsigned long long*)&_cvt_f32_56)); + float2 _mul_37; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_37) : "l"(*(const unsigned long long*)&act_36), "l"(*(const unsigned long long*)&_cvt_f32_57)); + __nv_bfloat162 _bf16x2_34 = __float22bfloat162_rn(make_float2(_mul_36.x, _mul_36.y)); + __nv_bfloat162 _bf16x2_35 = __float22bfloat162_rn(make_float2(_mul_37.x, _mul_37.y)); + dgate[1] = __as_u32(_bf16x2_34); + dup[1] = __as_u32(_bf16x2_35); + float2 _cvt_f32_58 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[2])); + float2 _cvt_f32_59 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[2])); + float2 _cvt_f32_60 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[2])); + float2 _f32x2_mul_b_44 = make_float2(-1.442695f, -1.442695f); + float2 t_39; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_39) : "l"(*(const unsigned long long*)&_cvt_f32_58), "l"(*(const unsigned long long*)&_f32x2_mul_b_44)); + float e0_39 = approx_exp2(t_39.x); + float e1_39 = approx_exp2(t_39.y); + float2 nd_39 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_39, e1_39)); + float d0_39 = -nd_39.x; + float d1_39 = -nd_39.y; + float r0_39 = approx_rcp(d0_39); + float r1_39 = approx_rcp(d1_39); + float2 r_39 = make_float2(r0_39, r1_39); + float2 s_39 = fma_f32x2_rn_noftz(r_39, fma_f32x2_rn_noftz(nd_39, r_39, make_float2(1.0f, 1.0f)), r_39); + float t0_39 = s_39.x; + float t1_39 = s_39.y; + band = band | ((fminf(t0_39, t1_39) == 0.0f) ? 4 : 0); + float2 s_38 = make_float2(max_noftz(t0_39, 0.0f), max_noftz(t1_39, 0.0f)); + float2 act_38; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_38) : "l"(*(const unsigned long long*)&_cvt_f32_58), "l"(*(const unsigned long long*)&s_38)); + float2 _f32x2_mul_a_45 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_38, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_38), s_38), _cvt_f32_60); + float2 _mul_38; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_38) : "l"(*(const unsigned long long*)&_f32x2_mul_a_45), "l"(*(const unsigned long long*)&_cvt_f32_59)); + float2 _mul_39; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_39) : "l"(*(const unsigned long long*)&act_38), "l"(*(const unsigned long long*)&_cvt_f32_60)); + __nv_bfloat162 _bf16x2_36 = __float22bfloat162_rn(make_float2(_mul_38.x, _mul_38.y)); + __nv_bfloat162 _bf16x2_37 = __float22bfloat162_rn(make_float2(_mul_39.x, _mul_39.y)); + dgate[2] = __as_u32(_bf16x2_36); + dup[2] = __as_u32(_bf16x2_37); + float2 _cvt_f32_61 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[3])); + float2 _cvt_f32_62 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[3])); + float2 _cvt_f32_63 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[3])); + float2 _f32x2_mul_b_46 = make_float2(-1.442695f, -1.442695f); + float2 t_41; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_41) : "l"(*(const unsigned long long*)&_cvt_f32_61), "l"(*(const unsigned long long*)&_f32x2_mul_b_46)); + float e0_41 = approx_exp2(t_41.x); + float e1_41 = approx_exp2(t_41.y); + float2 nd_41 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_41, e1_41)); + float d0_41 = -nd_41.x; + float d1_41 = -nd_41.y; + float r0_41 = approx_rcp(d0_41); + float r1_41 = approx_rcp(d1_41); + float2 r_41 = make_float2(r0_41, r1_41); + float2 s_41 = fma_f32x2_rn_noftz(r_41, fma_f32x2_rn_noftz(nd_41, r_41, make_float2(1.0f, 1.0f)), r_41); + float t0_41 = s_41.x; + float t1_41 = s_41.y; + band = band | ((fminf(t0_41, t1_41) == 0.0f) ? 8 : 0); + float2 s_40 = make_float2(max_noftz(t0_41, 0.0f), max_noftz(t1_41, 0.0f)); + float2 act_40; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_40) : "l"(*(const unsigned long long*)&_cvt_f32_61), "l"(*(const unsigned long long*)&s_40)); + float2 _f32x2_mul_a_47 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_40, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_40), s_40), _cvt_f32_63); + float2 _mul_40; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_40) : "l"(*(const unsigned long long*)&_f32x2_mul_a_47), "l"(*(const unsigned long long*)&_cvt_f32_62)); + float2 _mul_41; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_41) : "l"(*(const unsigned long long*)&act_40), "l"(*(const unsigned long long*)&_cvt_f32_63)); + __nv_bfloat162 _bf16x2_38 = __float22bfloat162_rn(make_float2(_mul_40.x, _mul_40.y)); + __nv_bfloat162 _bf16x2_39 = __float22bfloat162_rn(make_float2(_mul_41.x, _mul_41.y)); + dgate[3] = __as_u32(_bf16x2_38); + dup[3] = __as_u32(_bf16x2_39); + float2 _cvt_f32_64 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[4])); + float2 _cvt_f32_65 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[4])); + float2 _cvt_f32_66 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[4])); + float2 _f32x2_mul_b_48 = make_float2(-1.442695f, -1.442695f); + float2 t_43; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_43) : "l"(*(const unsigned long long*)&_cvt_f32_64), "l"(*(const unsigned long long*)&_f32x2_mul_b_48)); + float e0_43 = approx_exp2(t_43.x); + float e1_43 = approx_exp2(t_43.y); + float2 nd_43 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_43, e1_43)); + float d0_43 = -nd_43.x; + float d1_43 = -nd_43.y; + float r0_43 = approx_rcp(d0_43); + float r1_43 = approx_rcp(d1_43); + float2 r_43 = make_float2(r0_43, r1_43); + float2 s_43 = fma_f32x2_rn_noftz(r_43, fma_f32x2_rn_noftz(nd_43, r_43, make_float2(1.0f, 1.0f)), r_43); + float t0_43 = s_43.x; + float t1_43 = s_43.y; + band = band | ((fminf(t0_43, t1_43) == 0.0f) ? 16 : 0); + float2 s_42 = make_float2(max_noftz(t0_43, 0.0f), max_noftz(t1_43, 0.0f)); + float2 act_42; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_42) : "l"(*(const unsigned long long*)&_cvt_f32_64), "l"(*(const unsigned long long*)&s_42)); + float2 _f32x2_mul_a_49 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_42, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_42), s_42), _cvt_f32_66); + float2 _mul_42; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_42) : "l"(*(const unsigned long long*)&_f32x2_mul_a_49), "l"(*(const unsigned long long*)&_cvt_f32_65)); + float2 _mul_43; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_43) : "l"(*(const unsigned long long*)&act_42), "l"(*(const unsigned long long*)&_cvt_f32_66)); + __nv_bfloat162 _bf16x2_40 = __float22bfloat162_rn(make_float2(_mul_42.x, _mul_42.y)); + __nv_bfloat162 _bf16x2_41 = __float22bfloat162_rn(make_float2(_mul_43.x, _mul_43.y)); + dgate[4] = __as_u32(_bf16x2_40); + dup[4] = __as_u32(_bf16x2_41); + float2 _cvt_f32_67 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[5])); + float2 _cvt_f32_68 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[5])); + float2 _cvt_f32_69 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[5])); + float2 _f32x2_mul_b_50 = make_float2(-1.442695f, -1.442695f); + float2 t_45; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_45) : "l"(*(const unsigned long long*)&_cvt_f32_67), "l"(*(const unsigned long long*)&_f32x2_mul_b_50)); + float e0_45 = approx_exp2(t_45.x); + float e1_45 = approx_exp2(t_45.y); + float2 nd_45 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_45, e1_45)); + float d0_45 = -nd_45.x; + float d1_45 = -nd_45.y; + float r0_45 = approx_rcp(d0_45); + float r1_45 = approx_rcp(d1_45); + float2 r_45 = make_float2(r0_45, r1_45); + float2 s_45 = fma_f32x2_rn_noftz(r_45, fma_f32x2_rn_noftz(nd_45, r_45, make_float2(1.0f, 1.0f)), r_45); + float t0_45 = s_45.x; + float t1_45 = s_45.y; + band = band | ((fminf(t0_45, t1_45) == 0.0f) ? 32 : 0); + float2 s_44 = make_float2(max_noftz(t0_45, 0.0f), max_noftz(t1_45, 0.0f)); + float2 act_44; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_44) : "l"(*(const unsigned long long*)&_cvt_f32_67), "l"(*(const unsigned long long*)&s_44)); + float2 _f32x2_mul_a_51 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_44, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_44), s_44), _cvt_f32_69); + float2 _mul_44; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_44) : "l"(*(const unsigned long long*)&_f32x2_mul_a_51), "l"(*(const unsigned long long*)&_cvt_f32_68)); + float2 _mul_45; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_45) : "l"(*(const unsigned long long*)&act_44), "l"(*(const unsigned long long*)&_cvt_f32_69)); + __nv_bfloat162 _bf16x2_42 = __float22bfloat162_rn(make_float2(_mul_44.x, _mul_44.y)); + __nv_bfloat162 _bf16x2_43 = __float22bfloat162_rn(make_float2(_mul_45.x, _mul_45.y)); + dgate[5] = __as_u32(_bf16x2_42); + dup[5] = __as_u32(_bf16x2_43); + float2 _cvt_f32_70 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[6])); + float2 _cvt_f32_71 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[6])); + float2 _cvt_f32_72 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[6])); + float2 _f32x2_mul_b_52 = make_float2(-1.442695f, -1.442695f); + float2 t_47; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_47) : "l"(*(const unsigned long long*)&_cvt_f32_70), "l"(*(const unsigned long long*)&_f32x2_mul_b_52)); + float e0_47 = approx_exp2(t_47.x); + float e1_47 = approx_exp2(t_47.y); + float2 nd_47 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_47, e1_47)); + float d0_47 = -nd_47.x; + float d1_47 = -nd_47.y; + float r0_47 = approx_rcp(d0_47); + float r1_47 = approx_rcp(d1_47); + float2 r_47 = make_float2(r0_47, r1_47); + float2 s_47 = fma_f32x2_rn_noftz(r_47, fma_f32x2_rn_noftz(nd_47, r_47, make_float2(1.0f, 1.0f)), r_47); + float t0_47 = s_47.x; + float t1_47 = s_47.y; + band = band | ((fminf(t0_47, t1_47) == 0.0f) ? 64 : 0); + float2 s_46 = make_float2(max_noftz(t0_47, 0.0f), max_noftz(t1_47, 0.0f)); + float2 act_46; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_46) : "l"(*(const unsigned long long*)&_cvt_f32_70), "l"(*(const unsigned long long*)&s_46)); + float2 _f32x2_mul_a_53 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_46, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_46), s_46), _cvt_f32_72); + float2 _mul_46; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_46) : "l"(*(const unsigned long long*)&_f32x2_mul_a_53), "l"(*(const unsigned long long*)&_cvt_f32_71)); + float2 _mul_47; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_47) : "l"(*(const unsigned long long*)&act_46), "l"(*(const unsigned long long*)&_cvt_f32_72)); + __nv_bfloat162 _bf16x2_44 = __float22bfloat162_rn(make_float2(_mul_46.x, _mul_46.y)); + __nv_bfloat162 _bf16x2_45 = __float22bfloat162_rn(make_float2(_mul_47.x, _mul_47.y)); + dgate[6] = __as_u32(_bf16x2_44); + dup[6] = __as_u32(_bf16x2_45); + float2 _cvt_f32_73 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[7])); + float2 _cvt_f32_74 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[7])); + float2 _cvt_f32_75 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[7])); + float2 _f32x2_mul_b_54 = make_float2(-1.442695f, -1.442695f); + float2 t_49; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_49) : "l"(*(const unsigned long long*)&_cvt_f32_73), "l"(*(const unsigned long long*)&_f32x2_mul_b_54)); + float e0_49 = approx_exp2(t_49.x); + float e1_49 = approx_exp2(t_49.y); + float2 nd_49 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_49, e1_49)); + float d0_49 = -nd_49.x; + float d1_49 = -nd_49.y; + float r0_49 = approx_rcp(d0_49); + float r1_49 = approx_rcp(d1_49); + float2 r_49 = make_float2(r0_49, r1_49); + float2 s_49 = fma_f32x2_rn_noftz(r_49, fma_f32x2_rn_noftz(nd_49, r_49, make_float2(1.0f, 1.0f)), r_49); + float t0_49 = s_49.x; + float t1_49 = s_49.y; + band = band | ((fminf(t0_49, t1_49) == 0.0f) ? 128 : 0); + float2 s_48 = make_float2(max_noftz(t0_49, 0.0f), max_noftz(t1_49, 0.0f)); + float2 act_48; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_48) : "l"(*(const unsigned long long*)&_cvt_f32_73), "l"(*(const unsigned long long*)&s_48)); + float2 _f32x2_mul_a_55 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_48, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_48), s_48), _cvt_f32_75); + float2 _mul_48; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_48) : "l"(*(const unsigned long long*)&_f32x2_mul_a_55), "l"(*(const unsigned long long*)&_cvt_f32_74)); + float2 _mul_49; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_49) : "l"(*(const unsigned long long*)&act_48), "l"(*(const unsigned long long*)&_cvt_f32_75)); + __nv_bfloat162 _bf16x2_46 = __float22bfloat162_rn(make_float2(_mul_48.x, _mul_48.y)); + __nv_bfloat162 _bf16x2_47 = __float22bfloat162_rn(make_float2(_mul_49.x, _mul_49.y)); + dgate[7] = __as_u32(_bf16x2_46); + dup[7] = __as_u32(_bf16x2_47); + int band_hi = 0; + float2 _cvt_f32_76 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[0])); + float2 _cvt_f32_77 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[0])); + float2 _cvt_f32_78 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[0])); + float2 _f32x2_mul_b_56 = make_float2(-1.442695f, -1.442695f); + float2 t_51; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_51) : "l"(*(const unsigned long long*)&_cvt_f32_76), "l"(*(const unsigned long long*)&_f32x2_mul_b_56)); + float e0_51 = approx_exp2(t_51.x); + float e1_51 = approx_exp2(t_51.y); + float2 nd_51 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_51, e1_51)); + float d0_51 = -nd_51.x; + float d1_51 = -nd_51.y; + float r0_51 = approx_rcp(d0_51); + float r1_51 = approx_rcp(d1_51); + float2 r_51 = make_float2(r0_51, r1_51); + float2 s_51 = fma_f32x2_rn_noftz(r_51, fma_f32x2_rn_noftz(nd_51, r_51, make_float2(1.0f, 1.0f)), r_51); + float t0_51 = s_51.x; + float t1_51 = s_51.y; + band_hi = band_hi | ((fminf(t0_51, t1_51) == 0.0f) ? 1 : 0); + float2 s_50 = make_float2(max_noftz(t0_51, 0.0f), max_noftz(t1_51, 0.0f)); + float2 act_50; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_50) : "l"(*(const unsigned long long*)&_cvt_f32_76), "l"(*(const unsigned long long*)&s_50)); + float2 _f32x2_mul_a_57 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_50, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_50), s_50), _cvt_f32_78); + float2 _mul_50; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_50) : "l"(*(const unsigned long long*)&_f32x2_mul_a_57), "l"(*(const unsigned long long*)&_cvt_f32_77)); + float2 _mul_51; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_51) : "l"(*(const unsigned long long*)&act_50), "l"(*(const unsigned long long*)&_cvt_f32_78)); + __nv_bfloat162 _bf16x2_48 = __float22bfloat162_rn(make_float2(_mul_50.x, _mul_50.y)); + __nv_bfloat162 _bf16x2_49 = __float22bfloat162_rn(make_float2(_mul_51.x, _mul_51.y)); + dgate_hi[0] = __as_u32(_bf16x2_48); + dup_hi[0] = __as_u32(_bf16x2_49); + float2 _cvt_f32_79 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[1])); + float2 _cvt_f32_80 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[1])); + float2 _cvt_f32_81 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[1])); + float2 _f32x2_mul_b_58 = make_float2(-1.442695f, -1.442695f); + float2 t_53; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_53) : "l"(*(const unsigned long long*)&_cvt_f32_79), "l"(*(const unsigned long long*)&_f32x2_mul_b_58)); + float e0_53 = approx_exp2(t_53.x); + float e1_53 = approx_exp2(t_53.y); + float2 nd_53 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_53, e1_53)); + float d0_53 = -nd_53.x; + float d1_53 = -nd_53.y; + float r0_53 = approx_rcp(d0_53); + float r1_53 = approx_rcp(d1_53); + float2 r_53 = make_float2(r0_53, r1_53); + float2 s_53 = fma_f32x2_rn_noftz(r_53, fma_f32x2_rn_noftz(nd_53, r_53, make_float2(1.0f, 1.0f)), r_53); + float t0_53 = s_53.x; + float t1_53 = s_53.y; + band_hi = band_hi | ((fminf(t0_53, t1_53) == 0.0f) ? 2 : 0); + float2 s_52 = make_float2(max_noftz(t0_53, 0.0f), max_noftz(t1_53, 0.0f)); + float2 act_52; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_52) : "l"(*(const unsigned long long*)&_cvt_f32_79), "l"(*(const unsigned long long*)&s_52)); + float2 _f32x2_mul_a_59 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_52, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_52), s_52), _cvt_f32_81); + float2 _mul_52; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_52) : "l"(*(const unsigned long long*)&_f32x2_mul_a_59), "l"(*(const unsigned long long*)&_cvt_f32_80)); + float2 _mul_53; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_53) : "l"(*(const unsigned long long*)&act_52), "l"(*(const unsigned long long*)&_cvt_f32_81)); + __nv_bfloat162 _bf16x2_50 = __float22bfloat162_rn(make_float2(_mul_52.x, _mul_52.y)); + __nv_bfloat162 _bf16x2_51 = __float22bfloat162_rn(make_float2(_mul_53.x, _mul_53.y)); + dgate_hi[1] = __as_u32(_bf16x2_50); + dup_hi[1] = __as_u32(_bf16x2_51); + float2 _cvt_f32_82 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[2])); + float2 _cvt_f32_83 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[2])); + float2 _cvt_f32_84 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[2])); + float2 _f32x2_mul_b_60 = make_float2(-1.442695f, -1.442695f); + float2 t_55; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_55) : "l"(*(const unsigned long long*)&_cvt_f32_82), "l"(*(const unsigned long long*)&_f32x2_mul_b_60)); + float e0_55 = approx_exp2(t_55.x); + float e1_55 = approx_exp2(t_55.y); + float2 nd_55 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_55, e1_55)); + float d0_55 = -nd_55.x; + float d1_55 = -nd_55.y; + float r0_55 = approx_rcp(d0_55); + float r1_55 = approx_rcp(d1_55); + float2 r_55 = make_float2(r0_55, r1_55); + float2 s_55 = fma_f32x2_rn_noftz(r_55, fma_f32x2_rn_noftz(nd_55, r_55, make_float2(1.0f, 1.0f)), r_55); + float t0_55 = s_55.x; + float t1_55 = s_55.y; + band_hi = band_hi | ((fminf(t0_55, t1_55) == 0.0f) ? 4 : 0); + float2 s_54 = make_float2(max_noftz(t0_55, 0.0f), max_noftz(t1_55, 0.0f)); + float2 act_54; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_54) : "l"(*(const unsigned long long*)&_cvt_f32_82), "l"(*(const unsigned long long*)&s_54)); + float2 _f32x2_mul_a_61 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_54, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_54), s_54), _cvt_f32_84); + float2 _mul_54; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_54) : "l"(*(const unsigned long long*)&_f32x2_mul_a_61), "l"(*(const unsigned long long*)&_cvt_f32_83)); + float2 _mul_55; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_55) : "l"(*(const unsigned long long*)&act_54), "l"(*(const unsigned long long*)&_cvt_f32_84)); + __nv_bfloat162 _bf16x2_52 = __float22bfloat162_rn(make_float2(_mul_54.x, _mul_54.y)); + __nv_bfloat162 _bf16x2_53 = __float22bfloat162_rn(make_float2(_mul_55.x, _mul_55.y)); + dgate_hi[2] = __as_u32(_bf16x2_52); + dup_hi[2] = __as_u32(_bf16x2_53); + float2 _cvt_f32_85 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[3])); + float2 _cvt_f32_86 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[3])); + float2 _cvt_f32_87 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[3])); + float2 _f32x2_mul_b_62 = make_float2(-1.442695f, -1.442695f); + float2 t_57; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_57) : "l"(*(const unsigned long long*)&_cvt_f32_85), "l"(*(const unsigned long long*)&_f32x2_mul_b_62)); + float e0_57 = approx_exp2(t_57.x); + float e1_57 = approx_exp2(t_57.y); + float2 nd_57 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_57, e1_57)); + float d0_57 = -nd_57.x; + float d1_57 = -nd_57.y; + float r0_57 = approx_rcp(d0_57); + float r1_57 = approx_rcp(d1_57); + float2 r_57 = make_float2(r0_57, r1_57); + float2 s_57 = fma_f32x2_rn_noftz(r_57, fma_f32x2_rn_noftz(nd_57, r_57, make_float2(1.0f, 1.0f)), r_57); + float t0_57 = s_57.x; + float t1_57 = s_57.y; + band_hi = band_hi | ((fminf(t0_57, t1_57) == 0.0f) ? 8 : 0); + float2 s_56 = make_float2(max_noftz(t0_57, 0.0f), max_noftz(t1_57, 0.0f)); + float2 act_56; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_56) : "l"(*(const unsigned long long*)&_cvt_f32_85), "l"(*(const unsigned long long*)&s_56)); + float2 _f32x2_mul_a_63 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_56, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_56), s_56), _cvt_f32_87); + float2 _mul_56; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_56) : "l"(*(const unsigned long long*)&_f32x2_mul_a_63), "l"(*(const unsigned long long*)&_cvt_f32_86)); + float2 _mul_57; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_57) : "l"(*(const unsigned long long*)&act_56), "l"(*(const unsigned long long*)&_cvt_f32_87)); + __nv_bfloat162 _bf16x2_54 = __float22bfloat162_rn(make_float2(_mul_56.x, _mul_56.y)); + __nv_bfloat162 _bf16x2_55 = __float22bfloat162_rn(make_float2(_mul_57.x, _mul_57.y)); + dgate_hi[3] = __as_u32(_bf16x2_54); + dup_hi[3] = __as_u32(_bf16x2_55); + float2 _cvt_f32_88 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[4])); + float2 _cvt_f32_89 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[4])); + float2 _cvt_f32_90 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[4])); + float2 _f32x2_mul_b_64 = make_float2(-1.442695f, -1.442695f); + float2 t_59; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_59) : "l"(*(const unsigned long long*)&_cvt_f32_88), "l"(*(const unsigned long long*)&_f32x2_mul_b_64)); + float e0_59 = approx_exp2(t_59.x); + float e1_59 = approx_exp2(t_59.y); + float2 nd_59 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_59, e1_59)); + float d0_59 = -nd_59.x; + float d1_59 = -nd_59.y; + float r0_59 = approx_rcp(d0_59); + float r1_59 = approx_rcp(d1_59); + float2 r_59 = make_float2(r0_59, r1_59); + float2 s_59 = fma_f32x2_rn_noftz(r_59, fma_f32x2_rn_noftz(nd_59, r_59, make_float2(1.0f, 1.0f)), r_59); + float t0_59 = s_59.x; + float t1_59 = s_59.y; + band_hi = band_hi | ((fminf(t0_59, t1_59) == 0.0f) ? 16 : 0); + float2 s_58 = make_float2(max_noftz(t0_59, 0.0f), max_noftz(t1_59, 0.0f)); + float2 act_58; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_58) : "l"(*(const unsigned long long*)&_cvt_f32_88), "l"(*(const unsigned long long*)&s_58)); + float2 _f32x2_mul_a_65 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_58, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_58), s_58), _cvt_f32_90); + float2 _mul_58; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_58) : "l"(*(const unsigned long long*)&_f32x2_mul_a_65), "l"(*(const unsigned long long*)&_cvt_f32_89)); + float2 _mul_59; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_59) : "l"(*(const unsigned long long*)&act_58), "l"(*(const unsigned long long*)&_cvt_f32_90)); + __nv_bfloat162 _bf16x2_56 = __float22bfloat162_rn(make_float2(_mul_58.x, _mul_58.y)); + __nv_bfloat162 _bf16x2_57 = __float22bfloat162_rn(make_float2(_mul_59.x, _mul_59.y)); + dgate_hi[4] = __as_u32(_bf16x2_56); + dup_hi[4] = __as_u32(_bf16x2_57); + float2 _cvt_f32_91 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[5])); + float2 _cvt_f32_92 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[5])); + float2 _cvt_f32_93 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[5])); + float2 _f32x2_mul_b_66 = make_float2(-1.442695f, -1.442695f); + float2 t_61; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_61) : "l"(*(const unsigned long long*)&_cvt_f32_91), "l"(*(const unsigned long long*)&_f32x2_mul_b_66)); + float e0_61 = approx_exp2(t_61.x); + float e1_61 = approx_exp2(t_61.y); + float2 nd_61 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_61, e1_61)); + float d0_61 = -nd_61.x; + float d1_61 = -nd_61.y; + float r0_61 = approx_rcp(d0_61); + float r1_61 = approx_rcp(d1_61); + float2 r_61 = make_float2(r0_61, r1_61); + float2 s_61 = fma_f32x2_rn_noftz(r_61, fma_f32x2_rn_noftz(nd_61, r_61, make_float2(1.0f, 1.0f)), r_61); + float t0_61 = s_61.x; + float t1_61 = s_61.y; + band_hi = band_hi | ((fminf(t0_61, t1_61) == 0.0f) ? 32 : 0); + float2 s_60 = make_float2(max_noftz(t0_61, 0.0f), max_noftz(t1_61, 0.0f)); + float2 act_60; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_60) : "l"(*(const unsigned long long*)&_cvt_f32_91), "l"(*(const unsigned long long*)&s_60)); + float2 _f32x2_mul_a_67 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_60, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_60), s_60), _cvt_f32_93); + float2 _mul_60; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_60) : "l"(*(const unsigned long long*)&_f32x2_mul_a_67), "l"(*(const unsigned long long*)&_cvt_f32_92)); + float2 _mul_61; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_61) : "l"(*(const unsigned long long*)&act_60), "l"(*(const unsigned long long*)&_cvt_f32_93)); + __nv_bfloat162 _bf16x2_58 = __float22bfloat162_rn(make_float2(_mul_60.x, _mul_60.y)); + __nv_bfloat162 _bf16x2_59 = __float22bfloat162_rn(make_float2(_mul_61.x, _mul_61.y)); + dgate_hi[5] = __as_u32(_bf16x2_58); + dup_hi[5] = __as_u32(_bf16x2_59); + float2 _cvt_f32_94 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[6])); + float2 _cvt_f32_95 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[6])); + float2 _cvt_f32_96 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[6])); + float2 _f32x2_mul_b_68 = make_float2(-1.442695f, -1.442695f); + float2 t_63; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_63) : "l"(*(const unsigned long long*)&_cvt_f32_94), "l"(*(const unsigned long long*)&_f32x2_mul_b_68)); + float e0_63 = approx_exp2(t_63.x); + float e1_63 = approx_exp2(t_63.y); + float2 nd_63 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_63, e1_63)); + float d0_63 = -nd_63.x; + float d1_63 = -nd_63.y; + float r0_63 = approx_rcp(d0_63); + float r1_63 = approx_rcp(d1_63); + float2 r_63 = make_float2(r0_63, r1_63); + float2 s_63 = fma_f32x2_rn_noftz(r_63, fma_f32x2_rn_noftz(nd_63, r_63, make_float2(1.0f, 1.0f)), r_63); + float t0_63 = s_63.x; + float t1_63 = s_63.y; + band_hi = band_hi | ((fminf(t0_63, t1_63) == 0.0f) ? 64 : 0); + float2 s_62 = make_float2(max_noftz(t0_63, 0.0f), max_noftz(t1_63, 0.0f)); + float2 act_62; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_62) : "l"(*(const unsigned long long*)&_cvt_f32_94), "l"(*(const unsigned long long*)&s_62)); + float2 _f32x2_mul_a_69 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_62, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_62), s_62), _cvt_f32_96); + float2 _mul_62; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_62) : "l"(*(const unsigned long long*)&_f32x2_mul_a_69), "l"(*(const unsigned long long*)&_cvt_f32_95)); + float2 _mul_63; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_63) : "l"(*(const unsigned long long*)&act_62), "l"(*(const unsigned long long*)&_cvt_f32_96)); + __nv_bfloat162 _bf16x2_60 = __float22bfloat162_rn(make_float2(_mul_62.x, _mul_62.y)); + __nv_bfloat162 _bf16x2_61 = __float22bfloat162_rn(make_float2(_mul_63.x, _mul_63.y)); + dgate_hi[6] = __as_u32(_bf16x2_60); + dup_hi[6] = __as_u32(_bf16x2_61); + float2 _cvt_f32_97 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_6[7])); + float2 _cvt_f32_98 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_7[7])); + float2 _cvt_f32_99 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_8[7])); + float2 _f32x2_mul_b_70 = make_float2(-1.442695f, -1.442695f); + float2 t_65; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&t_65) : "l"(*(const unsigned long long*)&_cvt_f32_97), "l"(*(const unsigned long long*)&_f32x2_mul_b_70)); + float e0_65 = approx_exp2(t_65.x); + float e1_65 = approx_exp2(t_65.y); + float2 nd_65 = sub_f32x2_noftz(make_float2(-1.0f, -1.0f), make_float2(e0_65, e1_65)); + float d0_65 = -nd_65.x; + float d1_65 = -nd_65.y; + float r0_65 = approx_rcp(d0_65); + float r1_65 = approx_rcp(d1_65); + float2 r_65 = make_float2(r0_65, r1_65); + float2 s_65 = fma_f32x2_rn_noftz(r_65, fma_f32x2_rn_noftz(nd_65, r_65, make_float2(1.0f, 1.0f)), r_65); + float t0_65 = s_65.x; + float t1_65 = s_65.y; + band_hi = band_hi | ((fminf(t0_65, t1_65) == 0.0f) ? 128 : 0); + float2 s_64 = make_float2(max_noftz(t0_65, 0.0f), max_noftz(t1_65, 0.0f)); + float2 act_64; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_64) : "l"(*(const unsigned long long*)&_cvt_f32_97), "l"(*(const unsigned long long*)&s_64)); + float2 _f32x2_mul_a_71 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_64, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_64), s_64), _cvt_f32_99); + float2 _mul_64; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_64) : "l"(*(const unsigned long long*)&_f32x2_mul_a_71), "l"(*(const unsigned long long*)&_cvt_f32_98)); + float2 _mul_65; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_65) : "l"(*(const unsigned long long*)&act_64), "l"(*(const unsigned long long*)&_cvt_f32_99)); + __nv_bfloat162 _bf16x2_62 = __float22bfloat162_rn(make_float2(_mul_64.x, _mul_64.y)); + __nv_bfloat162 _bf16x2_63 = __float22bfloat162_rn(make_float2(_mul_65.x, _mul_65.y)); + dgate_hi[7] = __as_u32(_bf16x2_62); + dup_hi[7] = __as_u32(_bf16x2_63); + if (band != 0) { + unsigned int _popc_0 = (unsigned int)__popc(band); + #pragma unroll 1 + for (int _j = 0; _j < _popc_0; _j++) { + int _find_first_set_72 = __ffs(band); + int j = _find_first_set_72 - 1; + band = band & band - 1; + unsigned long long both = band_pair(gated_input, grad, gate_index + 2 * j, grow * K + col + 2 * j, K); + unsigned int dgate_new = (unsigned int)both; + unsigned int dup_new = (unsigned int)(both >> 32); + dgate[0] = ((j == 0) ? dgate_new : dgate[0]); + dup[0] = ((j == 0) ? dup_new : dup[0]); + dgate[1] = ((j == 1) ? dgate_new : dgate[1]); + dup[1] = ((j == 1) ? dup_new : dup[1]); + dgate[2] = ((j == 2) ? dgate_new : dgate[2]); + dup[2] = ((j == 2) ? dup_new : dup[2]); + dgate[3] = ((j == 3) ? dgate_new : dgate[3]); + dup[3] = ((j == 3) ? dup_new : dup[3]); + dgate[4] = ((j == 4) ? dgate_new : dgate[4]); + dup[4] = ((j == 4) ? dup_new : dup[4]); + dgate[5] = ((j == 5) ? dgate_new : dgate[5]); + dup[5] = ((j == 5) ? dup_new : dup[5]); + dgate[6] = ((j == 6) ? dgate_new : dgate[6]); + dup[6] = ((j == 6) ? dup_new : dup[6]); + dgate[7] = ((j == 7) ? dgate_new : dgate[7]); + dup[7] = ((j == 7) ? dup_new : dup[7]); + } + } + if (band_hi != 0) { + unsigned int _popc_1 = (unsigned int)__popc(band_hi); + #pragma unroll 1 + for (int _j_hi = 0; _j_hi < _popc_1; _j_hi++) { + int _find_first_set_73 = __ffs(band_hi); + int j_hi = _find_first_set_73 - 1; + band_hi = band_hi & band_hi - 1; + unsigned long long both_hi = band_pair(gated_input, grad, gate_index_hi + 2 * j_hi, grow_hi * K + col + 2 * j_hi, K); + unsigned int dgate_new_hi = (unsigned int)both_hi; + unsigned int dup_new_hi = (unsigned int)(both_hi >> 32); + dgate_hi[0] = ((j_hi == 0) ? dgate_new_hi : dgate_hi[0]); + dup_hi[0] = ((j_hi == 0) ? dup_new_hi : dup_hi[0]); + dgate_hi[1] = ((j_hi == 1) ? dgate_new_hi : dgate_hi[1]); + dup_hi[1] = ((j_hi == 1) ? dup_new_hi : dup_hi[1]); + dgate_hi[2] = ((j_hi == 2) ? dgate_new_hi : dgate_hi[2]); + dup_hi[2] = ((j_hi == 2) ? dup_new_hi : dup_hi[2]); + dgate_hi[3] = ((j_hi == 3) ? dgate_new_hi : dgate_hi[3]); + dup_hi[3] = ((j_hi == 3) ? dup_new_hi : dup_hi[3]); + dgate_hi[4] = ((j_hi == 4) ? dgate_new_hi : dgate_hi[4]); + dup_hi[4] = ((j_hi == 4) ? dup_new_hi : dup_hi[4]); + dgate_hi[5] = ((j_hi == 5) ? dgate_new_hi : dgate_hi[5]); + dup_hi[5] = ((j_hi == 5) ? dup_new_hi : dup_hi[5]); + dgate_hi[6] = ((j_hi == 6) ? dgate_new_hi : dgate_hi[6]); + dup_hi[6] = ((j_hi == 6) ? dup_new_hi : dup_hi[6]); + dgate_hi[7] = ((j_hi == 7) ? dgate_new_hi : dgate_hi[7]); + dup_hi[7] = ((j_hi == 7) ? dup_new_hi : dup_hi[7]); + } + } + } + unsigned int swizzle = (unsigned int)(row >> 3 & 6 | row & 1); + unsigned int row_base = tiles_addr + (unsigned int)(row * 128); + unsigned int chunk = (unsigned int)((tid & 3) * 2); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base + ((chunk ^ swizzle) << 4)), "r"(dgate[0]), "r"(dgate[1]), "r"(dgate[2]), "r"(dgate[3]) : "memory"); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base + ((chunk + 1 ^ swizzle) << 4)), "r"(dgate[4]), "r"(dgate[5]), "r"(dgate[6]), "r"(dgate[7]) : "memory"); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base + (8192 + ((chunk ^ swizzle) << 4))), "r"(dup[0]), "r"(dup[1]), "r"(dup[2]), "r"(dup[3]) : "memory"); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base + (8192 + ((chunk + 1 ^ swizzle) << 4))), "r"(dup[4]), "r"(dup[5]), "r"(dup[6]), "r"(dup[7]) : "memory"); + unsigned int swizzle_hi = swizzle ^ 4; + unsigned int row_base_hi = row_base + 4096; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base_hi + ((chunk ^ swizzle_hi) << 4)), "r"(dgate_hi[0]), "r"(dgate_hi[1]), "r"(dgate_hi[2]), "r"(dgate_hi[3]) : "memory"); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base_hi + ((chunk + 1 ^ swizzle_hi) << 4)), "r"(dgate_hi[4]), "r"(dgate_hi[5]), "r"(dgate_hi[6]), "r"(dgate_hi[7]) : "memory"); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base_hi + (8192 + ((chunk ^ swizzle_hi) << 4))), "r"(dup_hi[0]), "r"(dup_hi[1]), "r"(dup_hi[2]), "r"(dup_hi[3]) : "memory"); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(row_base_hi + (8192 + ((chunk + 1 ^ swizzle_hi) << 4))), "r"(dup_hi[4]), "r"(dup_hi[5]), "r"(dup_hi[6]), "r"(dup_hi[7]) : "memory"); + int scale_cols = K >> 6; + int scale_col = bx * 2 + blk; + int out_index = grow * (2 * K) + col; + __nv_bfloat162 amax = __as_bf16x2(0); + __nv_bfloat162 _hxa_75 = amax; + __nv_bfloat162 _hxb_75 = __as_bf16x2(dgate[0]); + unsigned int _hxr_75; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_75) : "r"(*reinterpret_cast(&_hxa_75)), "r"(*reinterpret_cast(&_hxb_75))); + __nv_bfloat162 _max_74 = *reinterpret_cast(&_hxr_75); + amax = _max_74; + __nv_bfloat162 _hxa_77 = amax; + __nv_bfloat162 _hxb_77 = __as_bf16x2(dgate[1]); + unsigned int _hxr_77; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_77) : "r"(*reinterpret_cast(&_hxa_77)), "r"(*reinterpret_cast(&_hxb_77))); + __nv_bfloat162 _max_76 = *reinterpret_cast(&_hxr_77); + amax = _max_76; + __nv_bfloat162 _hxa_79 = amax; + __nv_bfloat162 _hxb_79 = __as_bf16x2(dgate[2]); + unsigned int _hxr_79; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_79) : "r"(*reinterpret_cast(&_hxa_79)), "r"(*reinterpret_cast(&_hxb_79))); + __nv_bfloat162 _max_78 = *reinterpret_cast(&_hxr_79); + amax = _max_78; + __nv_bfloat162 _hxa_81 = amax; + __nv_bfloat162 _hxb_81 = __as_bf16x2(dgate[3]); + unsigned int _hxr_81; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_81) : "r"(*reinterpret_cast(&_hxa_81)), "r"(*reinterpret_cast(&_hxb_81))); + __nv_bfloat162 _max_80 = *reinterpret_cast(&_hxr_81); + amax = _max_80; + __nv_bfloat162 _hxa_83 = amax; + __nv_bfloat162 _hxb_83 = __as_bf16x2(dgate[4]); + unsigned int _hxr_83; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_83) : "r"(*reinterpret_cast(&_hxa_83)), "r"(*reinterpret_cast(&_hxb_83))); + __nv_bfloat162 _max_82 = *reinterpret_cast(&_hxr_83); + amax = _max_82; + __nv_bfloat162 _hxa_85 = amax; + __nv_bfloat162 _hxb_85 = __as_bf16x2(dgate[5]); + unsigned int _hxr_85; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_85) : "r"(*reinterpret_cast(&_hxa_85)), "r"(*reinterpret_cast(&_hxb_85))); + __nv_bfloat162 _max_84 = *reinterpret_cast(&_hxr_85); + amax = _max_84; + __nv_bfloat162 _hxa_87 = amax; + __nv_bfloat162 _hxb_87 = __as_bf16x2(dgate[6]); + unsigned int _hxr_87; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_87) : "r"(*reinterpret_cast(&_hxa_87)), "r"(*reinterpret_cast(&_hxb_87))); + __nv_bfloat162 _max_86 = *reinterpret_cast(&_hxr_87); + amax = _max_86; + __nv_bfloat162 _hxa_89 = amax; + __nv_bfloat162 _hxb_89 = __as_bf16x2(dgate[7]); + unsigned int _hxr_89; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_89) : "r"(*reinterpret_cast(&_hxa_89)), "r"(*reinterpret_cast(&_hxb_89))); + __nv_bfloat162 _max_88 = *reinterpret_cast(&_hxr_89); + amax = _max_88; + unsigned int mag = __as_u32(amax) & 2147450879; + unsigned int peer = __shfl_xor_sync(0xFFFFFFFF, mag, 1); + __nv_bfloat162 both_1 = __hmax2_nan(__as_bf16x2(mag), __as_bf16x2(peer)); + unsigned int folded = __as_u32(__hmax2_nan(both_1, __as_bf16x2(__as_u32(both_1) >> 16))); + unsigned int amax_bits = (folded & 65535) << 16; + int scale_i32 = (int)(amax_bits + 2031616 >> 23) - 8; + if (scale_i32 < 0) { + scale_i32 = 0; + } + if ((amax_bits & 2139095040) == 2139095040) { + scale_i32 = 255; + } + unsigned int scale = (unsigned int)scale_i32; + unsigned int inv = 254 - scale << 7; + if (scale == 255) { + inv = 32704; + } + unsigned int inv2 = inv | inv << 16; + int scale_index = ((grow >> 7) * scale_cols + (scale_col >> 2)) * 512 + (grow & 31) * 16 + (grow >> 5 & 3) * 4 + (scale_col & 3); + if (half == 0) { + *(reinterpret_cast(row_scales + scale_index) + (0)) = (unsigned char)(scale); + } + unsigned int words[4]; + uint32_t _bf16x2_mul_0; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_0) : "r"(dgate[0]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_0; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_0) : "r"(_bf16x2_mul_0)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_0); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_0) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_1; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_1) : "r"(dgate[1]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_91; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_91) : "r"(_bf16x2_mul_1)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_1); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_91) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_90; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_90) : "h"(_e4m3x2_bf16x2_0), "h"(_cvt_satfinite_e4m3x2_bf16x2_91)); + words[0] = _mov_b32_pack_u16x2_90; + uint32_t _bf16x2_mul_2; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_2) : "r"(dgate[2]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_1; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_1) : "r"(_bf16x2_mul_2)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_2); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_1) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_3; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_3) : "r"(dgate[3]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_93; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_93) : "r"(_bf16x2_mul_3)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_3); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_93) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_92; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_92) : "h"(_e4m3x2_bf16x2_1), "h"(_cvt_satfinite_e4m3x2_bf16x2_93)); + words[1] = _mov_b32_pack_u16x2_92; + uint32_t _bf16x2_mul_4; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_4) : "r"(dgate[4]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_2; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_2) : "r"(_bf16x2_mul_4)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_4); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_2) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_5; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_5) : "r"(dgate[5]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_95; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_95) : "r"(_bf16x2_mul_5)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_5); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_95) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_94; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_94) : "h"(_e4m3x2_bf16x2_2), "h"(_cvt_satfinite_e4m3x2_bf16x2_95)); + words[2] = _mov_b32_pack_u16x2_94; + uint32_t _bf16x2_mul_6; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_6) : "r"(dgate[6]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_3; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_3) : "r"(_bf16x2_mul_6)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_6); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_3) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_7; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_7) : "r"(dgate[7]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_97; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_97) : "r"(_bf16x2_mul_7)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_7); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_97) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_96; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_96) : "h"(_e4m3x2_bf16x2_3), "h"(_cvt_satfinite_e4m3x2_bf16x2_97)); + words[3] = _mov_b32_pack_u16x2_96; + reinterpret_cast(row_output + out_index)[0] = reinterpret_cast(words + 0)[0]; + __nv_bfloat162 amax_0 = __as_bf16x2(0); + __nv_bfloat162 _hxa_99 = amax_0; + __nv_bfloat162 _hxb_99 = __as_bf16x2(dup[0]); + unsigned int _hxr_99; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_99) : "r"(*reinterpret_cast(&_hxa_99)), "r"(*reinterpret_cast(&_hxb_99))); + __nv_bfloat162 _max_98 = *reinterpret_cast(&_hxr_99); + amax_0 = _max_98; + __nv_bfloat162 _hxa_101 = amax_0; + __nv_bfloat162 _hxb_101 = __as_bf16x2(dup[1]); + unsigned int _hxr_101; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_101) : "r"(*reinterpret_cast(&_hxa_101)), "r"(*reinterpret_cast(&_hxb_101))); + __nv_bfloat162 _max_100 = *reinterpret_cast(&_hxr_101); + amax_0 = _max_100; + __nv_bfloat162 _hxa_103 = amax_0; + __nv_bfloat162 _hxb_103 = __as_bf16x2(dup[2]); + unsigned int _hxr_103; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_103) : "r"(*reinterpret_cast(&_hxa_103)), "r"(*reinterpret_cast(&_hxb_103))); + __nv_bfloat162 _max_102 = *reinterpret_cast(&_hxr_103); + amax_0 = _max_102; + __nv_bfloat162 _hxa_105 = amax_0; + __nv_bfloat162 _hxb_105 = __as_bf16x2(dup[3]); + unsigned int _hxr_105; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_105) : "r"(*reinterpret_cast(&_hxa_105)), "r"(*reinterpret_cast(&_hxb_105))); + __nv_bfloat162 _max_104 = *reinterpret_cast(&_hxr_105); + amax_0 = _max_104; + __nv_bfloat162 _hxa_107 = amax_0; + __nv_bfloat162 _hxb_107 = __as_bf16x2(dup[4]); + unsigned int _hxr_107; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_107) : "r"(*reinterpret_cast(&_hxa_107)), "r"(*reinterpret_cast(&_hxb_107))); + __nv_bfloat162 _max_106 = *reinterpret_cast(&_hxr_107); + amax_0 = _max_106; + __nv_bfloat162 _hxa_109 = amax_0; + __nv_bfloat162 _hxb_109 = __as_bf16x2(dup[5]); + unsigned int _hxr_109; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_109) : "r"(*reinterpret_cast(&_hxa_109)), "r"(*reinterpret_cast(&_hxb_109))); + __nv_bfloat162 _max_108 = *reinterpret_cast(&_hxr_109); + amax_0 = _max_108; + __nv_bfloat162 _hxa_111 = amax_0; + __nv_bfloat162 _hxb_111 = __as_bf16x2(dup[6]); + unsigned int _hxr_111; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_111) : "r"(*reinterpret_cast(&_hxa_111)), "r"(*reinterpret_cast(&_hxb_111))); + __nv_bfloat162 _max_110 = *reinterpret_cast(&_hxr_111); + amax_0 = _max_110; + __nv_bfloat162 _hxa_113 = amax_0; + __nv_bfloat162 _hxb_113 = __as_bf16x2(dup[7]); + unsigned int _hxr_113; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_113) : "r"(*reinterpret_cast(&_hxa_113)), "r"(*reinterpret_cast(&_hxb_113))); + __nv_bfloat162 _max_112 = *reinterpret_cast(&_hxr_113); + amax_0 = _max_112; + unsigned int mag_1 = __as_u32(amax_0) & 2147450879; + unsigned int peer_2 = __shfl_xor_sync(0xFFFFFFFF, mag_1, 1); + __nv_bfloat162 both_3 = __hmax2_nan(__as_bf16x2(mag_1), __as_bf16x2(peer_2)); + unsigned int folded_4 = __as_u32(__hmax2_nan(both_3, __as_bf16x2(__as_u32(both_3) >> 16))); + unsigned int amax_bits_5 = (folded_4 & 65535) << 16; + int scale_i32_6 = (int)(amax_bits_5 + 2031616 >> 23) - 8; + if (scale_i32_6 < 0) { + scale_i32_6 = 0; + } + if ((amax_bits_5 & 2139095040) == 2139095040) { + scale_i32_6 = 255; + } + unsigned int scale_7 = (unsigned int)scale_i32_6; + unsigned int inv_8 = 254 - scale_7 << 7; + if (scale_7 == 255) { + inv_8 = 32704; + } + unsigned int inv2_9 = inv_8 | inv_8 << 16; + int scale_index_10 = ((grow >> 7) * scale_cols + (scale_col + (K >> 5) >> 2)) * 512 + (grow & 31) * 16 + (grow >> 5 & 3) * 4 + (scale_col + (K >> 5) & 3); + if (half == 0) { + *(reinterpret_cast(row_scales + scale_index_10) + (0)) = (unsigned char)(scale_7); + } + unsigned int words_11[4]; + uint32_t _bf16x2_mul_8; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_8) : "r"(dup[0]), "r"(inv2_9)); + uint16_t _e4m3x2_bf16x2_4; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_4) : "r"(_bf16x2_mul_8)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_8); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_4) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_9; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_9) : "r"(dup[1]), "r"(inv2_9)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_115; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_115) : "r"(_bf16x2_mul_9)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_9); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_115) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_114; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_114) : "h"(_e4m3x2_bf16x2_4), "h"(_cvt_satfinite_e4m3x2_bf16x2_115)); + words_11[0] = _mov_b32_pack_u16x2_114; + uint32_t _bf16x2_mul_10; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_10) : "r"(dup[2]), "r"(inv2_9)); + uint16_t _e4m3x2_bf16x2_5; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_5) : "r"(_bf16x2_mul_10)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_10); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_5) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_11; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_11) : "r"(dup[3]), "r"(inv2_9)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_117; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_117) : "r"(_bf16x2_mul_11)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_11); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_117) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_116; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_116) : "h"(_e4m3x2_bf16x2_5), "h"(_cvt_satfinite_e4m3x2_bf16x2_117)); + words_11[1] = _mov_b32_pack_u16x2_116; + uint32_t _bf16x2_mul_12; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_12) : "r"(dup[4]), "r"(inv2_9)); + uint16_t _e4m3x2_bf16x2_6; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_6) : "r"(_bf16x2_mul_12)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_12); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_6) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_13; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_13) : "r"(dup[5]), "r"(inv2_9)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_119; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_119) : "r"(_bf16x2_mul_13)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_13); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_119) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_118; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_118) : "h"(_e4m3x2_bf16x2_6), "h"(_cvt_satfinite_e4m3x2_bf16x2_119)); + words_11[2] = _mov_b32_pack_u16x2_118; + uint32_t _bf16x2_mul_14; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_14) : "r"(dup[6]), "r"(inv2_9)); + uint16_t _e4m3x2_bf16x2_7; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_7) : "r"(_bf16x2_mul_14)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_14); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_7) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_15; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_15) : "r"(dup[7]), "r"(inv2_9)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_121; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_121) : "r"(_bf16x2_mul_15)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_15); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_121) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_120; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_120) : "h"(_e4m3x2_bf16x2_7), "h"(_cvt_satfinite_e4m3x2_bf16x2_121)); + words_11[3] = _mov_b32_pack_u16x2_120; + reinterpret_cast(row_output + (out_index + K))[0] = reinterpret_cast(words_11 + 0)[0]; + int out_index_hi = grow_hi * (2 * K) + col; + __nv_bfloat162 amax_12 = __as_bf16x2(0); + __nv_bfloat162 _hxa_123 = amax_12; + __nv_bfloat162 _hxb_123 = __as_bf16x2(dgate_hi[0]); + unsigned int _hxr_123; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_123) : "r"(*reinterpret_cast(&_hxa_123)), "r"(*reinterpret_cast(&_hxb_123))); + __nv_bfloat162 _max_122 = *reinterpret_cast(&_hxr_123); + amax_12 = _max_122; + __nv_bfloat162 _hxa_125 = amax_12; + __nv_bfloat162 _hxb_125 = __as_bf16x2(dgate_hi[1]); + unsigned int _hxr_125; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_125) : "r"(*reinterpret_cast(&_hxa_125)), "r"(*reinterpret_cast(&_hxb_125))); + __nv_bfloat162 _max_124 = *reinterpret_cast(&_hxr_125); + amax_12 = _max_124; + __nv_bfloat162 _hxa_127 = amax_12; + __nv_bfloat162 _hxb_127 = __as_bf16x2(dgate_hi[2]); + unsigned int _hxr_127; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_127) : "r"(*reinterpret_cast(&_hxa_127)), "r"(*reinterpret_cast(&_hxb_127))); + __nv_bfloat162 _max_126 = *reinterpret_cast(&_hxr_127); + amax_12 = _max_126; + __nv_bfloat162 _hxa_129 = amax_12; + __nv_bfloat162 _hxb_129 = __as_bf16x2(dgate_hi[3]); + unsigned int _hxr_129; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_129) : "r"(*reinterpret_cast(&_hxa_129)), "r"(*reinterpret_cast(&_hxb_129))); + __nv_bfloat162 _max_128 = *reinterpret_cast(&_hxr_129); + amax_12 = _max_128; + __nv_bfloat162 _hxa_131 = amax_12; + __nv_bfloat162 _hxb_131 = __as_bf16x2(dgate_hi[4]); + unsigned int _hxr_131; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_131) : "r"(*reinterpret_cast(&_hxa_131)), "r"(*reinterpret_cast(&_hxb_131))); + __nv_bfloat162 _max_130 = *reinterpret_cast(&_hxr_131); + amax_12 = _max_130; + __nv_bfloat162 _hxa_133 = amax_12; + __nv_bfloat162 _hxb_133 = __as_bf16x2(dgate_hi[5]); + unsigned int _hxr_133; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_133) : "r"(*reinterpret_cast(&_hxa_133)), "r"(*reinterpret_cast(&_hxb_133))); + __nv_bfloat162 _max_132 = *reinterpret_cast(&_hxr_133); + amax_12 = _max_132; + __nv_bfloat162 _hxa_135 = amax_12; + __nv_bfloat162 _hxb_135 = __as_bf16x2(dgate_hi[6]); + unsigned int _hxr_135; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_135) : "r"(*reinterpret_cast(&_hxa_135)), "r"(*reinterpret_cast(&_hxb_135))); + __nv_bfloat162 _max_134 = *reinterpret_cast(&_hxr_135); + amax_12 = _max_134; + __nv_bfloat162 _hxa_137 = amax_12; + __nv_bfloat162 _hxb_137 = __as_bf16x2(dgate_hi[7]); + unsigned int _hxr_137; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_137) : "r"(*reinterpret_cast(&_hxa_137)), "r"(*reinterpret_cast(&_hxb_137))); + __nv_bfloat162 _max_136 = *reinterpret_cast(&_hxr_137); + amax_12 = _max_136; + unsigned int mag_13 = __as_u32(amax_12) & 2147450879; + unsigned int peer_14 = __shfl_xor_sync(0xFFFFFFFF, mag_13, 1); + __nv_bfloat162 both_15 = __hmax2_nan(__as_bf16x2(mag_13), __as_bf16x2(peer_14)); + unsigned int folded_16 = __as_u32(__hmax2_nan(both_15, __as_bf16x2(__as_u32(both_15) >> 16))); + unsigned int amax_bits_17 = (folded_16 & 65535) << 16; + int scale_i32_18 = (int)(amax_bits_17 + 2031616 >> 23) - 8; + if (scale_i32_18 < 0) { + scale_i32_18 = 0; + } + if ((amax_bits_17 & 2139095040) == 2139095040) { + scale_i32_18 = 255; + } + unsigned int scale_19 = (unsigned int)scale_i32_18; + unsigned int inv_20 = 254 - scale_19 << 7; + if (scale_19 == 255) { + inv_20 = 32704; + } + unsigned int inv2_21 = inv_20 | inv_20 << 16; + int scale_index_22 = ((grow_hi >> 7) * scale_cols + (scale_col >> 2)) * 512 + (grow_hi & 31) * 16 + (grow_hi >> 5 & 3) * 4 + (scale_col & 3); + if (half == 0) { + *(reinterpret_cast(row_scales + scale_index_22) + (0)) = (unsigned char)(scale_19); + } + unsigned int words_23[4]; + uint32_t _bf16x2_mul_16; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_16) : "r"(dgate_hi[0]), "r"(inv2_21)); + uint16_t _e4m3x2_bf16x2_8; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_8) : "r"(_bf16x2_mul_16)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_16); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_8) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_17; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_17) : "r"(dgate_hi[1]), "r"(inv2_21)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_139; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_139) : "r"(_bf16x2_mul_17)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_17); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_139) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_138; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_138) : "h"(_e4m3x2_bf16x2_8), "h"(_cvt_satfinite_e4m3x2_bf16x2_139)); + words_23[0] = _mov_b32_pack_u16x2_138; + uint32_t _bf16x2_mul_18; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_18) : "r"(dgate_hi[2]), "r"(inv2_21)); + uint16_t _e4m3x2_bf16x2_9; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_9) : "r"(_bf16x2_mul_18)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_18); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_9) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_19; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_19) : "r"(dgate_hi[3]), "r"(inv2_21)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_141; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_141) : "r"(_bf16x2_mul_19)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_19); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_141) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_140; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_140) : "h"(_e4m3x2_bf16x2_9), "h"(_cvt_satfinite_e4m3x2_bf16x2_141)); + words_23[1] = _mov_b32_pack_u16x2_140; + uint32_t _bf16x2_mul_20; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_20) : "r"(dgate_hi[4]), "r"(inv2_21)); + uint16_t _e4m3x2_bf16x2_10; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_10) : "r"(_bf16x2_mul_20)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_20); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_10) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_21; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_21) : "r"(dgate_hi[5]), "r"(inv2_21)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_143; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_143) : "r"(_bf16x2_mul_21)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_21); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_143) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_142; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_142) : "h"(_e4m3x2_bf16x2_10), "h"(_cvt_satfinite_e4m3x2_bf16x2_143)); + words_23[2] = _mov_b32_pack_u16x2_142; + uint32_t _bf16x2_mul_22; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_22) : "r"(dgate_hi[6]), "r"(inv2_21)); + uint16_t _e4m3x2_bf16x2_11; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_11) : "r"(_bf16x2_mul_22)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_22); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_11) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_23; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_23) : "r"(dgate_hi[7]), "r"(inv2_21)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_145; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_145) : "r"(_bf16x2_mul_23)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_23); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_145) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_144; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_144) : "h"(_e4m3x2_bf16x2_11), "h"(_cvt_satfinite_e4m3x2_bf16x2_145)); + words_23[3] = _mov_b32_pack_u16x2_144; + reinterpret_cast(row_output + out_index_hi)[0] = reinterpret_cast(words_23 + 0)[0]; + __nv_bfloat162 amax_24 = __as_bf16x2(0); + __nv_bfloat162 _hxa_147 = amax_24; + __nv_bfloat162 _hxb_147 = __as_bf16x2(dup_hi[0]); + unsigned int _hxr_147; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_147) : "r"(*reinterpret_cast(&_hxa_147)), "r"(*reinterpret_cast(&_hxb_147))); + __nv_bfloat162 _max_146 = *reinterpret_cast(&_hxr_147); + amax_24 = _max_146; + __nv_bfloat162 _hxa_149 = amax_24; + __nv_bfloat162 _hxb_149 = __as_bf16x2(dup_hi[1]); + unsigned int _hxr_149; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_149) : "r"(*reinterpret_cast(&_hxa_149)), "r"(*reinterpret_cast(&_hxb_149))); + __nv_bfloat162 _max_148 = *reinterpret_cast(&_hxr_149); + amax_24 = _max_148; + __nv_bfloat162 _hxa_151 = amax_24; + __nv_bfloat162 _hxb_151 = __as_bf16x2(dup_hi[2]); + unsigned int _hxr_151; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_151) : "r"(*reinterpret_cast(&_hxa_151)), "r"(*reinterpret_cast(&_hxb_151))); + __nv_bfloat162 _max_150 = *reinterpret_cast(&_hxr_151); + amax_24 = _max_150; + __nv_bfloat162 _hxa_153 = amax_24; + __nv_bfloat162 _hxb_153 = __as_bf16x2(dup_hi[3]); + unsigned int _hxr_153; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_153) : "r"(*reinterpret_cast(&_hxa_153)), "r"(*reinterpret_cast(&_hxb_153))); + __nv_bfloat162 _max_152 = *reinterpret_cast(&_hxr_153); + amax_24 = _max_152; + __nv_bfloat162 _hxa_155 = amax_24; + __nv_bfloat162 _hxb_155 = __as_bf16x2(dup_hi[4]); + unsigned int _hxr_155; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_155) : "r"(*reinterpret_cast(&_hxa_155)), "r"(*reinterpret_cast(&_hxb_155))); + __nv_bfloat162 _max_154 = *reinterpret_cast(&_hxr_155); + amax_24 = _max_154; + __nv_bfloat162 _hxa_157 = amax_24; + __nv_bfloat162 _hxb_157 = __as_bf16x2(dup_hi[5]); + unsigned int _hxr_157; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_157) : "r"(*reinterpret_cast(&_hxa_157)), "r"(*reinterpret_cast(&_hxb_157))); + __nv_bfloat162 _max_156 = *reinterpret_cast(&_hxr_157); + amax_24 = _max_156; + __nv_bfloat162 _hxa_159 = amax_24; + __nv_bfloat162 _hxb_159 = __as_bf16x2(dup_hi[6]); + unsigned int _hxr_159; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_159) : "r"(*reinterpret_cast(&_hxa_159)), "r"(*reinterpret_cast(&_hxb_159))); + __nv_bfloat162 _max_158 = *reinterpret_cast(&_hxr_159); + amax_24 = _max_158; + __nv_bfloat162 _hxa_161 = amax_24; + __nv_bfloat162 _hxb_161 = __as_bf16x2(dup_hi[7]); + unsigned int _hxr_161; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_161) : "r"(*reinterpret_cast(&_hxa_161)), "r"(*reinterpret_cast(&_hxb_161))); + __nv_bfloat162 _max_160 = *reinterpret_cast(&_hxr_161); + amax_24 = _max_160; + unsigned int mag_25 = __as_u32(amax_24) & 2147450879; + unsigned int peer_26 = __shfl_xor_sync(0xFFFFFFFF, mag_25, 1); + __nv_bfloat162 both_27 = __hmax2_nan(__as_bf16x2(mag_25), __as_bf16x2(peer_26)); + unsigned int folded_28 = __as_u32(__hmax2_nan(both_27, __as_bf16x2(__as_u32(both_27) >> 16))); + unsigned int amax_bits_29 = (folded_28 & 65535) << 16; + int scale_i32_30 = (int)(amax_bits_29 + 2031616 >> 23) - 8; + if (scale_i32_30 < 0) { + scale_i32_30 = 0; + } + if ((amax_bits_29 & 2139095040) == 2139095040) { + scale_i32_30 = 255; + } + unsigned int scale_31 = (unsigned int)scale_i32_30; + unsigned int inv_32 = 254 - scale_31 << 7; + if (scale_31 == 255) { + inv_32 = 32704; + } + unsigned int inv2_33 = inv_32 | inv_32 << 16; + int scale_index_34 = ((grow_hi >> 7) * scale_cols + (scale_col + (K >> 5) >> 2)) * 512 + (grow_hi & 31) * 16 + (grow_hi >> 5 & 3) * 4 + (scale_col + (K >> 5) & 3); + if (half == 0) { + *(reinterpret_cast(row_scales + scale_index_34) + (0)) = (unsigned char)(scale_31); + } + unsigned int words_35[4]; + uint32_t _bf16x2_mul_24; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_24) : "r"(dup_hi[0]), "r"(inv2_33)); + uint16_t _e4m3x2_bf16x2_12; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_12) : "r"(_bf16x2_mul_24)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_24); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_12) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_25; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_25) : "r"(dup_hi[1]), "r"(inv2_33)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_163; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_163) : "r"(_bf16x2_mul_25)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_25); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_163) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_162; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_162) : "h"(_e4m3x2_bf16x2_12), "h"(_cvt_satfinite_e4m3x2_bf16x2_163)); + words_35[0] = _mov_b32_pack_u16x2_162; + uint32_t _bf16x2_mul_26; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_26) : "r"(dup_hi[2]), "r"(inv2_33)); + uint16_t _e4m3x2_bf16x2_13; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_13) : "r"(_bf16x2_mul_26)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_26); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_13) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_27; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_27) : "r"(dup_hi[3]), "r"(inv2_33)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_165; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_165) : "r"(_bf16x2_mul_27)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_27); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_165) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_164; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_164) : "h"(_e4m3x2_bf16x2_13), "h"(_cvt_satfinite_e4m3x2_bf16x2_165)); + words_35[1] = _mov_b32_pack_u16x2_164; + uint32_t _bf16x2_mul_28; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_28) : "r"(dup_hi[4]), "r"(inv2_33)); + uint16_t _e4m3x2_bf16x2_14; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_14) : "r"(_bf16x2_mul_28)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_28); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_14) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_29; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_29) : "r"(dup_hi[5]), "r"(inv2_33)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_167; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_167) : "r"(_bf16x2_mul_29)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_29); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_167) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_166; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_166) : "h"(_e4m3x2_bf16x2_14), "h"(_cvt_satfinite_e4m3x2_bf16x2_167)); + words_35[2] = _mov_b32_pack_u16x2_166; + uint32_t _bf16x2_mul_30; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_30) : "r"(dup_hi[6]), "r"(inv2_33)); + uint16_t _e4m3x2_bf16x2_15; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_15) : "r"(_bf16x2_mul_30)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_30); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_15) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_31; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_31) : "r"(dup_hi[7]), "r"(inv2_33)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_169; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_169) : "r"(_bf16x2_mul_31)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_31); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_169) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_168; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_168) : "h"(_e4m3x2_bf16x2_15), "h"(_cvt_satfinite_e4m3x2_bf16x2_169)); + words_35[3] = _mov_b32_pack_u16x2_168; + reinterpret_cast(row_output + (out_index_hi + K))[0] = reinterpret_cast(words_35 + 0)[0]; + __syncthreads(); + int lane_1 = tid & 31; + int warp_1 = tid >> 5; + int k8 = lane_1 & 7; + int srow = (k8 >> 1) * 16 + (lane_1 >> 3) * 2 + (k8 & 1); + unsigned int sswizzle = (unsigned int)(srow >> 3 & 6 | srow & 1); + unsigned int cols[16]; + unsigned int strip_chunk0 = (unsigned int)(warp_1 * 2); + unsigned int strip_src0 = tiles_addr + (unsigned int)(srow * 128) + ((strip_chunk0 ^ sswizzle) << 4); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols[0]), "=r"(cols[1]), "=r"(cols[2]), "=r"(cols[3]) + : "r"(strip_src0) + : "memory"); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols[4]), "=r"(cols[5]), "=r"(cols[6]), "=r"(cols[7]) + : "r"(strip_src0 + 1024) + : "memory"); + unsigned int strip_chunk1 = (unsigned int)(warp_1 * 2 + 1); + unsigned int strip_src1 = tiles_addr + (unsigned int)(srow * 128) + ((strip_chunk1 ^ sswizzle) << 4); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols[8]), "=r"(cols[9]), "=r"(cols[10]), "=r"(cols[11]) + : "r"(strip_src1) + : "memory"); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols[12]), "=r"(cols[13]), "=r"(cols[14]), "=r"(cols[15]) + : "r"(strip_src1 + 1024) + : "memory"); + __nv_bfloat162 amax_a = __as_bf16x2(0); + __nv_bfloat162 amax_b = __as_bf16x2(0); + __nv_bfloat162 _hxa_171 = amax_a; + __nv_bfloat162 _hxb_171 = __as_bf16x2(cols[0]); + unsigned int _hxr_171; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_171) : "r"(*reinterpret_cast(&_hxa_171)), "r"(*reinterpret_cast(&_hxb_171))); + __nv_bfloat162 _max_170 = *reinterpret_cast(&_hxr_171); + amax_a = _max_170; + __nv_bfloat162 _hxa_173 = amax_b; + __nv_bfloat162 _hxb_173 = __as_bf16x2(cols[8]); + unsigned int _hxr_173; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_173) : "r"(*reinterpret_cast(&_hxa_173)), "r"(*reinterpret_cast(&_hxb_173))); + __nv_bfloat162 _max_172 = *reinterpret_cast(&_hxr_173); + amax_b = _max_172; + __nv_bfloat162 _hxa_175 = amax_a; + __nv_bfloat162 _hxb_175 = __as_bf16x2(cols[1]); + unsigned int _hxr_175; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_175) : "r"(*reinterpret_cast(&_hxa_175)), "r"(*reinterpret_cast(&_hxb_175))); + __nv_bfloat162 _max_174 = *reinterpret_cast(&_hxr_175); + amax_a = _max_174; + __nv_bfloat162 _hxa_177 = amax_b; + __nv_bfloat162 _hxb_177 = __as_bf16x2(cols[9]); + unsigned int _hxr_177; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_177) : "r"(*reinterpret_cast(&_hxa_177)), "r"(*reinterpret_cast(&_hxb_177))); + __nv_bfloat162 _max_176 = *reinterpret_cast(&_hxr_177); + amax_b = _max_176; + __nv_bfloat162 _hxa_179 = amax_a; + __nv_bfloat162 _hxb_179 = __as_bf16x2(cols[2]); + unsigned int _hxr_179; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_179) : "r"(*reinterpret_cast(&_hxa_179)), "r"(*reinterpret_cast(&_hxb_179))); + __nv_bfloat162 _max_178 = *reinterpret_cast(&_hxr_179); + amax_a = _max_178; + __nv_bfloat162 _hxa_181 = amax_b; + __nv_bfloat162 _hxb_181 = __as_bf16x2(cols[10]); + unsigned int _hxr_181; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_181) : "r"(*reinterpret_cast(&_hxa_181)), "r"(*reinterpret_cast(&_hxb_181))); + __nv_bfloat162 _max_180 = *reinterpret_cast(&_hxr_181); + amax_b = _max_180; + __nv_bfloat162 _hxa_183 = amax_a; + __nv_bfloat162 _hxb_183 = __as_bf16x2(cols[3]); + unsigned int _hxr_183; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_183) : "r"(*reinterpret_cast(&_hxa_183)), "r"(*reinterpret_cast(&_hxb_183))); + __nv_bfloat162 _max_182 = *reinterpret_cast(&_hxr_183); + amax_a = _max_182; + __nv_bfloat162 _hxa_185 = amax_b; + __nv_bfloat162 _hxb_185 = __as_bf16x2(cols[11]); + unsigned int _hxr_185; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_185) : "r"(*reinterpret_cast(&_hxa_185)), "r"(*reinterpret_cast(&_hxb_185))); + __nv_bfloat162 _max_184 = *reinterpret_cast(&_hxr_185); + amax_b = _max_184; + __nv_bfloat162 _hxa_187 = amax_a; + __nv_bfloat162 _hxb_187 = __as_bf16x2(cols[4]); + unsigned int _hxr_187; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_187) : "r"(*reinterpret_cast(&_hxa_187)), "r"(*reinterpret_cast(&_hxb_187))); + __nv_bfloat162 _max_186 = *reinterpret_cast(&_hxr_187); + amax_a = _max_186; + __nv_bfloat162 _hxa_189 = amax_b; + __nv_bfloat162 _hxb_189 = __as_bf16x2(cols[12]); + unsigned int _hxr_189; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_189) : "r"(*reinterpret_cast(&_hxa_189)), "r"(*reinterpret_cast(&_hxb_189))); + __nv_bfloat162 _max_188 = *reinterpret_cast(&_hxr_189); + amax_b = _max_188; + __nv_bfloat162 _hxa_191 = amax_a; + __nv_bfloat162 _hxb_191 = __as_bf16x2(cols[5]); + unsigned int _hxr_191; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_191) : "r"(*reinterpret_cast(&_hxa_191)), "r"(*reinterpret_cast(&_hxb_191))); + __nv_bfloat162 _max_190 = *reinterpret_cast(&_hxr_191); + amax_a = _max_190; + __nv_bfloat162 _hxa_193 = amax_b; + __nv_bfloat162 _hxb_193 = __as_bf16x2(cols[13]); + unsigned int _hxr_193; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_193) : "r"(*reinterpret_cast(&_hxa_193)), "r"(*reinterpret_cast(&_hxb_193))); + __nv_bfloat162 _max_192 = *reinterpret_cast(&_hxr_193); + amax_b = _max_192; + __nv_bfloat162 _hxa_195 = amax_a; + __nv_bfloat162 _hxb_195 = __as_bf16x2(cols[6]); + unsigned int _hxr_195; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_195) : "r"(*reinterpret_cast(&_hxa_195)), "r"(*reinterpret_cast(&_hxb_195))); + __nv_bfloat162 _max_194 = *reinterpret_cast(&_hxr_195); + amax_a = _max_194; + __nv_bfloat162 _hxa_197 = amax_b; + __nv_bfloat162 _hxb_197 = __as_bf16x2(cols[14]); + unsigned int _hxr_197; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_197) : "r"(*reinterpret_cast(&_hxa_197)), "r"(*reinterpret_cast(&_hxb_197))); + __nv_bfloat162 _max_196 = *reinterpret_cast(&_hxr_197); + amax_b = _max_196; + __nv_bfloat162 _hxa_199 = amax_a; + __nv_bfloat162 _hxb_199 = __as_bf16x2(cols[7]); + unsigned int _hxr_199; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_199) : "r"(*reinterpret_cast(&_hxa_199)), "r"(*reinterpret_cast(&_hxb_199))); + __nv_bfloat162 _max_198 = *reinterpret_cast(&_hxr_199); + amax_a = _max_198; + __nv_bfloat162 _hxa_201 = amax_b; + __nv_bfloat162 _hxb_201 = __as_bf16x2(cols[15]); + unsigned int _hxr_201; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_201) : "r"(*reinterpret_cast(&_hxa_201)), "r"(*reinterpret_cast(&_hxb_201))); + __nv_bfloat162 _max_200 = *reinterpret_cast(&_hxr_201); + amax_b = _max_200; + unsigned int mag_a = __as_u32(amax_a) & 2147450879; + unsigned int mag_b = __as_u32(amax_b) & 2147450879; + unsigned int lows_ab = mag_a & 65535 | mag_b << 16; + unsigned int highs_ab = mag_a >> 16 | mag_b & 4294901760u; + __nv_bfloat162 col_max = __hmax2_nan(__as_bf16x2(lows_ab), __as_bf16x2(highs_ab)); + unsigned int peer1 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(col_max), 1); + col_max = __hmax2_nan(col_max, __as_bf16x2(peer1)); + unsigned int col_max_bits = __as_u32(col_max); + unsigned int amax_bits_36 = (col_max_bits & 65535) << 16; + int scale_i32_37 = (int)(amax_bits_36 + 2031616 >> 23) - 8; + if (scale_i32_37 < 0) { + scale_i32_37 = 0; + } + if ((amax_bits_36 & 2139095040) == 2139095040) { + scale_i32_37 = 255; + } + unsigned int scale_38 = (unsigned int)scale_i32_37; + unsigned int inv_39 = 254 - scale_38 << 7; + if (scale_38 == 255) { + inv_39 = 32704; + } + unsigned int inv2_40 = inv_39 | inv_39 << 16; + unsigned int amax_bits_41 = (col_max_bits >> 16 & 65535) << 16; + int scale_i32_42 = (int)(amax_bits_41 + 2031616 >> 23) - 8; + if (scale_i32_42 < 0) { + scale_i32_42 = 0; + } + if ((amax_bits_41 & 2139095040) == 2139095040) { + scale_i32_42 = 255; + } + unsigned int scale_43 = (unsigned int)scale_i32_42; + unsigned int inv_44 = 254 - scale_43 << 7; + if (scale_43 == 255) { + inv_44 = 32704; + } + unsigned int inv2_45 = inv_44 | inv_44 << 16; + int gcol_a = bx * 64 + warp_1 * 16 + (lane_1 >> 2); + int gcol_b = gcol_a + 8; + int out_row = by * 64 + (lane_1 & 3) * 16; + unsigned int words_a[4]; + unsigned int words_b[4]; + uint32_t _bf16x2_mul_32; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_32) : "r"(cols[0]), "r"(inv2_40)); + uint16_t _e4m3x2_bf16x2_16; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_16) : "r"(_bf16x2_mul_32)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_32); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_16) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_33; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_33) : "r"(cols[1]), "r"(inv2_40)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_203; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_203) : "r"(_bf16x2_mul_33)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_33); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_203) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_202; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_202) : "h"(_e4m3x2_bf16x2_16), "h"(_cvt_satfinite_e4m3x2_bf16x2_203)); + words_a[0] = _mov_b32_pack_u16x2_202; + uint32_t _bf16x2_mul_34; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_34) : "r"(cols[8]), "r"(inv2_45)); + uint16_t _e4m3x2_bf16x2_17; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_17) : "r"(_bf16x2_mul_34)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_34); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_17) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_35; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_35) : "r"(cols[9]), "r"(inv2_45)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_205; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_205) : "r"(_bf16x2_mul_35)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_35); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_205) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_204; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_204) : "h"(_e4m3x2_bf16x2_17), "h"(_cvt_satfinite_e4m3x2_bf16x2_205)); + words_b[0] = _mov_b32_pack_u16x2_204; + uint32_t _bf16x2_mul_36; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_36) : "r"(cols[2]), "r"(inv2_40)); + uint16_t _e4m3x2_bf16x2_18; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_18) : "r"(_bf16x2_mul_36)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_36); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_18) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_37; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_37) : "r"(cols[3]), "r"(inv2_40)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_207; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_207) : "r"(_bf16x2_mul_37)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_37); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_207) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_206; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_206) : "h"(_e4m3x2_bf16x2_18), "h"(_cvt_satfinite_e4m3x2_bf16x2_207)); + words_a[1] = _mov_b32_pack_u16x2_206; + uint32_t _bf16x2_mul_38; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_38) : "r"(cols[10]), "r"(inv2_45)); + uint16_t _e4m3x2_bf16x2_19; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_19) : "r"(_bf16x2_mul_38)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_38); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_19) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_39; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_39) : "r"(cols[11]), "r"(inv2_45)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_209; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_209) : "r"(_bf16x2_mul_39)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_39); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_209) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_208; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_208) : "h"(_e4m3x2_bf16x2_19), "h"(_cvt_satfinite_e4m3x2_bf16x2_209)); + words_b[1] = _mov_b32_pack_u16x2_208; + uint32_t _bf16x2_mul_40; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_40) : "r"(cols[4]), "r"(inv2_40)); + uint16_t _e4m3x2_bf16x2_20; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_20) : "r"(_bf16x2_mul_40)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_40); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_20) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_41; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_41) : "r"(cols[5]), "r"(inv2_40)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_211; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_211) : "r"(_bf16x2_mul_41)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_41); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_211) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_210; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_210) : "h"(_e4m3x2_bf16x2_20), "h"(_cvt_satfinite_e4m3x2_bf16x2_211)); + words_a[2] = _mov_b32_pack_u16x2_210; + uint32_t _bf16x2_mul_42; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_42) : "r"(cols[12]), "r"(inv2_45)); + uint16_t _e4m3x2_bf16x2_21; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_21) : "r"(_bf16x2_mul_42)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_42); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_21) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_43; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_43) : "r"(cols[13]), "r"(inv2_45)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_213; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_213) : "r"(_bf16x2_mul_43)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_43); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_213) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_212; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_212) : "h"(_e4m3x2_bf16x2_21), "h"(_cvt_satfinite_e4m3x2_bf16x2_213)); + words_b[2] = _mov_b32_pack_u16x2_212; + uint32_t _bf16x2_mul_44; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_44) : "r"(cols[6]), "r"(inv2_40)); + uint16_t _e4m3x2_bf16x2_22; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_22) : "r"(_bf16x2_mul_44)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_44); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_22) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_45; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_45) : "r"(cols[7]), "r"(inv2_40)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_215; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_215) : "r"(_bf16x2_mul_45)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_45); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_215) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_214; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_214) : "h"(_e4m3x2_bf16x2_22), "h"(_cvt_satfinite_e4m3x2_bf16x2_215)); + words_a[3] = _mov_b32_pack_u16x2_214; + uint32_t _bf16x2_mul_46; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_46) : "r"(cols[14]), "r"(inv2_45)); + uint16_t _e4m3x2_bf16x2_23; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_23) : "r"(_bf16x2_mul_46)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_46); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_23) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_47; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_47) : "r"(cols[15]), "r"(inv2_45)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_217; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_217) : "r"(_bf16x2_mul_47)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_47); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_217) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_216; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_216) : "h"(_e4m3x2_bf16x2_23), "h"(_cvt_satfinite_e4m3x2_bf16x2_217)); + words_b[3] = _mov_b32_pack_u16x2_216; + reinterpret_cast(col_output + (gcol_a * M + out_row))[0] = reinterpret_cast(words_a + 0)[0]; + reinterpret_cast(col_output + (gcol_b * M + out_row))[0] = reinterpret_cast(words_b + 0)[0]; + if ((lane_1 & 1) == 0) { + int row_tiles = M >> 7; + int rblk = by * 2 + ((lane_1 & 3) >> 1); + int col_scale_a = ((gcol_a >> 7) * row_tiles + (rblk >> 2)) * 512 + (gcol_a & 31) * 16 + (gcol_a >> 5 & 3) * 4 + (rblk & 3); + int col_scale_b = ((gcol_b >> 7) * row_tiles + (rblk >> 2)) * 512 + (gcol_b & 31) * 16 + (gcol_b >> 5 & 3) * 4 + (rblk & 3); + *(reinterpret_cast(col_scales + col_scale_a) + (0)) = (unsigned char)(scale_38); + *(reinterpret_cast(col_scales + col_scale_b) + (0)) = (unsigned char)(scale_43); + } + int lane_46 = tid & 31; + int warp_47 = tid >> 5; + int k8_48 = lane_46 & 7; + int srow_49 = (k8_48 >> 1) * 16 + (lane_46 >> 3) * 2 + (k8_48 & 1); + unsigned int sswizzle_50 = (unsigned int)(srow_49 >> 3 & 6 | srow_49 & 1); + unsigned int cols_51[16]; + unsigned int strip_chunk0_52 = (unsigned int)(warp_47 * 2); + unsigned int strip_src0_53 = tiles_addr + 8192 + (unsigned int)(srow_49 * 128) + ((strip_chunk0_52 ^ sswizzle_50) << 4); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols_51[0]), "=r"(cols_51[1]), "=r"(cols_51[2]), "=r"(cols_51[3]) + : "r"(strip_src0_53) + : "memory"); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols_51[4]), "=r"(cols_51[5]), "=r"(cols_51[6]), "=r"(cols_51[7]) + : "r"(strip_src0_53 + 1024) + : "memory"); + unsigned int strip_chunk1_54 = (unsigned int)(warp_47 * 2 + 1); + unsigned int strip_src1_55 = tiles_addr + 8192 + (unsigned int)(srow_49 * 128) + ((strip_chunk1_54 ^ sswizzle_50) << 4); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols_51[8]), "=r"(cols_51[9]), "=r"(cols_51[10]), "=r"(cols_51[11]) + : "r"(strip_src1_55) + : "memory"); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(cols_51[12]), "=r"(cols_51[13]), "=r"(cols_51[14]), "=r"(cols_51[15]) + : "r"(strip_src1_55 + 1024) + : "memory"); + __nv_bfloat162 amax_a_56 = __as_bf16x2(0); + __nv_bfloat162 amax_b_57 = __as_bf16x2(0); + __nv_bfloat162 _hxa_219 = amax_a_56; + __nv_bfloat162 _hxb_219 = __as_bf16x2(cols_51[0]); + unsigned int _hxr_219; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_219) : "r"(*reinterpret_cast(&_hxa_219)), "r"(*reinterpret_cast(&_hxb_219))); + __nv_bfloat162 _max_218 = *reinterpret_cast(&_hxr_219); + amax_a_56 = _max_218; + __nv_bfloat162 _hxa_221 = amax_b_57; + __nv_bfloat162 _hxb_221 = __as_bf16x2(cols_51[8]); + unsigned int _hxr_221; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_221) : "r"(*reinterpret_cast(&_hxa_221)), "r"(*reinterpret_cast(&_hxb_221))); + __nv_bfloat162 _max_220 = *reinterpret_cast(&_hxr_221); + amax_b_57 = _max_220; + __nv_bfloat162 _hxa_223 = amax_a_56; + __nv_bfloat162 _hxb_223 = __as_bf16x2(cols_51[1]); + unsigned int _hxr_223; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_223) : "r"(*reinterpret_cast(&_hxa_223)), "r"(*reinterpret_cast(&_hxb_223))); + __nv_bfloat162 _max_222 = *reinterpret_cast(&_hxr_223); + amax_a_56 = _max_222; + __nv_bfloat162 _hxa_225 = amax_b_57; + __nv_bfloat162 _hxb_225 = __as_bf16x2(cols_51[9]); + unsigned int _hxr_225; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_225) : "r"(*reinterpret_cast(&_hxa_225)), "r"(*reinterpret_cast(&_hxb_225))); + __nv_bfloat162 _max_224 = *reinterpret_cast(&_hxr_225); + amax_b_57 = _max_224; + __nv_bfloat162 _hxa_227 = amax_a_56; + __nv_bfloat162 _hxb_227 = __as_bf16x2(cols_51[2]); + unsigned int _hxr_227; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_227) : "r"(*reinterpret_cast(&_hxa_227)), "r"(*reinterpret_cast(&_hxb_227))); + __nv_bfloat162 _max_226 = *reinterpret_cast(&_hxr_227); + amax_a_56 = _max_226; + __nv_bfloat162 _hxa_229 = amax_b_57; + __nv_bfloat162 _hxb_229 = __as_bf16x2(cols_51[10]); + unsigned int _hxr_229; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_229) : "r"(*reinterpret_cast(&_hxa_229)), "r"(*reinterpret_cast(&_hxb_229))); + __nv_bfloat162 _max_228 = *reinterpret_cast(&_hxr_229); + amax_b_57 = _max_228; + __nv_bfloat162 _hxa_231 = amax_a_56; + __nv_bfloat162 _hxb_231 = __as_bf16x2(cols_51[3]); + unsigned int _hxr_231; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_231) : "r"(*reinterpret_cast(&_hxa_231)), "r"(*reinterpret_cast(&_hxb_231))); + __nv_bfloat162 _max_230 = *reinterpret_cast(&_hxr_231); + amax_a_56 = _max_230; + __nv_bfloat162 _hxa_233 = amax_b_57; + __nv_bfloat162 _hxb_233 = __as_bf16x2(cols_51[11]); + unsigned int _hxr_233; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_233) : "r"(*reinterpret_cast(&_hxa_233)), "r"(*reinterpret_cast(&_hxb_233))); + __nv_bfloat162 _max_232 = *reinterpret_cast(&_hxr_233); + amax_b_57 = _max_232; + __nv_bfloat162 _hxa_235 = amax_a_56; + __nv_bfloat162 _hxb_235 = __as_bf16x2(cols_51[4]); + unsigned int _hxr_235; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_235) : "r"(*reinterpret_cast(&_hxa_235)), "r"(*reinterpret_cast(&_hxb_235))); + __nv_bfloat162 _max_234 = *reinterpret_cast(&_hxr_235); + amax_a_56 = _max_234; + __nv_bfloat162 _hxa_237 = amax_b_57; + __nv_bfloat162 _hxb_237 = __as_bf16x2(cols_51[12]); + unsigned int _hxr_237; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_237) : "r"(*reinterpret_cast(&_hxa_237)), "r"(*reinterpret_cast(&_hxb_237))); + __nv_bfloat162 _max_236 = *reinterpret_cast(&_hxr_237); + amax_b_57 = _max_236; + __nv_bfloat162 _hxa_239 = amax_a_56; + __nv_bfloat162 _hxb_239 = __as_bf16x2(cols_51[5]); + unsigned int _hxr_239; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_239) : "r"(*reinterpret_cast(&_hxa_239)), "r"(*reinterpret_cast(&_hxb_239))); + __nv_bfloat162 _max_238 = *reinterpret_cast(&_hxr_239); + amax_a_56 = _max_238; + __nv_bfloat162 _hxa_241 = amax_b_57; + __nv_bfloat162 _hxb_241 = __as_bf16x2(cols_51[13]); + unsigned int _hxr_241; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_241) : "r"(*reinterpret_cast(&_hxa_241)), "r"(*reinterpret_cast(&_hxb_241))); + __nv_bfloat162 _max_240 = *reinterpret_cast(&_hxr_241); + amax_b_57 = _max_240; + __nv_bfloat162 _hxa_243 = amax_a_56; + __nv_bfloat162 _hxb_243 = __as_bf16x2(cols_51[6]); + unsigned int _hxr_243; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_243) : "r"(*reinterpret_cast(&_hxa_243)), "r"(*reinterpret_cast(&_hxb_243))); + __nv_bfloat162 _max_242 = *reinterpret_cast(&_hxr_243); + amax_a_56 = _max_242; + __nv_bfloat162 _hxa_245 = amax_b_57; + __nv_bfloat162 _hxb_245 = __as_bf16x2(cols_51[14]); + unsigned int _hxr_245; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_245) : "r"(*reinterpret_cast(&_hxa_245)), "r"(*reinterpret_cast(&_hxb_245))); + __nv_bfloat162 _max_244 = *reinterpret_cast(&_hxr_245); + amax_b_57 = _max_244; + __nv_bfloat162 _hxa_247 = amax_a_56; + __nv_bfloat162 _hxb_247 = __as_bf16x2(cols_51[7]); + unsigned int _hxr_247; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_247) : "r"(*reinterpret_cast(&_hxa_247)), "r"(*reinterpret_cast(&_hxb_247))); + __nv_bfloat162 _max_246 = *reinterpret_cast(&_hxr_247); + amax_a_56 = _max_246; + __nv_bfloat162 _hxa_249 = amax_b_57; + __nv_bfloat162 _hxb_249 = __as_bf16x2(cols_51[15]); + unsigned int _hxr_249; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_249) : "r"(*reinterpret_cast(&_hxa_249)), "r"(*reinterpret_cast(&_hxb_249))); + __nv_bfloat162 _max_248 = *reinterpret_cast(&_hxr_249); + amax_b_57 = _max_248; + unsigned int mag_a_58 = __as_u32(amax_a_56) & 2147450879; + unsigned int mag_b_59 = __as_u32(amax_b_57) & 2147450879; + unsigned int lows_ab_60 = mag_a_58 & 65535 | mag_b_59 << 16; + unsigned int highs_ab_61 = mag_a_58 >> 16 | mag_b_59 & 4294901760u; + __nv_bfloat162 col_max_62 = __hmax2_nan(__as_bf16x2(lows_ab_60), __as_bf16x2(highs_ab_61)); + unsigned int peer1_63 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(col_max_62), 1); + col_max_62 = __hmax2_nan(col_max_62, __as_bf16x2(peer1_63)); + unsigned int col_max_bits_64 = __as_u32(col_max_62); + unsigned int amax_bits_65 = (col_max_bits_64 & 65535) << 16; + int scale_i32_66 = (int)(amax_bits_65 + 2031616 >> 23) - 8; + if (scale_i32_66 < 0) { + scale_i32_66 = 0; + } + if ((amax_bits_65 & 2139095040) == 2139095040) { + scale_i32_66 = 255; + } + unsigned int scale_67 = (unsigned int)scale_i32_66; + unsigned int inv_68 = 254 - scale_67 << 7; + if (scale_67 == 255) { + inv_68 = 32704; + } + unsigned int inv2_69 = inv_68 | inv_68 << 16; + unsigned int amax_bits_70 = (col_max_bits_64 >> 16 & 65535) << 16; + int scale_i32_71 = (int)(amax_bits_70 + 2031616 >> 23) - 8; + if (scale_i32_71 < 0) { + scale_i32_71 = 0; + } + if ((amax_bits_70 & 2139095040) == 2139095040) { + scale_i32_71 = 255; + } + unsigned int scale_72 = (unsigned int)scale_i32_71; + unsigned int inv_73 = 254 - scale_72 << 7; + if (scale_72 == 255) { + inv_73 = 32704; + } + unsigned int inv2_74 = inv_73 | inv_73 << 16; + int gcol_a_75 = bx * 64 + K + warp_47 * 16 + (lane_46 >> 2); + int gcol_b_76 = gcol_a_75 + 8; + int out_row_77 = by * 64 + (lane_46 & 3) * 16; + unsigned int words_a_78[4]; + unsigned int words_b_79[4]; + uint32_t _bf16x2_mul_48; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_48) : "r"(cols_51[0]), "r"(inv2_69)); + uint16_t _e4m3x2_bf16x2_24; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_24) : "r"(_bf16x2_mul_48)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_48); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_24) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_49; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_49) : "r"(cols_51[1]), "r"(inv2_69)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_251; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_251) : "r"(_bf16x2_mul_49)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_49); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_251) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_250; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_250) : "h"(_e4m3x2_bf16x2_24), "h"(_cvt_satfinite_e4m3x2_bf16x2_251)); + words_a_78[0] = _mov_b32_pack_u16x2_250; + uint32_t _bf16x2_mul_50; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_50) : "r"(cols_51[8]), "r"(inv2_74)); + uint16_t _e4m3x2_bf16x2_25; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_25) : "r"(_bf16x2_mul_50)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_50); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_25) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_51; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_51) : "r"(cols_51[9]), "r"(inv2_74)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_253; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_253) : "r"(_bf16x2_mul_51)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_51); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_253) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_252; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_252) : "h"(_e4m3x2_bf16x2_25), "h"(_cvt_satfinite_e4m3x2_bf16x2_253)); + words_b_79[0] = _mov_b32_pack_u16x2_252; + uint32_t _bf16x2_mul_52; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_52) : "r"(cols_51[2]), "r"(inv2_69)); + uint16_t _e4m3x2_bf16x2_26; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_26) : "r"(_bf16x2_mul_52)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_52); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_26) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_53; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_53) : "r"(cols_51[3]), "r"(inv2_69)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_255; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_255) : "r"(_bf16x2_mul_53)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_53); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_255) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_254; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_254) : "h"(_e4m3x2_bf16x2_26), "h"(_cvt_satfinite_e4m3x2_bf16x2_255)); + words_a_78[1] = _mov_b32_pack_u16x2_254; + uint32_t _bf16x2_mul_54; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_54) : "r"(cols_51[10]), "r"(inv2_74)); + uint16_t _e4m3x2_bf16x2_27; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_27) : "r"(_bf16x2_mul_54)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_54); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_27) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_55; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_55) : "r"(cols_51[11]), "r"(inv2_74)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_257; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_257) : "r"(_bf16x2_mul_55)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_55); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_257) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_256; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_256) : "h"(_e4m3x2_bf16x2_27), "h"(_cvt_satfinite_e4m3x2_bf16x2_257)); + words_b_79[1] = _mov_b32_pack_u16x2_256; + uint32_t _bf16x2_mul_56; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_56) : "r"(cols_51[4]), "r"(inv2_69)); + uint16_t _e4m3x2_bf16x2_28; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_28) : "r"(_bf16x2_mul_56)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_56); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_28) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_57; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_57) : "r"(cols_51[5]), "r"(inv2_69)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_259; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_259) : "r"(_bf16x2_mul_57)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_57); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_259) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_258; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_258) : "h"(_e4m3x2_bf16x2_28), "h"(_cvt_satfinite_e4m3x2_bf16x2_259)); + words_a_78[2] = _mov_b32_pack_u16x2_258; + uint32_t _bf16x2_mul_58; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_58) : "r"(cols_51[12]), "r"(inv2_74)); + uint16_t _e4m3x2_bf16x2_29; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_29) : "r"(_bf16x2_mul_58)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_58); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_29) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_59; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_59) : "r"(cols_51[13]), "r"(inv2_74)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_261; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_261) : "r"(_bf16x2_mul_59)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_59); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_261) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_260; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_260) : "h"(_e4m3x2_bf16x2_29), "h"(_cvt_satfinite_e4m3x2_bf16x2_261)); + words_b_79[2] = _mov_b32_pack_u16x2_260; + uint32_t _bf16x2_mul_60; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_60) : "r"(cols_51[6]), "r"(inv2_69)); + uint16_t _e4m3x2_bf16x2_30; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_30) : "r"(_bf16x2_mul_60)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_60); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_30) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_61; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_61) : "r"(cols_51[7]), "r"(inv2_69)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_263; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_263) : "r"(_bf16x2_mul_61)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_61); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_263) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_262; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_262) : "h"(_e4m3x2_bf16x2_30), "h"(_cvt_satfinite_e4m3x2_bf16x2_263)); + words_a_78[3] = _mov_b32_pack_u16x2_262; + uint32_t _bf16x2_mul_62; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_62) : "r"(cols_51[14]), "r"(inv2_74)); + uint16_t _e4m3x2_bf16x2_31; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_31) : "r"(_bf16x2_mul_62)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_62); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_31) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_63; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_63) : "r"(cols_51[15]), "r"(inv2_74)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_265; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_265) : "r"(_bf16x2_mul_63)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_63); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_265) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_264; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_264) : "h"(_e4m3x2_bf16x2_31), "h"(_cvt_satfinite_e4m3x2_bf16x2_265)); + words_b_79[3] = _mov_b32_pack_u16x2_264; + reinterpret_cast(col_output + (gcol_a_75 * M + out_row_77))[0] = reinterpret_cast(words_a_78 + 0)[0]; + reinterpret_cast(col_output + (gcol_b_76 * M + out_row_77))[0] = reinterpret_cast(words_b_79 + 0)[0]; + if ((lane_46 & 1) == 0) { + int row_tiles_1 = M >> 7; + int rblk_1 = by * 2 + ((lane_46 & 3) >> 1); + int col_scale_a_1 = ((gcol_a_75 >> 7) * row_tiles_1 + (rblk_1 >> 2)) * 512 + (gcol_a_75 & 31) * 16 + (gcol_a_75 >> 5 & 3) * 4 + (rblk_1 & 3); + int col_scale_b_1 = ((gcol_b_76 >> 7) * row_tiles_1 + (rblk_1 >> 2)) * 512 + (gcol_b_76 & 31) * 16 + (gcol_b_76 >> 5 & 3) * 4 + (rblk_1 & 3); + *(reinterpret_cast(col_scales + col_scale_a_1) + (0)) = (unsigned char)(scale_67); + *(reinterpret_cast(col_scales + col_scale_b_1) + (0)) = (unsigned char)(scale_72); + } +} + +} // extern "C" diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_binding.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_binding.cu new file mode 100644 index 00000000000..a8d0af82193 --- /dev/null +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_binding.cu @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2026 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gated_act_mxfp8_54573bbcd9b4aadbd39f, for the device source that defines it. Do not edit. +#include +#include + +#include "cake_ffi_checks.h" + +#include + +extern "C" __global__ void kernel_cake_gated_act_mxfp8_54573bbcd9b4aadbd39f(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K); + +namespace cake_binding_kernel_cake_gated_act_mxfp8_54573bbcd9b4aadbd39f { + +using tvm::ffi::TensorView; + +constexpr const char* KERNEL_NAME = "kernel_cake_gated_act_mxfp8_54573bbcd9b4aadbd39f"; + +void CheckCuda(cudaError_t status, const char* what) { + TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) + << KERNEL_NAME << ": " << what << " failed: " << cudaGetErrorString(status); +} + +void* DataPtr(const TensorView& t) { return static_cast(t.data_ptr()) + t.byte_offset(); } + +void Run(TensorView a_gated_input, TensorView a_grad, TensorView a_row_output, TensorView a_col_output, TensorView a_row_scales, TensorView a_col_scales, int64_t a_M, int64_t a_K, int64_t grid_x, int64_t grid_y, int64_t grid_z) { + DLDevice dev = a_gated_input.device(); + tvm::ffi::CUDADeviceGuard device_guard(dev.device_id); + check_cuda_tensor(a_gated_input, "gated_input"); + check_dtype(a_gated_input, DLDataType{kDLBfloat, 16, 1}, "gated_input"); + check_cuda_tensor(a_grad, "grad"); + check_dtype(a_grad, DLDataType{kDLBfloat, 16, 1}, "grad"); + check_cuda_tensor(a_row_output, "row_output"); + check_dtype(a_row_output, DLDataType{kDLUInt, 8, 1}, "row_output"); + check_cuda_tensor(a_col_output, "col_output"); + check_dtype(a_col_output, DLDataType{kDLUInt, 8, 1}, "col_output"); + check_cuda_tensor(a_row_scales, "row_scales"); + check_dtype(a_row_scales, DLDataType{kDLUInt, 8, 1}, "row_scales"); + check_cuda_tensor(a_col_scales, "col_scales"); + check_dtype(a_col_scales, DLDataType{kDLUInt, 8, 1}, "col_scales"); + TVM_FFI_CHECK(a_M >= -2147483648LL && a_M <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'M' = " << a_M + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_K >= -2147483648LL && a_K <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'K' = " << a_K + << " is outside the i32 range [-2147483648, 2147483647]"; + check_contiguous(a_gated_input, "gated_input"); + check_contiguous(a_grad, "grad"); + check_same_device(a_grad, a_gated_input, "grad", "gated_input"); + check_contiguous(a_row_output, "row_output"); + check_same_device(a_row_output, a_gated_input, "row_output", "gated_input"); + check_contiguous(a_col_output, "col_output"); + check_same_device(a_col_output, a_gated_input, "col_output", "gated_input"); + check_contiguous(a_row_scales, "row_scales"); + check_same_device(a_row_scales, a_gated_input, "row_scales", "gated_input"); + check_contiguous(a_col_scales, "col_scales"); + check_same_device(a_col_scales, a_gated_input, "col_scales", "gated_input"); + TVM_FFI_CHECK(grid_x > 0 && grid_y > 0 && grid_z > 0, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") must be positive"; + TVM_FFI_CHECK(grid_x <= 2147483647 && grid_y <= 65535 && grid_z <= 65535, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") exceeds CUDA's limits"; + cudaStream_t stream = static_cast(TVMFFIEnvGetStream(dev.device_type, dev.device_id)); + void* v_gated_input = DataPtr(a_gated_input); + void* v_grad = DataPtr(a_grad); + void* v_row_output = DataPtr(a_row_output); + void* v_col_output = DataPtr(a_col_output); + void* v_row_scales = DataPtr(a_row_scales); + void* v_col_scales = DataPtr(a_col_scales); + int32_t v_M = static_cast(a_M); + int32_t v_K = static_cast(a_K); + void* kernel_args[] = {&v_gated_input, &v_grad, &v_row_output, &v_col_output, &v_row_scales, &v_col_scales, &v_M, &v_K}; + cudaLaunchConfig_t config{}; + config.gridDim = dim3(static_cast(grid_x), static_cast(grid_y), static_cast(grid_z)); + config.blockDim = dim3(128, 1, 1); + config.dynamicSmemBytes = 0; + config.stream = stream; + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gated_act_mxfp8_54573bbcd9b4aadbd39f), kernel_args), + "cudaLaunchKernelExC"); +} + +} // namespace cake_binding_kernel_cake_gated_act_mxfp8_54573bbcd9b4aadbd39f + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gated_act_mxfp8_54573bbcd9b4aadbd39f::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_kernel.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_kernel.cu new file mode 100644 index 00000000000..ca440228d82 --- /dev/null +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_54573bbcd9b4aadbd39f_kernel.cu @@ -0,0 +1,891 @@ +/* + * Copyright (c) 2026 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +#if defined(__CUDACC_RTC__) +typedef unsigned long long uint64_t; +#else +typedef unsigned long uint64_t; +#endif +static_assert(sizeof(uint64_t) == 8, "Cake requires an LP64 CUDA host ABI"); +typedef signed int int32_t; +typedef short int int16_t; +struct __align__(64) CakeTensorMap64 { uint64_t opaque[16]; }; +static_assert(sizeof(CakeTensorMap64) == 128, "64-aligned tensor-map ABI size"); +static_assert(alignof(CakeTensorMap64) == 64, "64-aligned tensor-map ABI alignment"); + +#if defined(__CUDACC_RTC__) +typedef struct __align__(128) { uint64_t opaque[16]; } CUtensorMap; +#else +#include +#endif + +static_assert(sizeof(CUtensorMap) == 128, "CUtensorMap CUDA ABI must be 128 bytes"); +#include +#include + +__device__ __forceinline__ int make_warp_uniform(int x) { + int result; + asm volatile("shfl.sync.idx.b32 %0, %1, 0, 0x1F, 0xFFFFFFFF;" + : "=r"(result) : "r"(x)); + return result; +} + +#define CAKE_INF CUDART_INF_F +#define NUM_MAIN_STAGES 1 +#define THREADS 128 +#define LAUNCH_MIN_BLOCKS 12 + +#include + +__device__ __forceinline__ float approx_exp2(float x) { + float y; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + +__device__ __forceinline__ float max_noftz(float a, float b) { + float c; + asm("max.f32 %0, %1, %2;" : "=f"(c) : "f"(a), "f"(b)); + return c; +} + + +__device__ __forceinline__ float2 sub_f32x2_noftz(float2 a, float2 b) { + float2 r; + asm("sub.f32x2 %0, %1, %2;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(unsigned long long*)&a), "l"(*(unsigned long long*)&b)); + return r; +} + +__device__ __forceinline__ float2 mul_f32x2_noftz(float2 a, float2 b) { + float2 r; + asm("mul.rn.f32x2 %0, %1, %2;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(unsigned long long*)&a), "l"(*(unsigned long long*)&b)); + return r; +} + + +// ex2_emulation_f32x2 defined in softmax_frag_exp2_cast helper (or standalone) + +__device__ __forceinline__ float2 fma_f32x2_rn_noftz(float2 a, float2 b, float2 c) { + float2 r; + asm("fma.rn.f32x2 %0, %1, %2, %3;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(const unsigned long long*)&a), + "l"(*(const unsigned long long*)&b), + "l"(*(const unsigned long long*)&c)); + return r; +} + + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + +__device__ __forceinline__ __nv_bfloat162 __as_bf16x2(unsigned int v) { + __nv_bfloat162_raw raw; + raw.x = static_cast(v); + raw.y = static_cast(v >> 16); + return __nv_bfloat162(raw); +} + +__device__ __forceinline__ float2 cake_bf16x2_to_f32x2(__nv_bfloat162 v) { + const unsigned int w = *reinterpret_cast(&v); + return make_float2(__uint_as_float(w << 16), __uint_as_float(w & 0xffff0000u)); +} + +__device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { + float r; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} + +extern "C" { + +__global__ __launch_bounds__(128, LAUNCH_MIN_BLOCKS) void +kernel_cake_gated_act_mxfp8_54573bbcd9b4aadbd39f(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K) +{ + const int tid = threadIdx.x; + const int warp = make_warp_uniform(tid / 32); + const int lane = tid % 32; + + + const int bid = blockIdx.x; + const int num_bids = gridDim.x; + + const int cta_rank = 0; + + // === Task calls (declaration order) === + int quarter = tid & 3; + int row = tid >> 2; + int bx = blockIdx.x; + int by = blockIdx.y; + int grow = by * 32 + row; + int col = bx * 32 + quarter * 8; + int gate_index = grow * (2 * K) + col; + unsigned int _vec_load_0[4]; + { + uint32_t _uv4_0_0; + uint32_t _uv4_0_1; + uint32_t _uv4_0_2; + uint32_t _uv4_0_3; + asm volatile("ld.global.L2::cache_hint.v4.b32 {%0, %1, %2, %3}, [%4], %5;" + : "=r"(_uv4_0_0), "=r"(_uv4_0_1), "=r"(_uv4_0_2), "=r"(_uv4_0_3) : "l"((const void*)(gated_input + gate_index + (0))), "l"(0x12F0000000000000ULL) : "memory"); + _vec_load_0[0 + 0] = _uv4_0_0; + _vec_load_0[0 + 1] = _uv4_0_1; + _vec_load_0[0 + 2] = _uv4_0_2; + _vec_load_0[0 + 3] = _uv4_0_3; + } + unsigned int _vec_load_1[4]; + { + uint32_t _uv4_1_0; + uint32_t _uv4_1_1; + uint32_t _uv4_1_2; + uint32_t _uv4_1_3; + asm volatile("ld.global.L2::cache_hint.v4.b32 {%0, %1, %2, %3}, [%4], %5;" + : "=r"(_uv4_1_0), "=r"(_uv4_1_1), "=r"(_uv4_1_2), "=r"(_uv4_1_3) : "l"((const void*)(gated_input + (gate_index + K) + (0))), "l"(0x12F0000000000000ULL) : "memory"); + _vec_load_1[0 + 0] = _uv4_1_0; + _vec_load_1[0 + 1] = _uv4_1_1; + _vec_load_1[0 + 2] = _uv4_1_2; + _vec_load_1[0 + 3] = _uv4_1_3; + } + unsigned int _vec_load_2[4]; + { + uint32_t _uv4_2_0; + uint32_t _uv4_2_1; + uint32_t _uv4_2_2; + uint32_t _uv4_2_3; + asm volatile("ld.global.L2::cache_hint.v4.b32 {%0, %1, %2, %3}, [%4], %5;" + : "=r"(_uv4_2_0), "=r"(_uv4_2_1), "=r"(_uv4_2_2), "=r"(_uv4_2_3) : "l"((const void*)(grad + (grow * K + col) + (0))), "l"(0x12F0000000000000ULL) : "memory"); + _vec_load_2[0 + 0] = _uv4_2_0; + _vec_load_2[0 + 1] = _uv4_2_1; + _vec_load_2[0 + 2] = _uv4_2_2; + _vec_load_2[0 + 3] = _uv4_2_3; + } + unsigned int dgate[4]; + unsigned int dup[4]; + __nv_bfloat162 lowest = __as_bf16x2(_vec_load_0[0]); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[1])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[2])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_0[3])); + float2 _cvt_f32_0 = cake_bf16x2_to_f32x2(lowest); + float _min_4; + asm("min.NaN.f32 %0, %1, %2;" : "=f"(_min_4) : "f"(_cvt_f32_0.x), "f"(_cvt_f32_0.y)); + int _warp_vote_3 = __all_sync(0xFFFFFFFF, _min_4 > -87.0f); + if (_warp_vote_3 != 0) { + float2 _cvt_f32_1 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[0])); + float2 _cvt_f32_2 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[0])); + float2 _cvt_f32_3 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[0])); + float e0_1 = approx_exp2((-_cvt_f32_1.x) * 1.442695f); + float e1_1 = approx_exp2((-_cvt_f32_1.y) * 1.442695f); + float d0_1 = 1.0f + e0_1; + float d1_1 = 1.0f + e1_1; + float r0_1 = approx_rcp(d0_1); + float r1_1 = approx_rcp(d1_1); + float _fma_6; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_6) : "f"(-d0_1), "f"(r0_1), "f"(1.0f)); + float _fma_5; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_5) : "f"(r0_1), "f"(_fma_6), "f"(r0_1)); + float _fma_8; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_8) : "f"(-d1_1), "f"(r1_1), "f"(1.0f)); + float _fma_7; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_7) : "f"(r1_1), "f"(_fma_8), "f"(r1_1)); + float2 s_0 = make_float2(_fma_5, _fma_7); + float2 act_0; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_0) : "l"(*(const unsigned long long*)&_cvt_f32_1), "l"(*(const unsigned long long*)&s_0)); + float2 _f32x2_mul_a_9 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_0, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_0), s_0), _cvt_f32_3); + float2 _mul_0; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_9), "l"(*(const unsigned long long*)&_cvt_f32_2)); + float2 _mul_1; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_1) : "l"(*(const unsigned long long*)&act_0), "l"(*(const unsigned long long*)&_cvt_f32_3)); + __nv_bfloat162 _bf16x2_0 = __float22bfloat162_rn(make_float2(_mul_0.x, _mul_0.y)); + __nv_bfloat162 _bf16x2_1 = __float22bfloat162_rn(make_float2(_mul_1.x, _mul_1.y)); + dgate[0] = __as_u32(_bf16x2_0); + dup[0] = __as_u32(_bf16x2_1); + float2 _cvt_f32_4 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[1])); + float2 _cvt_f32_5 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[1])); + float2 _cvt_f32_6 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[1])); + float e0_3 = approx_exp2((-_cvt_f32_4.x) * 1.442695f); + float e1_3 = approx_exp2((-_cvt_f32_4.y) * 1.442695f); + float d0_3 = 1.0f + e0_3; + float d1_3 = 1.0f + e1_3; + float r0_3 = approx_rcp(d0_3); + float r1_3 = approx_rcp(d1_3); + float _fma_11; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_11) : "f"(-d0_3), "f"(r0_3), "f"(1.0f)); + float _fma_10; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_10) : "f"(r0_3), "f"(_fma_11), "f"(r0_3)); + float _fma_13; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_13) : "f"(-d1_3), "f"(r1_3), "f"(1.0f)); + float _fma_12; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_12) : "f"(r1_3), "f"(_fma_13), "f"(r1_3)); + float2 s_2 = make_float2(_fma_10, _fma_12); + float2 act_2; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_2) : "l"(*(const unsigned long long*)&_cvt_f32_4), "l"(*(const unsigned long long*)&s_2)); + float2 _f32x2_mul_a_14 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_2, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_2), s_2), _cvt_f32_6); + float2 _mul_2; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_14), "l"(*(const unsigned long long*)&_cvt_f32_5)); + float2 _mul_3; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_3) : "l"(*(const unsigned long long*)&act_2), "l"(*(const unsigned long long*)&_cvt_f32_6)); + __nv_bfloat162 _bf16x2_2 = __float22bfloat162_rn(make_float2(_mul_2.x, _mul_2.y)); + __nv_bfloat162 _bf16x2_3 = __float22bfloat162_rn(make_float2(_mul_3.x, _mul_3.y)); + dgate[1] = __as_u32(_bf16x2_2); + dup[1] = __as_u32(_bf16x2_3); + float2 _cvt_f32_7 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[2])); + float2 _cvt_f32_8 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[2])); + float2 _cvt_f32_9 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[2])); + float e0_5 = approx_exp2((-_cvt_f32_7.x) * 1.442695f); + float e1_5 = approx_exp2((-_cvt_f32_7.y) * 1.442695f); + float d0_5 = 1.0f + e0_5; + float d1_5 = 1.0f + e1_5; + float r0_5 = approx_rcp(d0_5); + float r1_5 = approx_rcp(d1_5); + float _fma_16; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_16) : "f"(-d0_5), "f"(r0_5), "f"(1.0f)); + float _fma_15; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_15) : "f"(r0_5), "f"(_fma_16), "f"(r0_5)); + float _fma_18; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_18) : "f"(-d1_5), "f"(r1_5), "f"(1.0f)); + float _fma_17; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_17) : "f"(r1_5), "f"(_fma_18), "f"(r1_5)); + float2 s_4 = make_float2(_fma_15, _fma_17); + float2 act_4; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_4) : "l"(*(const unsigned long long*)&_cvt_f32_7), "l"(*(const unsigned long long*)&s_4)); + float2 _f32x2_mul_a_19 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_4, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_4), s_4), _cvt_f32_9); + float2 _mul_4; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_19), "l"(*(const unsigned long long*)&_cvt_f32_8)); + float2 _mul_5; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_5) : "l"(*(const unsigned long long*)&act_4), "l"(*(const unsigned long long*)&_cvt_f32_9)); + __nv_bfloat162 _bf16x2_4 = __float22bfloat162_rn(make_float2(_mul_4.x, _mul_4.y)); + __nv_bfloat162 _bf16x2_5 = __float22bfloat162_rn(make_float2(_mul_5.x, _mul_5.y)); + dgate[2] = __as_u32(_bf16x2_4); + dup[2] = __as_u32(_bf16x2_5); + float2 _cvt_f32_10 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[3])); + float2 _cvt_f32_11 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[3])); + float2 _cvt_f32_12 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[3])); + float e0_7 = approx_exp2((-_cvt_f32_10.x) * 1.442695f); + float e1_7 = approx_exp2((-_cvt_f32_10.y) * 1.442695f); + float d0_7 = 1.0f + e0_7; + float d1_7 = 1.0f + e1_7; + float r0_7 = approx_rcp(d0_7); + float r1_7 = approx_rcp(d1_7); + float _fma_21; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_21) : "f"(-d0_7), "f"(r0_7), "f"(1.0f)); + float _fma_20; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_20) : "f"(r0_7), "f"(_fma_21), "f"(r0_7)); + float _fma_23; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_23) : "f"(-d1_7), "f"(r1_7), "f"(1.0f)); + float _fma_22; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_22) : "f"(r1_7), "f"(_fma_23), "f"(r1_7)); + float2 s_6 = make_float2(_fma_20, _fma_22); + float2 act_6; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_6) : "l"(*(const unsigned long long*)&_cvt_f32_10), "l"(*(const unsigned long long*)&s_6)); + float2 _f32x2_mul_a_24 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_6, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_6), s_6), _cvt_f32_12); + float2 _mul_6; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_24), "l"(*(const unsigned long long*)&_cvt_f32_11)); + float2 _mul_7; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_7) : "l"(*(const unsigned long long*)&act_6), "l"(*(const unsigned long long*)&_cvt_f32_12)); + __nv_bfloat162 _bf16x2_6 = __float22bfloat162_rn(make_float2(_mul_6.x, _mul_6.y)); + __nv_bfloat162 _bf16x2_7 = __float22bfloat162_rn(make_float2(_mul_7.x, _mul_7.y)); + dgate[3] = __as_u32(_bf16x2_6); + dup[3] = __as_u32(_bf16x2_7); + } else { + int band = 0; + float2 _cvt_f32_13 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[0])); + float2 _cvt_f32_14 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[0])); + float2 _cvt_f32_15 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[0])); + float e0_9 = approx_exp2((-_cvt_f32_13.x) * 1.442695f); + float e1_9 = approx_exp2((-_cvt_f32_13.y) * 1.442695f); + float d0_9 = 1.0f + e0_9; + float d1_9 = 1.0f + e1_9; + float r0_9 = approx_rcp(d0_9); + float r1_9 = approx_rcp(d1_9); + float _fma_25; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_25) : "f"(-d0_9), "f"(r0_9), "f"(1.0f)); + float _fma_0; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_0) : "f"(r0_9), "f"(_fma_25), "f"(r0_9)); + float _fma_26; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_26) : "f"(-d1_9), "f"(r1_9), "f"(1.0f)); + float _fma_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_1) : "f"(r1_9), "f"(_fma_26), "f"(r1_9)); + float t0_9 = _fma_0; + float t1_9 = _fma_1; + int _warp_vote_27 = __any_sync(0xFFFFFFFF, fminf(t0_9, t1_9) == 0.0f); + if (_warp_vote_27 != 0) { + float d4_9a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_9a) : "f"(d0_9), "f"(0.25f)); + float r4_9a = approx_rcp(d4_9a); + float _fma_29; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_29) : "f"(-d4_9a), "f"(r4_9a), "f"(1.0f)); + float _fma_28; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_28) : "f"(r4_9a), "f"(_fma_29), "f"(r4_9a)); + float big_9a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_9a) : "f"(mul_f32_rn_noftz(_fma_28, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_30; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_30) : "f"(big_9a), "f"(8.388608e+06f)); + float whole_9a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_9a) : "f"(_add_30), "f"(8.388608e+06f)); + float above_9a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_9a) : "f"(-d4_9a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_9a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_32; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_32) : "f"(big_9a), "f"(whole_9a)); + float _abs_31; + asm("abs.f32 %0, %1;" : "=f"(_abs_31) : "f"(_sub_32)); + float _add_33; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_33) : "f"(big_9a), "f"(0.5f)); + float _sub_34; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_34) : "f"(big_9a), "f"(0.5f)); + float rounded_9a = ((_abs_31 == 0.5f) ? ((above_9a > 0.0f) ? _add_33 : _sub_34) : whole_9a); + t0_9 = ((d0_9 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_9a, 7.888609e-31f), 1.7763568e-15f) : t0_9); + float d4_9b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_9b) : "f"(d1_9), "f"(0.25f)); + float r4_9b = approx_rcp(d4_9b); + float _fma_36; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_36) : "f"(-d4_9b), "f"(r4_9b), "f"(1.0f)); + float _fma_35; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_35) : "f"(r4_9b), "f"(_fma_36), "f"(r4_9b)); + float big_9b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_9b) : "f"(mul_f32_rn_noftz(_fma_35, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_37; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_37) : "f"(big_9b), "f"(8.388608e+06f)); + float whole_9b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_9b) : "f"(_add_37), "f"(8.388608e+06f)); + float above_9b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_9b) : "f"(-d4_9b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_9b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_39; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_39) : "f"(big_9b), "f"(whole_9b)); + float _abs_38; + asm("abs.f32 %0, %1;" : "=f"(_abs_38) : "f"(_sub_39)); + float _add_40; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_40) : "f"(big_9b), "f"(0.5f)); + float _sub_41; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_41) : "f"(big_9b), "f"(0.5f)); + float rounded_9b = ((_abs_38 == 0.5f) ? ((above_9b > 0.0f) ? _add_40 : _sub_41) : whole_9b); + t1_9 = ((d1_9 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_9b, 7.888609e-31f), 1.7763568e-15f) : t1_9); + } + float2 s_8 = make_float2(max_noftz(t0_9, 0.0f), max_noftz(t1_9, 0.0f)); + float2 act_8; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_8) : "l"(*(const unsigned long long*)&_cvt_f32_13), "l"(*(const unsigned long long*)&s_8)); + float2 _f32x2_mul_a_42 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_8, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_8), s_8), _cvt_f32_15); + float2 _mul_8; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_42), "l"(*(const unsigned long long*)&_cvt_f32_14)); + float2 _mul_9; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_9) : "l"(*(const unsigned long long*)&act_8), "l"(*(const unsigned long long*)&_cvt_f32_15)); + __nv_bfloat162 _bf16x2_8 = __float22bfloat162_rn(make_float2(_mul_8.x, _mul_8.y)); + __nv_bfloat162 _bf16x2_9 = __float22bfloat162_rn(make_float2(_mul_9.x, _mul_9.y)); + dgate[0] = __as_u32(_bf16x2_8); + dup[0] = __as_u32(_bf16x2_9); + float2 _cvt_f32_16 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[1])); + float2 _cvt_f32_17 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[1])); + float2 _cvt_f32_18 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[1])); + float e0_11 = approx_exp2((-_cvt_f32_16.x) * 1.442695f); + float e1_11 = approx_exp2((-_cvt_f32_16.y) * 1.442695f); + float d0_11 = 1.0f + e0_11; + float d1_11 = 1.0f + e1_11; + float r0_11 = approx_rcp(d0_11); + float r1_11 = approx_rcp(d1_11); + float _fma_43; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_43) : "f"(-d0_11), "f"(r0_11), "f"(1.0f)); + float _fma_2; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_2) : "f"(r0_11), "f"(_fma_43), "f"(r0_11)); + float _fma_44; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_44) : "f"(-d1_11), "f"(r1_11), "f"(1.0f)); + float _fma_3; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_3) : "f"(r1_11), "f"(_fma_44), "f"(r1_11)); + float t0_11 = _fma_2; + float t1_11 = _fma_3; + int _warp_vote_45 = __any_sync(0xFFFFFFFF, fminf(t0_11, t1_11) == 0.0f); + if (_warp_vote_45 != 0) { + float d4_11a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_11a) : "f"(d0_11), "f"(0.25f)); + float r4_11a = approx_rcp(d4_11a); + float _fma_47; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_47) : "f"(-d4_11a), "f"(r4_11a), "f"(1.0f)); + float _fma_46; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_46) : "f"(r4_11a), "f"(_fma_47), "f"(r4_11a)); + float big_11a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_11a) : "f"(mul_f32_rn_noftz(_fma_46, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_48; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_48) : "f"(big_11a), "f"(8.388608e+06f)); + float whole_11a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_11a) : "f"(_add_48), "f"(8.388608e+06f)); + float above_11a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_11a) : "f"(-d4_11a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_11a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_50; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_50) : "f"(big_11a), "f"(whole_11a)); + float _abs_49; + asm("abs.f32 %0, %1;" : "=f"(_abs_49) : "f"(_sub_50)); + float _add_51; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_51) : "f"(big_11a), "f"(0.5f)); + float _sub_52; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_52) : "f"(big_11a), "f"(0.5f)); + float rounded_11a = ((_abs_49 == 0.5f) ? ((above_11a > 0.0f) ? _add_51 : _sub_52) : whole_11a); + t0_11 = ((d0_11 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_11a, 7.888609e-31f), 1.7763568e-15f) : t0_11); + float d4_11b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_11b) : "f"(d1_11), "f"(0.25f)); + float r4_11b = approx_rcp(d4_11b); + float _fma_54; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_54) : "f"(-d4_11b), "f"(r4_11b), "f"(1.0f)); + float _fma_53; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_53) : "f"(r4_11b), "f"(_fma_54), "f"(r4_11b)); + float big_11b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_11b) : "f"(mul_f32_rn_noftz(_fma_53, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_55; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_55) : "f"(big_11b), "f"(8.388608e+06f)); + float whole_11b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_11b) : "f"(_add_55), "f"(8.388608e+06f)); + float above_11b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_11b) : "f"(-d4_11b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_11b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_57; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_57) : "f"(big_11b), "f"(whole_11b)); + float _abs_56; + asm("abs.f32 %0, %1;" : "=f"(_abs_56) : "f"(_sub_57)); + float _add_58; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_58) : "f"(big_11b), "f"(0.5f)); + float _sub_59; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_59) : "f"(big_11b), "f"(0.5f)); + float rounded_11b = ((_abs_56 == 0.5f) ? ((above_11b > 0.0f) ? _add_58 : _sub_59) : whole_11b); + t1_11 = ((d1_11 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_11b, 7.888609e-31f), 1.7763568e-15f) : t1_11); + } + float2 s_10 = make_float2(max_noftz(t0_11, 0.0f), max_noftz(t1_11, 0.0f)); + float2 act_10; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_10) : "l"(*(const unsigned long long*)&_cvt_f32_16), "l"(*(const unsigned long long*)&s_10)); + float2 _f32x2_mul_a_60 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_10, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_10), s_10), _cvt_f32_18); + float2 _mul_10; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_60), "l"(*(const unsigned long long*)&_cvt_f32_17)); + float2 _mul_11; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_11) : "l"(*(const unsigned long long*)&act_10), "l"(*(const unsigned long long*)&_cvt_f32_18)); + __nv_bfloat162 _bf16x2_10 = __float22bfloat162_rn(make_float2(_mul_10.x, _mul_10.y)); + __nv_bfloat162 _bf16x2_11 = __float22bfloat162_rn(make_float2(_mul_11.x, _mul_11.y)); + dgate[1] = __as_u32(_bf16x2_10); + dup[1] = __as_u32(_bf16x2_11); + float2 _cvt_f32_19 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[2])); + float2 _cvt_f32_20 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[2])); + float2 _cvt_f32_21 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[2])); + float e0_13 = approx_exp2((-_cvt_f32_19.x) * 1.442695f); + float e1_13 = approx_exp2((-_cvt_f32_19.y) * 1.442695f); + float d0_13 = 1.0f + e0_13; + float d1_13 = 1.0f + e1_13; + float r0_13 = approx_rcp(d0_13); + float r1_13 = approx_rcp(d1_13); + float _fma_61; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_61) : "f"(-d0_13), "f"(r0_13), "f"(1.0f)); + float _fma_4; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_4) : "f"(r0_13), "f"(_fma_61), "f"(r0_13)); + float _fma_62; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_62) : "f"(-d1_13), "f"(r1_13), "f"(1.0f)); + float _fma_5_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_5_1) : "f"(r1_13), "f"(_fma_62), "f"(r1_13)); + float t0_13 = _fma_4; + float t1_13 = _fma_5_1; + int _warp_vote_63 = __any_sync(0xFFFFFFFF, fminf(t0_13, t1_13) == 0.0f); + if (_warp_vote_63 != 0) { + float d4_13a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_13a) : "f"(d0_13), "f"(0.25f)); + float r4_13a = approx_rcp(d4_13a); + float _fma_65; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_65) : "f"(-d4_13a), "f"(r4_13a), "f"(1.0f)); + float _fma_64; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_64) : "f"(r4_13a), "f"(_fma_65), "f"(r4_13a)); + float big_13a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_13a) : "f"(mul_f32_rn_noftz(_fma_64, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_66; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_66) : "f"(big_13a), "f"(8.388608e+06f)); + float whole_13a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_13a) : "f"(_add_66), "f"(8.388608e+06f)); + float above_13a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_13a) : "f"(-d4_13a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_13a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_68; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_68) : "f"(big_13a), "f"(whole_13a)); + float _abs_67; + asm("abs.f32 %0, %1;" : "=f"(_abs_67) : "f"(_sub_68)); + float _add_69; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_69) : "f"(big_13a), "f"(0.5f)); + float _sub_70; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_70) : "f"(big_13a), "f"(0.5f)); + float rounded_13a = ((_abs_67 == 0.5f) ? ((above_13a > 0.0f) ? _add_69 : _sub_70) : whole_13a); + t0_13 = ((d0_13 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_13a, 7.888609e-31f), 1.7763568e-15f) : t0_13); + float d4_13b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_13b) : "f"(d1_13), "f"(0.25f)); + float r4_13b = approx_rcp(d4_13b); + float _fma_72; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_72) : "f"(-d4_13b), "f"(r4_13b), "f"(1.0f)); + float _fma_71; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_71) : "f"(r4_13b), "f"(_fma_72), "f"(r4_13b)); + float big_13b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_13b) : "f"(mul_f32_rn_noftz(_fma_71, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_73; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_73) : "f"(big_13b), "f"(8.388608e+06f)); + float whole_13b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_13b) : "f"(_add_73), "f"(8.388608e+06f)); + float above_13b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_13b) : "f"(-d4_13b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_13b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_75; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_75) : "f"(big_13b), "f"(whole_13b)); + float _abs_74; + asm("abs.f32 %0, %1;" : "=f"(_abs_74) : "f"(_sub_75)); + float _add_76; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_76) : "f"(big_13b), "f"(0.5f)); + float _sub_77; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_77) : "f"(big_13b), "f"(0.5f)); + float rounded_13b = ((_abs_74 == 0.5f) ? ((above_13b > 0.0f) ? _add_76 : _sub_77) : whole_13b); + t1_13 = ((d1_13 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_13b, 7.888609e-31f), 1.7763568e-15f) : t1_13); + } + float2 s_12 = make_float2(max_noftz(t0_13, 0.0f), max_noftz(t1_13, 0.0f)); + float2 act_12; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_12) : "l"(*(const unsigned long long*)&_cvt_f32_19), "l"(*(const unsigned long long*)&s_12)); + float2 _f32x2_mul_a_78 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_12, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_12), s_12), _cvt_f32_21); + float2 _mul_12; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_78), "l"(*(const unsigned long long*)&_cvt_f32_20)); + float2 _mul_13; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_13) : "l"(*(const unsigned long long*)&act_12), "l"(*(const unsigned long long*)&_cvt_f32_21)); + __nv_bfloat162 _bf16x2_12 = __float22bfloat162_rn(make_float2(_mul_12.x, _mul_12.y)); + __nv_bfloat162 _bf16x2_13 = __float22bfloat162_rn(make_float2(_mul_13.x, _mul_13.y)); + dgate[2] = __as_u32(_bf16x2_12); + dup[2] = __as_u32(_bf16x2_13); + float2 _cvt_f32_22 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[3])); + float2 _cvt_f32_23 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[3])); + float2 _cvt_f32_24 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[3])); + float e0_15 = approx_exp2((-_cvt_f32_22.x) * 1.442695f); + float e1_15 = approx_exp2((-_cvt_f32_22.y) * 1.442695f); + float d0_15 = 1.0f + e0_15; + float d1_15 = 1.0f + e1_15; + float r0_15 = approx_rcp(d0_15); + float r1_15 = approx_rcp(d1_15); + float _fma_79; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_79) : "f"(-d0_15), "f"(r0_15), "f"(1.0f)); + float _fma_6_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_6_1) : "f"(r0_15), "f"(_fma_79), "f"(r0_15)); + float _fma_80; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_80) : "f"(-d1_15), "f"(r1_15), "f"(1.0f)); + float _fma_7_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_7_1) : "f"(r1_15), "f"(_fma_80), "f"(r1_15)); + float t0_15 = _fma_6_1; + float t1_15 = _fma_7_1; + int _warp_vote_81 = __any_sync(0xFFFFFFFF, fminf(t0_15, t1_15) == 0.0f); + if (_warp_vote_81 != 0) { + float d4_15a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_15a) : "f"(d0_15), "f"(0.25f)); + float r4_15a = approx_rcp(d4_15a); + float _fma_83; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_83) : "f"(-d4_15a), "f"(r4_15a), "f"(1.0f)); + float _fma_82; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_82) : "f"(r4_15a), "f"(_fma_83), "f"(r4_15a)); + float big_15a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_15a) : "f"(mul_f32_rn_noftz(_fma_82, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_84; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_84) : "f"(big_15a), "f"(8.388608e+06f)); + float whole_15a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_15a) : "f"(_add_84), "f"(8.388608e+06f)); + float above_15a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_15a) : "f"(-d4_15a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_15a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_86; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_86) : "f"(big_15a), "f"(whole_15a)); + float _abs_85; + asm("abs.f32 %0, %1;" : "=f"(_abs_85) : "f"(_sub_86)); + float _add_87; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_87) : "f"(big_15a), "f"(0.5f)); + float _sub_88; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_88) : "f"(big_15a), "f"(0.5f)); + float rounded_15a = ((_abs_85 == 0.5f) ? ((above_15a > 0.0f) ? _add_87 : _sub_88) : whole_15a); + t0_15 = ((d0_15 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_15a, 7.888609e-31f), 1.7763568e-15f) : t0_15); + float d4_15b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_15b) : "f"(d1_15), "f"(0.25f)); + float r4_15b = approx_rcp(d4_15b); + float _fma_90; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_90) : "f"(-d4_15b), "f"(r4_15b), "f"(1.0f)); + float _fma_89; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_89) : "f"(r4_15b), "f"(_fma_90), "f"(r4_15b)); + float big_15b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_15b) : "f"(mul_f32_rn_noftz(_fma_89, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_91; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_91) : "f"(big_15b), "f"(8.388608e+06f)); + float whole_15b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_15b) : "f"(_add_91), "f"(8.388608e+06f)); + float above_15b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_15b) : "f"(-d4_15b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_15b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_93; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_93) : "f"(big_15b), "f"(whole_15b)); + float _abs_92; + asm("abs.f32 %0, %1;" : "=f"(_abs_92) : "f"(_sub_93)); + float _add_94; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_94) : "f"(big_15b), "f"(0.5f)); + float _sub_95; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_95) : "f"(big_15b), "f"(0.5f)); + float rounded_15b = ((_abs_92 == 0.5f) ? ((above_15b > 0.0f) ? _add_94 : _sub_95) : whole_15b); + t1_15 = ((d1_15 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_15b, 7.888609e-31f), 1.7763568e-15f) : t1_15); + } + float2 s_14 = make_float2(max_noftz(t0_15, 0.0f), max_noftz(t1_15, 0.0f)); + float2 act_14; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_14) : "l"(*(const unsigned long long*)&_cvt_f32_22), "l"(*(const unsigned long long*)&s_14)); + float2 _f32x2_mul_a_96 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_14, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_14), s_14), _cvt_f32_24); + float2 _mul_14; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_96), "l"(*(const unsigned long long*)&_cvt_f32_23)); + float2 _mul_15; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_15) : "l"(*(const unsigned long long*)&act_14), "l"(*(const unsigned long long*)&_cvt_f32_24)); + __nv_bfloat162 _bf16x2_14 = __float22bfloat162_rn(make_float2(_mul_14.x, _mul_14.y)); + __nv_bfloat162 _bf16x2_15 = __float22bfloat162_rn(make_float2(_mul_15.x, _mul_15.y)); + dgate[3] = __as_u32(_bf16x2_14); + dup[3] = __as_u32(_bf16x2_15); + } + int scale_cols = K >> 6; + int out_index = grow * (2 * K) + col; + __nv_bfloat162 amax = __as_bf16x2(0); + __nv_bfloat162 _hxa_98 = amax; + __nv_bfloat162 _hxb_98 = __as_bf16x2(dgate[0]); + unsigned int _hxr_98; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_98) : "r"(*reinterpret_cast(&_hxa_98)), "r"(*reinterpret_cast(&_hxb_98))); + __nv_bfloat162 _max_97 = *reinterpret_cast(&_hxr_98); + amax = _max_97; + __nv_bfloat162 _hxa_100 = amax; + __nv_bfloat162 _hxb_100 = __as_bf16x2(dgate[1]); + unsigned int _hxr_100; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_100) : "r"(*reinterpret_cast(&_hxa_100)), "r"(*reinterpret_cast(&_hxb_100))); + __nv_bfloat162 _max_99 = *reinterpret_cast(&_hxr_100); + amax = _max_99; + __nv_bfloat162 _hxa_102 = amax; + __nv_bfloat162 _hxb_102 = __as_bf16x2(dgate[2]); + unsigned int _hxr_102; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_102) : "r"(*reinterpret_cast(&_hxa_102)), "r"(*reinterpret_cast(&_hxb_102))); + __nv_bfloat162 _max_101 = *reinterpret_cast(&_hxr_102); + amax = _max_101; + __nv_bfloat162 _hxa_104 = amax; + __nv_bfloat162 _hxb_104 = __as_bf16x2(dgate[3]); + unsigned int _hxr_104; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_104) : "r"(*reinterpret_cast(&_hxa_104)), "r"(*reinterpret_cast(&_hxb_104))); + __nv_bfloat162 _max_103 = *reinterpret_cast(&_hxr_104); + amax = _max_103; + unsigned int mag = __as_u32(amax) & 2147450879; + unsigned int peer1 = __shfl_xor_sync(0xFFFFFFFF, mag, 1); + __nv_bfloat162 pair_max = __hmax2_nan(__as_bf16x2(mag), __as_bf16x2(peer1)); + unsigned int peer2 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(pair_max), 2); + __nv_bfloat162 quad_max = __hmax2_nan(pair_max, __as_bf16x2(peer2)); + unsigned int folded = __as_u32(__hmax2_nan(quad_max, __as_bf16x2(__as_u32(quad_max) >> 16))); + unsigned int amax_bits = (folded & 65535) << 16; + int scale_i32 = (int)(amax_bits + 2031616 >> 23) - 8; + if (scale_i32 < 0) { + scale_i32 = 0; + } + if ((amax_bits & 2139095040) == 2139095040) { + scale_i32 = 255; + } + unsigned int scale = (unsigned int)scale_i32; + unsigned int inv = 254 - scale << 7; + if (scale == 255) { + inv = 32704; + } + unsigned int inv2 = inv | inv << 16; + int scale_index = ((grow >> 7) * scale_cols + (bx >> 2)) * 512 + (grow & 31) * 16 + (grow >> 5 & 3) * 4 + (bx & 3); + if (quarter == 0) { + *(reinterpret_cast(row_scales + scale_index) + (0)) = (unsigned char)(scale); + } + unsigned int words[2]; + uint32_t _bf16x2_mul_0; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_0) : "r"(dgate[0]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_0; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_0) : "r"(_bf16x2_mul_0)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_0); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_0) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_1; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_1) : "r"(dgate[1]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_106; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_106) : "r"(_bf16x2_mul_1)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_1); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_106) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_105; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_105) : "h"(_e4m3x2_bf16x2_0), "h"(_cvt_satfinite_e4m3x2_bf16x2_106)); + words[0] = _mov_b32_pack_u16x2_105; + uint32_t _bf16x2_mul_2; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_2) : "r"(dgate[2]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_1; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_1) : "r"(_bf16x2_mul_2)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_2); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_1) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_3; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_3) : "r"(dgate[3]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_108; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_108) : "r"(_bf16x2_mul_3)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_3); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_108) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_107; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_107) : "h"(_e4m3x2_bf16x2_1), "h"(_cvt_satfinite_e4m3x2_bf16x2_108)); + words[1] = _mov_b32_pack_u16x2_107; + reinterpret_cast(row_output + out_index)[0] = reinterpret_cast(words + 0)[0]; + __nv_bfloat162 amax_0 = __as_bf16x2(0); + __nv_bfloat162 _hxa_110 = amax_0; + __nv_bfloat162 _hxb_110 = __as_bf16x2(dup[0]); + unsigned int _hxr_110; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_110) : "r"(*reinterpret_cast(&_hxa_110)), "r"(*reinterpret_cast(&_hxb_110))); + __nv_bfloat162 _max_109 = *reinterpret_cast(&_hxr_110); + amax_0 = _max_109; + __nv_bfloat162 _hxa_112 = amax_0; + __nv_bfloat162 _hxb_112 = __as_bf16x2(dup[1]); + unsigned int _hxr_112; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_112) : "r"(*reinterpret_cast(&_hxa_112)), "r"(*reinterpret_cast(&_hxb_112))); + __nv_bfloat162 _max_111 = *reinterpret_cast(&_hxr_112); + amax_0 = _max_111; + __nv_bfloat162 _hxa_114 = amax_0; + __nv_bfloat162 _hxb_114 = __as_bf16x2(dup[2]); + unsigned int _hxr_114; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_114) : "r"(*reinterpret_cast(&_hxa_114)), "r"(*reinterpret_cast(&_hxb_114))); + __nv_bfloat162 _max_113 = *reinterpret_cast(&_hxr_114); + amax_0 = _max_113; + __nv_bfloat162 _hxa_116 = amax_0; + __nv_bfloat162 _hxb_116 = __as_bf16x2(dup[3]); + unsigned int _hxr_116; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_116) : "r"(*reinterpret_cast(&_hxa_116)), "r"(*reinterpret_cast(&_hxb_116))); + __nv_bfloat162 _max_115 = *reinterpret_cast(&_hxr_116); + amax_0 = _max_115; + unsigned int mag_1 = __as_u32(amax_0) & 2147450879; + unsigned int peer1_2 = __shfl_xor_sync(0xFFFFFFFF, mag_1, 1); + __nv_bfloat162 pair_max_3 = __hmax2_nan(__as_bf16x2(mag_1), __as_bf16x2(peer1_2)); + unsigned int peer2_4 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(pair_max_3), 2); + __nv_bfloat162 quad_max_5 = __hmax2_nan(pair_max_3, __as_bf16x2(peer2_4)); + unsigned int folded_6 = __as_u32(__hmax2_nan(quad_max_5, __as_bf16x2(__as_u32(quad_max_5) >> 16))); + unsigned int amax_bits_7 = (folded_6 & 65535) << 16; + int scale_i32_8 = (int)(amax_bits_7 + 2031616 >> 23) - 8; + if (scale_i32_8 < 0) { + scale_i32_8 = 0; + } + if ((amax_bits_7 & 2139095040) == 2139095040) { + scale_i32_8 = 255; + } + unsigned int scale_9 = (unsigned int)scale_i32_8; + unsigned int inv_10 = 254 - scale_9 << 7; + if (scale_9 == 255) { + inv_10 = 32704; + } + unsigned int inv2_11 = inv_10 | inv_10 << 16; + int scale_index_12 = ((grow >> 7) * scale_cols + (bx + (K >> 5) >> 2)) * 512 + (grow & 31) * 16 + (grow >> 5 & 3) * 4 + (bx + (K >> 5) & 3); + if (quarter == 0) { + *(reinterpret_cast(row_scales + scale_index_12) + (0)) = (unsigned char)(scale_9); + } + unsigned int words_13[2]; + uint32_t _bf16x2_mul_4; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_4) : "r"(dup[0]), "r"(inv2_11)); + uint16_t _e4m3x2_bf16x2_2; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_2) : "r"(_bf16x2_mul_4)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_4); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_2) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_5; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_5) : "r"(dup[1]), "r"(inv2_11)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_118; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_118) : "r"(_bf16x2_mul_5)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_5); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_118) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_117; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_117) : "h"(_e4m3x2_bf16x2_2), "h"(_cvt_satfinite_e4m3x2_bf16x2_118)); + words_13[0] = _mov_b32_pack_u16x2_117; + uint32_t _bf16x2_mul_6; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_6) : "r"(dup[2]), "r"(inv2_11)); + uint16_t _e4m3x2_bf16x2_3; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_3) : "r"(_bf16x2_mul_6)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_6); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_3) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_7; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_7) : "r"(dup[3]), "r"(inv2_11)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_120; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_120) : "r"(_bf16x2_mul_7)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_7); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_120) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_119; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_119) : "h"(_e4m3x2_bf16x2_3), "h"(_cvt_satfinite_e4m3x2_bf16x2_120)); + words_13[1] = _mov_b32_pack_u16x2_119; + reinterpret_cast(row_output + (out_index + K))[0] = reinterpret_cast(words_13 + 0)[0]; +} + +} // extern "C" diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_binding.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_binding.cu new file mode 100644 index 00000000000..f97237f3d19 --- /dev/null +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_binding.cu @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2026 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gated_act_mxfp8_9c68fc985d41586ecbe8, for the device source that defines it. Do not edit. +#include +#include + +#include "cake_ffi_checks.h" + +#include + +extern "C" __global__ void kernel_cake_gated_act_mxfp8_9c68fc985d41586ecbe8(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K); + +namespace cake_binding_kernel_cake_gated_act_mxfp8_9c68fc985d41586ecbe8 { + +using tvm::ffi::TensorView; + +constexpr const char* KERNEL_NAME = "kernel_cake_gated_act_mxfp8_9c68fc985d41586ecbe8"; + +void CheckCuda(cudaError_t status, const char* what) { + TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) + << KERNEL_NAME << ": " << what << " failed: " << cudaGetErrorString(status); +} + +void* DataPtr(const TensorView& t) { return static_cast(t.data_ptr()) + t.byte_offset(); } + +void Run(TensorView a_gated_input, TensorView a_grad, TensorView a_row_output, TensorView a_col_output, TensorView a_row_scales, TensorView a_col_scales, int64_t a_M, int64_t a_K, int64_t grid_x, int64_t grid_y, int64_t grid_z) { + DLDevice dev = a_gated_input.device(); + tvm::ffi::CUDADeviceGuard device_guard(dev.device_id); + check_cuda_tensor(a_gated_input, "gated_input"); + check_dtype(a_gated_input, DLDataType{kDLBfloat, 16, 1}, "gated_input"); + check_cuda_tensor(a_grad, "grad"); + check_dtype(a_grad, DLDataType{kDLBfloat, 16, 1}, "grad"); + check_cuda_tensor(a_row_output, "row_output"); + check_dtype(a_row_output, DLDataType{kDLUInt, 8, 1}, "row_output"); + check_cuda_tensor(a_col_output, "col_output"); + check_dtype(a_col_output, DLDataType{kDLUInt, 8, 1}, "col_output"); + check_cuda_tensor(a_row_scales, "row_scales"); + check_dtype(a_row_scales, DLDataType{kDLUInt, 8, 1}, "row_scales"); + check_cuda_tensor(a_col_scales, "col_scales"); + check_dtype(a_col_scales, DLDataType{kDLUInt, 8, 1}, "col_scales"); + TVM_FFI_CHECK(a_M >= -2147483648LL && a_M <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'M' = " << a_M + << " is outside the i32 range [-2147483648, 2147483647]"; + TVM_FFI_CHECK(a_K >= -2147483648LL && a_K <= 2147483647LL, ValueError) + << KERNEL_NAME << ": 'K' = " << a_K + << " is outside the i32 range [-2147483648, 2147483647]"; + check_contiguous(a_gated_input, "gated_input"); + check_contiguous(a_grad, "grad"); + check_same_device(a_grad, a_gated_input, "grad", "gated_input"); + check_contiguous(a_row_output, "row_output"); + check_same_device(a_row_output, a_gated_input, "row_output", "gated_input"); + check_contiguous(a_col_output, "col_output"); + check_same_device(a_col_output, a_gated_input, "col_output", "gated_input"); + check_contiguous(a_row_scales, "row_scales"); + check_same_device(a_row_scales, a_gated_input, "row_scales", "gated_input"); + check_contiguous(a_col_scales, "col_scales"); + check_same_device(a_col_scales, a_gated_input, "col_scales", "gated_input"); + TVM_FFI_CHECK(grid_x > 0 && grid_y > 0 && grid_z > 0, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") must be positive"; + TVM_FFI_CHECK(grid_x <= 2147483647 && grid_y <= 65535 && grid_z <= 65535, ValueError) + << KERNEL_NAME << ": the grid (" << grid_x << ", " << grid_y << ", " << grid_z << ") exceeds CUDA's limits"; + cudaStream_t stream = static_cast(TVMFFIEnvGetStream(dev.device_type, dev.device_id)); + void* v_gated_input = DataPtr(a_gated_input); + void* v_grad = DataPtr(a_grad); + void* v_row_output = DataPtr(a_row_output); + void* v_col_output = DataPtr(a_col_output); + void* v_row_scales = DataPtr(a_row_scales); + void* v_col_scales = DataPtr(a_col_scales); + int32_t v_M = static_cast(a_M); + int32_t v_K = static_cast(a_K); + void* kernel_args[] = {&v_gated_input, &v_grad, &v_row_output, &v_col_output, &v_row_scales, &v_col_scales, &v_M, &v_K}; + cudaLaunchConfig_t config{}; + config.gridDim = dim3(static_cast(grid_x), static_cast(grid_y), static_cast(grid_z)); + config.blockDim = dim3(128, 1, 1); + config.dynamicSmemBytes = 0; + config.stream = stream; + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gated_act_mxfp8_9c68fc985d41586ecbe8), kernel_args), + "cudaLaunchKernelExC"); +} + +} // namespace cake_binding_kernel_cake_gated_act_mxfp8_9c68fc985d41586ecbe8 + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gated_act_mxfp8_9c68fc985d41586ecbe8::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_kernel.cu b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_kernel.cu new file mode 100644 index 00000000000..e4c83c453fc --- /dev/null +++ b/flashinfer/cake_gated_act_mxfp8_backend/csrc/cake_gated_act_mxfp8_9c68fc985d41586ecbe8_kernel.cu @@ -0,0 +1,798 @@ +/* + * Copyright (c) 2026 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +#if defined(__CUDACC_RTC__) +typedef unsigned long long uint64_t; +#else +typedef unsigned long uint64_t; +#endif +static_assert(sizeof(uint64_t) == 8, "Cake requires an LP64 CUDA host ABI"); +typedef signed int int32_t; +typedef short int int16_t; +struct __align__(64) CakeTensorMap64 { uint64_t opaque[16]; }; +static_assert(sizeof(CakeTensorMap64) == 128, "64-aligned tensor-map ABI size"); +static_assert(alignof(CakeTensorMap64) == 64, "64-aligned tensor-map ABI alignment"); + +#if defined(__CUDACC_RTC__) +typedef struct __align__(128) { uint64_t opaque[16]; } CUtensorMap; +#else +#include +#endif + +static_assert(sizeof(CUtensorMap) == 128, "CUtensorMap CUDA ABI must be 128 bytes"); +#include +#include + +__device__ __forceinline__ int make_warp_uniform(int x) { + int result; + asm volatile("shfl.sync.idx.b32 %0, %1, 0, 0x1F, 0xFFFFFFFF;" + : "=r"(result) : "r"(x)); + return result; +} + +#define CAKE_INF CUDART_INF_F +#define NUM_MAIN_STAGES 1 +#define THREADS 128 +#define LAUNCH_MIN_BLOCKS 12 + +#include + +__device__ __forceinline__ float approx_exp2(float x) { + float y; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + +__device__ __forceinline__ float approx_rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + + +__device__ __forceinline__ float max_noftz(float a, float b) { + float c; + asm("max.f32 %0, %1, %2;" : "=f"(c) : "f"(a), "f"(b)); + return c; +} + + +__device__ __forceinline__ float2 sub_f32x2_noftz(float2 a, float2 b) { + float2 r; + asm("sub.f32x2 %0, %1, %2;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(unsigned long long*)&a), "l"(*(unsigned long long*)&b)); + return r; +} + +__device__ __forceinline__ float2 mul_f32x2_noftz(float2 a, float2 b) { + float2 r; + asm("mul.rn.f32x2 %0, %1, %2;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(unsigned long long*)&a), "l"(*(unsigned long long*)&b)); + return r; +} + + +// ex2_emulation_f32x2 defined in softmax_frag_exp2_cast helper (or standalone) + +__device__ __forceinline__ float2 fma_f32x2_rn_noftz(float2 a, float2 b, float2 c) { + float2 r; + asm("fma.rn.f32x2 %0, %1, %2, %3;" + : "=l"(*(unsigned long long*)&r) + : "l"(*(const unsigned long long*)&a), + "l"(*(const unsigned long long*)&b), + "l"(*(const unsigned long long*)&c)); + return r; +} + + +__device__ __forceinline__ unsigned int __as_u32(float v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "f"(v)); + return u; +} +__device__ __forceinline__ unsigned int __as_u32(__nv_bfloat162 v) { + return *reinterpret_cast(&v); +} +__device__ __forceinline__ unsigned int __as_u32(unsigned int v) { return v; } +__device__ __forceinline__ unsigned int __as_u32(int v) { + unsigned int u; + asm("mov.b32 %0, %1;" : "=r"(u) : "r"(v)); + return u; +} + +__device__ __forceinline__ __nv_bfloat162 __as_bf16x2(unsigned int v) { + __nv_bfloat162_raw raw; + raw.x = static_cast(v); + raw.y = static_cast(v >> 16); + return __nv_bfloat162(raw); +} + +__device__ __forceinline__ float2 cake_bf16x2_to_f32x2(__nv_bfloat162 v) { + const unsigned int w = *reinterpret_cast(&v); + return make_float2(__uint_as_float(w << 16), __uint_as_float(w & 0xffff0000u)); +} + +__device__ __forceinline__ float mul_f32_rn_noftz(float a, float b) { + float r; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} + +__device__ __noinline__ unsigned long long band_pair(__nv_bfloat16* src, __nv_bfloat16* dsrc, int index, int grad_index, int k) +{ + unsigned int _vec_load_0[1]; + { + _vec_load_0[0] = *reinterpret_cast(src + index + 0); + } + unsigned int _vec_load_1[1]; + { + _vec_load_1[0] = *reinterpret_cast(src + (index + k) + 0); + } + unsigned int _vec_load_2[1]; + { + _vec_load_2[0] = *reinterpret_cast(dsrc + grad_index + 0); + } + float2 _cvt_f32_0 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_0[0])); + float2 _cvt_f32_1 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_1[0])); + float2 _cvt_f32_2 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_2[0])); + float e0_1 = approx_exp2((-_cvt_f32_0.x) * 1.442695f); + float e1_1 = approx_exp2((-_cvt_f32_0.y) * 1.442695f); + float d0_1 = 1.0f + e0_1; + float d1_1 = 1.0f + e1_1; + float r0_1 = approx_rcp(d0_1); + float r1_1 = approx_rcp(d1_1); + float _fma_0; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_0) : "f"(-d0_1), "f"(r0_1), "f"(1.0f)); + float _fma_0_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_0_1) : "f"(r0_1), "f"(_fma_0), "f"(r0_1)); + float _fma_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_1) : "f"(-d1_1), "f"(r1_1), "f"(1.0f)); + float _fma_1_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_1_1) : "f"(r1_1), "f"(_fma_1), "f"(r1_1)); + float t0_1 = _fma_0_1; + float t1_1 = _fma_1_1; + float d4_1a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_1a) : "f"(d0_1), "f"(0.25f)); + float r4_1a = approx_rcp(d4_1a); + float _fma_3; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_3) : "f"(-d4_1a), "f"(r4_1a), "f"(1.0f)); + float _fma_2; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_2) : "f"(r4_1a), "f"(_fma_3), "f"(r4_1a)); + float big_1a; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_1a) : "f"(mul_f32_rn_noftz(_fma_2, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_4; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_4) : "f"(big_1a), "f"(8.388608e+06f)); + float whole_1a; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_1a) : "f"(_add_4), "f"(8.388608e+06f)); + float above_1a; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_1a) : "f"(-d4_1a), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_1a, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_6; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_6) : "f"(big_1a), "f"(whole_1a)); + float _abs_5; + asm("abs.f32 %0, %1;" : "=f"(_abs_5) : "f"(_sub_6)); + float _add_7; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_7) : "f"(big_1a), "f"(0.5f)); + float _sub_8; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_8) : "f"(big_1a), "f"(0.5f)); + float rounded_1a = ((_abs_5 == 0.5f) ? ((above_1a > 0.0f) ? _add_7 : _sub_8) : whole_1a); + float u0_1 = ((d0_1 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_1a, 7.888609e-31f), 1.7763568e-15f) : t0_1); + float d4_1b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(d4_1b) : "f"(d1_1), "f"(0.25f)); + float r4_1b = approx_rcp(d4_1b); + float _fma_10; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_10) : "f"(-d4_1b), "f"(r4_1b), "f"(1.0f)); + float _fma_9; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_9) : "f"(r4_1b), "f"(_fma_10), "f"(r4_1b)); + float big_1b; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(big_1b) : "f"(mul_f32_rn_noftz(_fma_9, 1.2676506e+30f)), "f"(1.4073749e+14f)); + float _add_11; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_11) : "f"(big_1b), "f"(8.388608e+06f)); + float whole_1b; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(whole_1b) : "f"(_add_11), "f"(8.388608e+06f)); + float above_1b; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(above_1b) : "f"(-d4_1b), "f"(mul_f32_rn_noftz(mul_f32_rn_noftz(big_1b, 7.888609e-31f), 7.1054274e-15f)), "f"(1.0f)); + float _sub_13; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_13) : "f"(big_1b), "f"(whole_1b)); + float _abs_12; + asm("abs.f32 %0, %1;" : "=f"(_abs_12) : "f"(_sub_13)); + float _add_14; + asm("add.rn.f32 %0, %1, %2;" : "=f"(_add_14) : "f"(big_1b), "f"(0.5f)); + float _sub_15; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(_sub_15) : "f"(big_1b), "f"(0.5f)); + float rounded_1b = ((_abs_12 == 0.5f) ? ((above_1b > 0.0f) ? _add_14 : _sub_15) : whole_1b); + float u1_1 = ((d1_1 >= 8.507059e+37f) ? mul_f32_rn_noftz(mul_f32_rn_noftz(rounded_1b, 7.888609e-31f), 1.7763568e-15f) : t1_1); + float2 s_0 = make_float2(max_noftz(u0_1, 0.0f), max_noftz(u1_1, 0.0f)); + float2 act_0; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_0) : "l"(*(const unsigned long long*)&_cvt_f32_0), "l"(*(const unsigned long long*)&s_0)); + float2 _f32x2_mul_a_16 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_0, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_0), s_0), _cvt_f32_2); + float2 _mul_0; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_0) : "l"(*(const unsigned long long*)&_f32x2_mul_a_16), "l"(*(const unsigned long long*)&_cvt_f32_1)); + float2 _mul_1; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_1) : "l"(*(const unsigned long long*)&act_0), "l"(*(const unsigned long long*)&_cvt_f32_2)); + unsigned long long band_pair_out = (unsigned long long)__as_u32(__float22bfloat162_rn(make_float2(_mul_0.x, _mul_0.y))) | (unsigned long long)__as_u32(__float22bfloat162_rn(make_float2(_mul_1.x, _mul_1.y))) << 32; + return band_pair_out; +} + +extern "C" { + +__global__ __launch_bounds__(128, LAUNCH_MIN_BLOCKS) void +kernel_cake_gated_act_mxfp8_9c68fc985d41586ecbe8(__nv_bfloat16* __restrict__ gated_input, __nv_bfloat16* __restrict__ grad, uint8_t* __restrict__ row_output, uint8_t* __restrict__ col_output, uint8_t* __restrict__ row_scales, uint8_t* __restrict__ col_scales, int M, int K) +{ + const int tid = threadIdx.x; + const int warp = make_warp_uniform(tid / 32); + const int lane = tid % 32; + + + const int bid = blockIdx.x; + const int num_bids = gridDim.x; + + const int cta_rank = 0; + + // === Task calls (declaration order) === + int quarter = tid & 3; + int row = tid >> 2; + int bx = blockIdx.x; + int by = blockIdx.y; + int grow = by * 32 + row; + int col = bx * 32 + quarter * 8; + int gate_index = grow * (2 * K) + col; + unsigned int _vec_load_3[4]; + { + uint32_t _uv4_0_0; + uint32_t _uv4_0_1; + uint32_t _uv4_0_2; + uint32_t _uv4_0_3; + asm volatile("ld.global.L2::cache_hint.v4.b32 {%0, %1, %2, %3}, [%4], %5;" + : "=r"(_uv4_0_0), "=r"(_uv4_0_1), "=r"(_uv4_0_2), "=r"(_uv4_0_3) : "l"((const void*)(gated_input + gate_index + (0))), "l"(0x12F0000000000000ULL) : "memory"); + _vec_load_3[0 + 0] = _uv4_0_0; + _vec_load_3[0 + 1] = _uv4_0_1; + _vec_load_3[0 + 2] = _uv4_0_2; + _vec_load_3[0 + 3] = _uv4_0_3; + } + unsigned int _vec_load_4[4]; + { + uint32_t _uv4_1_0; + uint32_t _uv4_1_1; + uint32_t _uv4_1_2; + uint32_t _uv4_1_3; + asm volatile("ld.global.L2::cache_hint.v4.b32 {%0, %1, %2, %3}, [%4], %5;" + : "=r"(_uv4_1_0), "=r"(_uv4_1_1), "=r"(_uv4_1_2), "=r"(_uv4_1_3) : "l"((const void*)(gated_input + (gate_index + K) + (0))), "l"(0x12F0000000000000ULL) : "memory"); + _vec_load_4[0 + 0] = _uv4_1_0; + _vec_load_4[0 + 1] = _uv4_1_1; + _vec_load_4[0 + 2] = _uv4_1_2; + _vec_load_4[0 + 3] = _uv4_1_3; + } + unsigned int _vec_load_5[4]; + { + uint32_t _uv4_2_0; + uint32_t _uv4_2_1; + uint32_t _uv4_2_2; + uint32_t _uv4_2_3; + asm volatile("ld.global.L2::cache_hint.v4.b32 {%0, %1, %2, %3}, [%4], %5;" + : "=r"(_uv4_2_0), "=r"(_uv4_2_1), "=r"(_uv4_2_2), "=r"(_uv4_2_3) : "l"((const void*)(grad + (grow * K + col) + (0))), "l"(0x12F0000000000000ULL) : "memory"); + _vec_load_5[0 + 0] = _uv4_2_0; + _vec_load_5[0 + 1] = _uv4_2_1; + _vec_load_5[0 + 2] = _uv4_2_2; + _vec_load_5[0 + 3] = _uv4_2_3; + } + unsigned int dgate[4]; + unsigned int dup[4]; + __nv_bfloat162 lowest = __as_bf16x2(_vec_load_3[0]); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_3[1])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_3[2])); + lowest = __hmin2_nan(lowest, __as_bf16x2(_vec_load_3[3])); + float2 _cvt_f32_3 = cake_bf16x2_to_f32x2(lowest); + float _min_4; + asm("min.NaN.f32 %0, %1, %2;" : "=f"(_min_4) : "f"(_cvt_f32_3.x), "f"(_cvt_f32_3.y)); + int _warp_vote_3 = __all_sync(0xFFFFFFFF, _min_4 > -87.0f); + if (_warp_vote_3 != 0) { + float2 _cvt_f32_4 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[0])); + float2 _cvt_f32_5 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[0])); + float2 _cvt_f32_6 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[0])); + float e0_3 = approx_exp2((-_cvt_f32_4.x) * 1.442695f); + float e1_3 = approx_exp2((-_cvt_f32_4.y) * 1.442695f); + float d0_3 = 1.0f + e0_3; + float d1_3 = 1.0f + e1_3; + float r0_3 = approx_rcp(d0_3); + float r1_3 = approx_rcp(d1_3); + float _fma_6; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_6) : "f"(-d0_3), "f"(r0_3), "f"(1.0f)); + float _fma_5; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_5) : "f"(r0_3), "f"(_fma_6), "f"(r0_3)); + float _fma_8; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_8) : "f"(-d1_3), "f"(r1_3), "f"(1.0f)); + float _fma_7; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_7) : "f"(r1_3), "f"(_fma_8), "f"(r1_3)); + float2 s_2 = make_float2(_fma_5, _fma_7); + float2 act_2; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_2) : "l"(*(const unsigned long long*)&_cvt_f32_4), "l"(*(const unsigned long long*)&s_2)); + float2 _f32x2_mul_a_9 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_2, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_2), s_2), _cvt_f32_6); + float2 _mul_2; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_2) : "l"(*(const unsigned long long*)&_f32x2_mul_a_9), "l"(*(const unsigned long long*)&_cvt_f32_5)); + float2 _mul_3; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_3) : "l"(*(const unsigned long long*)&act_2), "l"(*(const unsigned long long*)&_cvt_f32_6)); + __nv_bfloat162 _bf16x2_0 = __float22bfloat162_rn(make_float2(_mul_2.x, _mul_2.y)); + __nv_bfloat162 _bf16x2_1 = __float22bfloat162_rn(make_float2(_mul_3.x, _mul_3.y)); + dgate[0] = __as_u32(_bf16x2_0); + dup[0] = __as_u32(_bf16x2_1); + float2 _cvt_f32_7 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[1])); + float2 _cvt_f32_8 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[1])); + float2 _cvt_f32_9 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[1])); + float e0_5 = approx_exp2((-_cvt_f32_7.x) * 1.442695f); + float e1_5 = approx_exp2((-_cvt_f32_7.y) * 1.442695f); + float d0_5 = 1.0f + e0_5; + float d1_5 = 1.0f + e1_5; + float r0_5 = approx_rcp(d0_5); + float r1_5 = approx_rcp(d1_5); + float _fma_11; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_11) : "f"(-d0_5), "f"(r0_5), "f"(1.0f)); + float _fma_10; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_10) : "f"(r0_5), "f"(_fma_11), "f"(r0_5)); + float _fma_13; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_13) : "f"(-d1_5), "f"(r1_5), "f"(1.0f)); + float _fma_12; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_12) : "f"(r1_5), "f"(_fma_13), "f"(r1_5)); + float2 s_4 = make_float2(_fma_10, _fma_12); + float2 act_4; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_4) : "l"(*(const unsigned long long*)&_cvt_f32_7), "l"(*(const unsigned long long*)&s_4)); + float2 _f32x2_mul_a_14 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_4, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_4), s_4), _cvt_f32_9); + float2 _mul_4; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_4) : "l"(*(const unsigned long long*)&_f32x2_mul_a_14), "l"(*(const unsigned long long*)&_cvt_f32_8)); + float2 _mul_5; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_5) : "l"(*(const unsigned long long*)&act_4), "l"(*(const unsigned long long*)&_cvt_f32_9)); + __nv_bfloat162 _bf16x2_2 = __float22bfloat162_rn(make_float2(_mul_4.x, _mul_4.y)); + __nv_bfloat162 _bf16x2_3 = __float22bfloat162_rn(make_float2(_mul_5.x, _mul_5.y)); + dgate[1] = __as_u32(_bf16x2_2); + dup[1] = __as_u32(_bf16x2_3); + float2 _cvt_f32_10 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[2])); + float2 _cvt_f32_11 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[2])); + float2 _cvt_f32_12 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[2])); + float e0_7 = approx_exp2((-_cvt_f32_10.x) * 1.442695f); + float e1_7 = approx_exp2((-_cvt_f32_10.y) * 1.442695f); + float d0_7 = 1.0f + e0_7; + float d1_7 = 1.0f + e1_7; + float r0_7 = approx_rcp(d0_7); + float r1_7 = approx_rcp(d1_7); + float _fma_16; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_16) : "f"(-d0_7), "f"(r0_7), "f"(1.0f)); + float _fma_15; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_15) : "f"(r0_7), "f"(_fma_16), "f"(r0_7)); + float _fma_18; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_18) : "f"(-d1_7), "f"(r1_7), "f"(1.0f)); + float _fma_17; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_17) : "f"(r1_7), "f"(_fma_18), "f"(r1_7)); + float2 s_6 = make_float2(_fma_15, _fma_17); + float2 act_6; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_6) : "l"(*(const unsigned long long*)&_cvt_f32_10), "l"(*(const unsigned long long*)&s_6)); + float2 _f32x2_mul_a_19 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_6, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_6), s_6), _cvt_f32_12); + float2 _mul_6; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_6) : "l"(*(const unsigned long long*)&_f32x2_mul_a_19), "l"(*(const unsigned long long*)&_cvt_f32_11)); + float2 _mul_7; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_7) : "l"(*(const unsigned long long*)&act_6), "l"(*(const unsigned long long*)&_cvt_f32_12)); + __nv_bfloat162 _bf16x2_4 = __float22bfloat162_rn(make_float2(_mul_6.x, _mul_6.y)); + __nv_bfloat162 _bf16x2_5 = __float22bfloat162_rn(make_float2(_mul_7.x, _mul_7.y)); + dgate[2] = __as_u32(_bf16x2_4); + dup[2] = __as_u32(_bf16x2_5); + float2 _cvt_f32_13 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[3])); + float2 _cvt_f32_14 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[3])); + float2 _cvt_f32_15 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[3])); + float e0_9 = approx_exp2((-_cvt_f32_13.x) * 1.442695f); + float e1_9 = approx_exp2((-_cvt_f32_13.y) * 1.442695f); + float d0_9 = 1.0f + e0_9; + float d1_9 = 1.0f + e1_9; + float r0_9 = approx_rcp(d0_9); + float r1_9 = approx_rcp(d1_9); + float _fma_21; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_21) : "f"(-d0_9), "f"(r0_9), "f"(1.0f)); + float _fma_20; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_20) : "f"(r0_9), "f"(_fma_21), "f"(r0_9)); + float _fma_23; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_23) : "f"(-d1_9), "f"(r1_9), "f"(1.0f)); + float _fma_22; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_22) : "f"(r1_9), "f"(_fma_23), "f"(r1_9)); + float2 s_8 = make_float2(_fma_20, _fma_22); + float2 act_8; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_8) : "l"(*(const unsigned long long*)&_cvt_f32_13), "l"(*(const unsigned long long*)&s_8)); + float2 _f32x2_mul_a_24 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_8, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_8), s_8), _cvt_f32_15); + float2 _mul_8; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_8) : "l"(*(const unsigned long long*)&_f32x2_mul_a_24), "l"(*(const unsigned long long*)&_cvt_f32_14)); + float2 _mul_9; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_9) : "l"(*(const unsigned long long*)&act_8), "l"(*(const unsigned long long*)&_cvt_f32_15)); + __nv_bfloat162 _bf16x2_6 = __float22bfloat162_rn(make_float2(_mul_8.x, _mul_8.y)); + __nv_bfloat162 _bf16x2_7 = __float22bfloat162_rn(make_float2(_mul_9.x, _mul_9.y)); + dgate[3] = __as_u32(_bf16x2_6); + dup[3] = __as_u32(_bf16x2_7); + } else { + int band = 0; + float2 _cvt_f32_16 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[0])); + float2 _cvt_f32_17 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[0])); + float2 _cvt_f32_18 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[0])); + float e0_11 = approx_exp2((-_cvt_f32_16.x) * 1.442695f); + float e1_11 = approx_exp2((-_cvt_f32_16.y) * 1.442695f); + float d0_11 = 1.0f + e0_11; + float d1_11 = 1.0f + e1_11; + float r0_11 = approx_rcp(d0_11); + float r1_11 = approx_rcp(d1_11); + float _fma_25; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_25) : "f"(-d0_11), "f"(r0_11), "f"(1.0f)); + float _fma_2; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_2) : "f"(r0_11), "f"(_fma_25), "f"(r0_11)); + float _fma_26; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_26) : "f"(-d1_11), "f"(r1_11), "f"(1.0f)); + float _fma_3; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_3) : "f"(r1_11), "f"(_fma_26), "f"(r1_11)); + float t0_11 = _fma_2; + float t1_11 = _fma_3; + band = band | ((fminf(t0_11, t1_11) == 0.0f) ? 1 : 0); + float2 s_10 = make_float2(max_noftz(t0_11, 0.0f), max_noftz(t1_11, 0.0f)); + float2 act_10; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_10) : "l"(*(const unsigned long long*)&_cvt_f32_16), "l"(*(const unsigned long long*)&s_10)); + float2 _f32x2_mul_a_27 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_10, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_10), s_10), _cvt_f32_18); + float2 _mul_10; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_10) : "l"(*(const unsigned long long*)&_f32x2_mul_a_27), "l"(*(const unsigned long long*)&_cvt_f32_17)); + float2 _mul_11; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_11) : "l"(*(const unsigned long long*)&act_10), "l"(*(const unsigned long long*)&_cvt_f32_18)); + __nv_bfloat162 _bf16x2_8 = __float22bfloat162_rn(make_float2(_mul_10.x, _mul_10.y)); + __nv_bfloat162 _bf16x2_9 = __float22bfloat162_rn(make_float2(_mul_11.x, _mul_11.y)); + dgate[0] = __as_u32(_bf16x2_8); + dup[0] = __as_u32(_bf16x2_9); + float2 _cvt_f32_19 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[1])); + float2 _cvt_f32_20 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[1])); + float2 _cvt_f32_21 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[1])); + float e0_13 = approx_exp2((-_cvt_f32_19.x) * 1.442695f); + float e1_13 = approx_exp2((-_cvt_f32_19.y) * 1.442695f); + float d0_13 = 1.0f + e0_13; + float d1_13 = 1.0f + e1_13; + float r0_13 = approx_rcp(d0_13); + float r1_13 = approx_rcp(d1_13); + float _fma_28; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_28) : "f"(-d0_13), "f"(r0_13), "f"(1.0f)); + float _fma_4; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_4) : "f"(r0_13), "f"(_fma_28), "f"(r0_13)); + float _fma_29; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_29) : "f"(-d1_13), "f"(r1_13), "f"(1.0f)); + float _fma_5_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_5_1) : "f"(r1_13), "f"(_fma_29), "f"(r1_13)); + float t0_13 = _fma_4; + float t1_13 = _fma_5_1; + band = band | ((fminf(t0_13, t1_13) == 0.0f) ? 2 : 0); + float2 s_12 = make_float2(max_noftz(t0_13, 0.0f), max_noftz(t1_13, 0.0f)); + float2 act_12; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_12) : "l"(*(const unsigned long long*)&_cvt_f32_19), "l"(*(const unsigned long long*)&s_12)); + float2 _f32x2_mul_a_30 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_12, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_12), s_12), _cvt_f32_21); + float2 _mul_12; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_12) : "l"(*(const unsigned long long*)&_f32x2_mul_a_30), "l"(*(const unsigned long long*)&_cvt_f32_20)); + float2 _mul_13; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_13) : "l"(*(const unsigned long long*)&act_12), "l"(*(const unsigned long long*)&_cvt_f32_21)); + __nv_bfloat162 _bf16x2_10 = __float22bfloat162_rn(make_float2(_mul_12.x, _mul_12.y)); + __nv_bfloat162 _bf16x2_11 = __float22bfloat162_rn(make_float2(_mul_13.x, _mul_13.y)); + dgate[1] = __as_u32(_bf16x2_10); + dup[1] = __as_u32(_bf16x2_11); + float2 _cvt_f32_22 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[2])); + float2 _cvt_f32_23 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[2])); + float2 _cvt_f32_24 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[2])); + float e0_15 = approx_exp2((-_cvt_f32_22.x) * 1.442695f); + float e1_15 = approx_exp2((-_cvt_f32_22.y) * 1.442695f); + float d0_15 = 1.0f + e0_15; + float d1_15 = 1.0f + e1_15; + float r0_15 = approx_rcp(d0_15); + float r1_15 = approx_rcp(d1_15); + float _fma_31; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_31) : "f"(-d0_15), "f"(r0_15), "f"(1.0f)); + float _fma_6_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_6_1) : "f"(r0_15), "f"(_fma_31), "f"(r0_15)); + float _fma_32; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_32) : "f"(-d1_15), "f"(r1_15), "f"(1.0f)); + float _fma_7_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_7_1) : "f"(r1_15), "f"(_fma_32), "f"(r1_15)); + float t0_15 = _fma_6_1; + float t1_15 = _fma_7_1; + band = band | ((fminf(t0_15, t1_15) == 0.0f) ? 4 : 0); + float2 s_14 = make_float2(max_noftz(t0_15, 0.0f), max_noftz(t1_15, 0.0f)); + float2 act_14; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_14) : "l"(*(const unsigned long long*)&_cvt_f32_22), "l"(*(const unsigned long long*)&s_14)); + float2 _f32x2_mul_a_33 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_14, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_14), s_14), _cvt_f32_24); + float2 _mul_14; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_14) : "l"(*(const unsigned long long*)&_f32x2_mul_a_33), "l"(*(const unsigned long long*)&_cvt_f32_23)); + float2 _mul_15; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_15) : "l"(*(const unsigned long long*)&act_14), "l"(*(const unsigned long long*)&_cvt_f32_24)); + __nv_bfloat162 _bf16x2_12 = __float22bfloat162_rn(make_float2(_mul_14.x, _mul_14.y)); + __nv_bfloat162 _bf16x2_13 = __float22bfloat162_rn(make_float2(_mul_15.x, _mul_15.y)); + dgate[2] = __as_u32(_bf16x2_12); + dup[2] = __as_u32(_bf16x2_13); + float2 _cvt_f32_25 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_3[3])); + float2 _cvt_f32_26 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_4[3])); + float2 _cvt_f32_27 = cake_bf16x2_to_f32x2(__as_bf16x2(_vec_load_5[3])); + float e0_17 = approx_exp2((-_cvt_f32_25.x) * 1.442695f); + float e1_17 = approx_exp2((-_cvt_f32_25.y) * 1.442695f); + float d0_17 = 1.0f + e0_17; + float d1_17 = 1.0f + e1_17; + float r0_17 = approx_rcp(d0_17); + float r1_17 = approx_rcp(d1_17); + float _fma_34; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_34) : "f"(-d0_17), "f"(r0_17), "f"(1.0f)); + float _fma_8_1; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_8_1) : "f"(r0_17), "f"(_fma_34), "f"(r0_17)); + float _fma_35; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_35) : "f"(-d1_17), "f"(r1_17), "f"(1.0f)); + float _fma_9; + asm("fma.rn.f32 %0, %1, %2, %3;" : "=f"(_fma_9) : "f"(r1_17), "f"(_fma_35), "f"(r1_17)); + float t0_17 = _fma_8_1; + float t1_17 = _fma_9; + band = band | ((fminf(t0_17, t1_17) == 0.0f) ? 8 : 0); + float2 s_16 = make_float2(max_noftz(t0_17, 0.0f), max_noftz(t1_17, 0.0f)); + float2 act_16; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&act_16) : "l"(*(const unsigned long long*)&_cvt_f32_25), "l"(*(const unsigned long long*)&s_16)); + float2 _f32x2_mul_a_36 = mul_f32x2_noftz(fma_f32x2_rn_noftz(act_16, sub_f32x2_noftz(make_float2(1.0f, 1.0f), s_16), s_16), _cvt_f32_27); + float2 _mul_16; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_16) : "l"(*(const unsigned long long*)&_f32x2_mul_a_36), "l"(*(const unsigned long long*)&_cvt_f32_26)); + float2 _mul_17; + asm("mul.rn.f32x2 %0, %1, %2;" : "=l"(*(unsigned long long*)&_mul_17) : "l"(*(const unsigned long long*)&act_16), "l"(*(const unsigned long long*)&_cvt_f32_27)); + __nv_bfloat162 _bf16x2_14 = __float22bfloat162_rn(make_float2(_mul_16.x, _mul_16.y)); + __nv_bfloat162 _bf16x2_15 = __float22bfloat162_rn(make_float2(_mul_17.x, _mul_17.y)); + dgate[3] = __as_u32(_bf16x2_14); + dup[3] = __as_u32(_bf16x2_15); + if (band != 0) { + unsigned int _popc_0 = (unsigned int)__popc(band); + #pragma unroll 1 + for (int _j = 0; _j < _popc_0; _j++) { + int _find_first_set_37 = __ffs(band); + int j = _find_first_set_37 - 1; + band = band & band - 1; + unsigned long long both = band_pair(gated_input, grad, gate_index + 2 * j, grow * K + col + 2 * j, K); + unsigned int dgate_new = (unsigned int)both; + unsigned int dup_new = (unsigned int)(both >> 32); + dgate[0] = ((j == 0) ? dgate_new : dgate[0]); + dup[0] = ((j == 0) ? dup_new : dup[0]); + dgate[1] = ((j == 1) ? dgate_new : dgate[1]); + dup[1] = ((j == 1) ? dup_new : dup[1]); + dgate[2] = ((j == 2) ? dgate_new : dgate[2]); + dup[2] = ((j == 2) ? dup_new : dup[2]); + dgate[3] = ((j == 3) ? dgate_new : dgate[3]); + dup[3] = ((j == 3) ? dup_new : dup[3]); + } + } + } + int scale_cols = K >> 6; + int out_index = grow * (2 * K) + col; + __nv_bfloat162 amax = __as_bf16x2(0); + __nv_bfloat162 _hxa_39 = amax; + __nv_bfloat162 _hxb_39 = __as_bf16x2(dgate[0]); + unsigned int _hxr_39; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_39) : "r"(*reinterpret_cast(&_hxa_39)), "r"(*reinterpret_cast(&_hxb_39))); + __nv_bfloat162 _max_38 = *reinterpret_cast(&_hxr_39); + amax = _max_38; + __nv_bfloat162 _hxa_41 = amax; + __nv_bfloat162 _hxb_41 = __as_bf16x2(dgate[1]); + unsigned int _hxr_41; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_41) : "r"(*reinterpret_cast(&_hxa_41)), "r"(*reinterpret_cast(&_hxb_41))); + __nv_bfloat162 _max_40 = *reinterpret_cast(&_hxr_41); + amax = _max_40; + __nv_bfloat162 _hxa_43 = amax; + __nv_bfloat162 _hxb_43 = __as_bf16x2(dgate[2]); + unsigned int _hxr_43; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_43) : "r"(*reinterpret_cast(&_hxa_43)), "r"(*reinterpret_cast(&_hxb_43))); + __nv_bfloat162 _max_42 = *reinterpret_cast(&_hxr_43); + amax = _max_42; + __nv_bfloat162 _hxa_45 = amax; + __nv_bfloat162 _hxb_45 = __as_bf16x2(dgate[3]); + unsigned int _hxr_45; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_45) : "r"(*reinterpret_cast(&_hxa_45)), "r"(*reinterpret_cast(&_hxb_45))); + __nv_bfloat162 _max_44 = *reinterpret_cast(&_hxr_45); + amax = _max_44; + unsigned int mag = __as_u32(amax) & 2147450879; + unsigned int peer1 = __shfl_xor_sync(0xFFFFFFFF, mag, 1); + __nv_bfloat162 pair_max = __hmax2_nan(__as_bf16x2(mag), __as_bf16x2(peer1)); + unsigned int peer2 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(pair_max), 2); + __nv_bfloat162 quad_max = __hmax2_nan(pair_max, __as_bf16x2(peer2)); + unsigned int folded = __as_u32(__hmax2_nan(quad_max, __as_bf16x2(__as_u32(quad_max) >> 16))); + unsigned int amax_bits = (folded & 65535) << 16; + int scale_i32 = (int)(amax_bits + 2031616 >> 23) - 8; + if (scale_i32 < 0) { + scale_i32 = 0; + } + if ((amax_bits & 2139095040) == 2139095040) { + scale_i32 = 255; + } + unsigned int scale = (unsigned int)scale_i32; + unsigned int inv = 254 - scale << 7; + if (scale == 255) { + inv = 32704; + } + unsigned int inv2 = inv | inv << 16; + int scale_index = ((grow >> 7) * scale_cols + (bx >> 2)) * 512 + (grow & 31) * 16 + (grow >> 5 & 3) * 4 + (bx & 3); + if (quarter == 0) { + *(reinterpret_cast(row_scales + scale_index) + (0)) = (unsigned char)(scale); + } + unsigned int words[2]; + uint32_t _bf16x2_mul_0; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_0) : "r"(dgate[0]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_0; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_0) : "r"(_bf16x2_mul_0)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_0); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_0) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_1; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_1) : "r"(dgate[1]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_47; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_47) : "r"(_bf16x2_mul_1)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_1); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_47) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_46; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_46) : "h"(_e4m3x2_bf16x2_0), "h"(_cvt_satfinite_e4m3x2_bf16x2_47)); + words[0] = _mov_b32_pack_u16x2_46; + uint32_t _bf16x2_mul_2; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_2) : "r"(dgate[2]), "r"(inv2)); + uint16_t _e4m3x2_bf16x2_1; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_1) : "r"(_bf16x2_mul_2)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_2); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_1) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_3; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_3) : "r"(dgate[3]), "r"(inv2)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_49; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_49) : "r"(_bf16x2_mul_3)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_3); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_49) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_48; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_48) : "h"(_e4m3x2_bf16x2_1), "h"(_cvt_satfinite_e4m3x2_bf16x2_49)); + words[1] = _mov_b32_pack_u16x2_48; + reinterpret_cast(row_output + out_index)[0] = reinterpret_cast(words + 0)[0]; + __nv_bfloat162 amax_0 = __as_bf16x2(0); + __nv_bfloat162 _hxa_51 = amax_0; + __nv_bfloat162 _hxb_51 = __as_bf16x2(dup[0]); + unsigned int _hxr_51; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_51) : "r"(*reinterpret_cast(&_hxa_51)), "r"(*reinterpret_cast(&_hxb_51))); + __nv_bfloat162 _max_50 = *reinterpret_cast(&_hxr_51); + amax_0 = _max_50; + __nv_bfloat162 _hxa_53 = amax_0; + __nv_bfloat162 _hxb_53 = __as_bf16x2(dup[1]); + unsigned int _hxr_53; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_53) : "r"(*reinterpret_cast(&_hxa_53)), "r"(*reinterpret_cast(&_hxb_53))); + __nv_bfloat162 _max_52 = *reinterpret_cast(&_hxr_53); + amax_0 = _max_52; + __nv_bfloat162 _hxa_55 = amax_0; + __nv_bfloat162 _hxb_55 = __as_bf16x2(dup[2]); + unsigned int _hxr_55; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_55) : "r"(*reinterpret_cast(&_hxa_55)), "r"(*reinterpret_cast(&_hxb_55))); + __nv_bfloat162 _max_54 = *reinterpret_cast(&_hxr_55); + amax_0 = _max_54; + __nv_bfloat162 _hxa_57 = amax_0; + __nv_bfloat162 _hxb_57 = __as_bf16x2(dup[3]); + unsigned int _hxr_57; + asm("max.NaN.xorsign.abs.bf16x2 %0, %1, %2;" : "=r"(_hxr_57) : "r"(*reinterpret_cast(&_hxa_57)), "r"(*reinterpret_cast(&_hxb_57))); + __nv_bfloat162 _max_56 = *reinterpret_cast(&_hxr_57); + amax_0 = _max_56; + unsigned int mag_1 = __as_u32(amax_0) & 2147450879; + unsigned int peer1_2 = __shfl_xor_sync(0xFFFFFFFF, mag_1, 1); + __nv_bfloat162 pair_max_3 = __hmax2_nan(__as_bf16x2(mag_1), __as_bf16x2(peer1_2)); + unsigned int peer2_4 = __shfl_xor_sync(0xFFFFFFFF, __as_u32(pair_max_3), 2); + __nv_bfloat162 quad_max_5 = __hmax2_nan(pair_max_3, __as_bf16x2(peer2_4)); + unsigned int folded_6 = __as_u32(__hmax2_nan(quad_max_5, __as_bf16x2(__as_u32(quad_max_5) >> 16))); + unsigned int amax_bits_7 = (folded_6 & 65535) << 16; + int scale_i32_8 = (int)(amax_bits_7 + 2031616 >> 23) - 8; + if (scale_i32_8 < 0) { + scale_i32_8 = 0; + } + if ((amax_bits_7 & 2139095040) == 2139095040) { + scale_i32_8 = 255; + } + unsigned int scale_9 = (unsigned int)scale_i32_8; + unsigned int inv_10 = 254 - scale_9 << 7; + if (scale_9 == 255) { + inv_10 = 32704; + } + unsigned int inv2_11 = inv_10 | inv_10 << 16; + int scale_index_12 = ((grow >> 7) * scale_cols + (bx + (K >> 5) >> 2)) * 512 + (grow & 31) * 16 + (grow >> 5 & 3) * 4 + (bx + (K >> 5) & 3); + if (quarter == 0) { + *(reinterpret_cast(row_scales + scale_index_12) + (0)) = (unsigned char)(scale_9); + } + unsigned int words_13[2]; + uint32_t _bf16x2_mul_4; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_4) : "r"(dup[0]), "r"(inv2_11)); + uint16_t _e4m3x2_bf16x2_2; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_2) : "r"(_bf16x2_mul_4)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_4); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_2) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_5; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_5) : "r"(dup[1]), "r"(inv2_11)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_59; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_59) : "r"(_bf16x2_mul_5)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_5); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_59) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_58; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_58) : "h"(_e4m3x2_bf16x2_2), "h"(_cvt_satfinite_e4m3x2_bf16x2_59)); + words_13[0] = _mov_b32_pack_u16x2_58; + uint32_t _bf16x2_mul_6; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_6) : "r"(dup[2]), "r"(inv2_11)); + uint16_t _e4m3x2_bf16x2_3; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_e4m3x2_bf16x2_3) : "r"(_bf16x2_mul_6)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_6); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_e4m3x2_bf16x2_3) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _bf16x2_mul_7; + asm volatile("mul.rn.bf16x2 %0, %1, %2;" : "=r"(_bf16x2_mul_7) : "r"(dup[3]), "r"(inv2_11)); + uint16_t _cvt_satfinite_e4m3x2_bf16x2_61; + #if !defined(CAKE_CVT_SATFINITE_E4M3X2_BF16X2_FALLBACK) && defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13 || (__CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ >= 1)) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + asm("cvt.rn.satfinite.e4m3x2.bf16x2 %0, %1;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_61) : "r"(_bf16x2_mul_7)); + #else + { + const uint32_t _bf16x2 = (uint32_t)(_bf16x2_mul_7); + const float _lo = __uint_as_float(_bf16x2 << 16); + const float _hi = __uint_as_float(_bf16x2 & 0xFFFF0000u); + asm("cvt.rn.satfinite.e4m3x2.f32 %0, %1, %2;" : "=h"(_cvt_satfinite_e4m3x2_bf16x2_61) : "f"(_hi), "f"(_lo)); + } + #endif + uint32_t _mov_b32_pack_u16x2_60; + asm("mov.b32 %0, {%1, %2};" : "=r"(_mov_b32_pack_u16x2_60) : "h"(_e4m3x2_bf16x2_3), "h"(_cvt_satfinite_e4m3x2_bf16x2_61)); + words_13[1] = _mov_b32_pack_u16x2_60; + reinterpret_cast(row_output + (out_index + K))[0] = reinterpret_cast(words_13 + 0)[0]; +} + +} // extern "C"